fix(server): retry initial connection to postgres

coadler · coadler · commit 615b33ac5b8c · 2023-04-28T18:06:02.000Z
diff --git a/cli/server.go b/cli/server.go
@@ -88,6 +88,7 @@ import (
 	"github.com/coder/coder/provisionersdk"
 	sdkproto "github.com/coder/coder/provisionersdk/proto"
 	"github.com/coder/coder/tailnet"
+	"github.com/coder/retry"
 	"github.com/coder/wgtunnel/tunnelsdk"
 )
 
@@ -1733,26 +1734,44 @@ func BuildLogger(inv *clibase.Invocation, cfg *codersdk.DeploymentValues) (slog.
 
 func connectToPostgres(ctx context.Context, logger slog.Logger, driver string, dbURL string) (*sql.DB, error) {
 	logger.Debug(ctx, "connecting to postgresql")
-	sqlDB, err := sql.Open(driver, dbURL)
+
+	// Try to connect for 30 seconds.
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	var (
+		sqlDB *sql.DB
+		err   error
+		ok    = false
+		tries int
+	)
+	for r := retry.New(time.Second, 3*time.Second); r.Wait(ctx); {
+		tries++
+
+		sqlDB, err = sql.Open(driver, dbURL)
+		if err != nil {
+			logger.Warn(ctx, "connect to postgres; retrying", slog.Error(err), slog.F("try", tries))
+			continue
+		}
+
+		err = pingPostgres(ctx, sqlDB)
+		if err != nil {
+			logger.Warn(ctx, "ping postgres; retrying", slog.Error(err), slog.F("try", tries))
+			continue
+		}
+
+		break
+	}
 	if err != nil {
-		return nil, xerrors.Errorf("dial postgres: %w", err)
+		return nil, xerrors.Errorf("connect to postgres; tries %d; last error: %w", tries, err)
 	}
 
-	ok := false
 	defer func() {
-		if !ok {
+		if !ok && sqlDB != nil {
 			_ = sqlDB.Close()
 		}
 	}()
 
-	pingCtx, pingCancel := context.WithTimeout(ctx, 15*time.Second)
-	defer pingCancel()
-
-	err = sqlDB.PingContext(pingCtx)
-	if err != nil {
-		return nil, xerrors.Errorf("ping postgres: %w", err)
-	}
-
 	// Ensure the PostgreSQL version is >=13.0.0!
 	version, err := sqlDB.QueryContext(ctx, "SHOW server_version_num;")
 	if err != nil {
@@ -1799,6 +1818,12 @@ func connectToPostgres(ctx context.Context, logger slog.Logger, driver string, d
 	return sqlDB, nil
 }
 
+func pingPostgres(ctx context.Context, db *sql.DB) error {
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+	return db.PingContext(ctx)
+}
+
 type HTTPServers struct {
 	HTTPUrl      *url.URL
 	HTTPListener net.Listener
