Skip to content

chore: retry postgres connection on reset by peer in tests #18632

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 27, 2025
Merged

chore: retry postgres connection on reset by peer in tests #18632

Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5c7efe5
chore: retry postgres connection on reset by peer in tests
spikecurtis Jun 27, 2025
0985a84
Update coderd/database/dbtestutil/postgres.go
spikecurtis Jun 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 43 additions & 16 deletions coderd/database/dbtestutil/postgres.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,13 @@ var (
connectionParamsInitOnce sync.Once
defaultConnectionParams ConnectionParams
errDefaultConnectionParamsInit error
retryableErrSubstrings = []string{
"connection reset by peer",
}
noPostgresRunningErrSubstrings = []string{
"connection refused", // nothing is listening on the port
"No connection could be made", // Windows variant of the above
}
)

// initDefaultConnection initializes the default postgres connection parameters.
Expand All @@ -59,28 +66,38 @@ func initDefaultConnection(t TBSubset) error {
DBName: "postgres",
}
dsn := params.DSN()
db, dbErr := sql.Open("postgres", dsn)
if dbErr == nil {
dbErr = db.Ping()
if closeErr := db.Close(); closeErr != nil {
return xerrors.Errorf("close db: %w", closeErr)

// Helper closure to try opening and pinging the default Postgres instance.
// Used within a single retry loop that handles both retryable and permanent errors.
attemptConn := func() error {
db, err := sql.Open("postgres", dsn)
if err == nil {
err = db.Ping()
if closeErr := db.Close(); closeErr != nil {
return xerrors.Errorf("close db: %w", closeErr)
}
}
return err
}
shouldOpenContainer := false
if dbErr != nil {
errSubstrings := []string{
"connection refused", // this happens on Linux when there's nothing listening on the port
"No connection could be made", // like above but Windows

var dbErr error
// Retry up to 3 seconds for temporary errors.
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
for r := retry.New(10*time.Millisecond, 500*time.Millisecond); r.Wait(ctx); {
dbErr = attemptConn()
if dbErr == nil {
break
}
errString := dbErr.Error()
for _, errSubstring := range errSubstrings {
if strings.Contains(errString, errSubstring) {
shouldOpenContainer = true
break
}
if !containsAnySubstring(errString, retryableErrSubstrings) {
break
}
t.Logf("failed to connect to postgres, retrying: %s", errString)
}
if dbErr != nil && shouldOpenContainer {

// After the loop dbErr is the last connection error (if any).
if dbErr != nil && containsAnySubstring(dbErr.Error(), noPostgresRunningErrSubstrings) {
// If there's no database running on the default port, we'll start a
// postgres container. We won't be cleaning it up so it can be reused
// by subsequent tests. It'll keep on running until the user terminates
Expand Down Expand Up @@ -110,6 +127,7 @@ func initDefaultConnection(t TBSubset) error {
if connErr == nil {
break
}
t.Logf("failed to connect to postgres after starting container, may retry: %s", connErr.Error())
}
} else if dbErr != nil {
return xerrors.Errorf("open postgres connection: %w", dbErr)
Expand Down Expand Up @@ -523,3 +541,12 @@ func OpenContainerized(t TBSubset, opts DBContainerOptions) (string, func(), err

return dbURL, containerCleanup, nil
}

func containsAnySubstring(s string, substrings []string) bool {
for _, substr := range substrings {
if strings.Contains(s, substr) {
return true
}
}
return false
}
Loading