coder · johnstcn · Nov 13, 2023 · Nov 10, 2023 · Nov 10, 2023 · Nov 10, 2023
diff --git a/cli/testdata/coder_server_--help.golden b/cli/testdata/coder_server_--help.golden
@@ -80,6 +80,15 @@ Use a YAML configuration file when your server launch become unwieldy.
 
           Write out the current server config as YAML to stdout.
 
+INTROSPECTION / HEALTH CHECK OPTIONS: 
+      --health-check-refresh duration, $CODER_HEALTH_CHECK_REFRESH (default: 10m0s)
+          Refresh interval for healthchecks.
+
+      --health-check-threshold-database duration, $CODER_HEALTH_CHECK_THRESHOLD_DATABASE (default: 15ms)
+          The threshold for the database health check. If the median latency of
+          the database exceeds this threshold over 5 attempts, the database is
+          considered unhealthy. The default value is 15ms.
+
 INTROSPECTION / LOGGING OPTIONS: 
       --enable-terraform-debug-mode bool, $CODER_ENABLE_TERRAFORM_DEBUG_MODE (default: false)
           Allow administrators to enable Terraform debug output.

diff --git a/cli/testdata/server-config.yaml.golden b/cli/testdata/server-config.yaml.golden
@@ -232,6 +232,15 @@ introspection:
     # Allow administrators to enable Terraform debug output.
     # (default: false, type: bool)
     enableTerraformDebugMode: false
+  healthcheck:
+    # Refresh interval for healthchecks.
+    # (default: 10m0s, type: duration)
+    refresh: 10m0s
+    # The threshold for the database health check. If the median latency of the
+    # database exceeds this threshold over 5 attempts, the database is considered
+    # unhealthy. The default value is 15ms.
+    # (default: 15ms, type: duration)
+    thresholdDatabase: 15ms
 oauth2:
   github:
     # Client ID for Login with GitHub.

diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go
diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json
diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -38,6 +38,7 @@ import (
 	// Used for swagger docs.
 	_ "github.com/coder/coder/v2/coderd/apidoc"
 	"github.com/coder/coder/v2/coderd/externalauth"
+	"github.com/coder/coder/v2/coderd/healthcheck/derphealth"
 
 	"cdr.dev/slog"
 	"github.com/coder/coder/v2/buildinfo"
@@ -398,18 +399,28 @@ func New(options *Options) *API {
 	if options.HealthcheckFunc == nil {
 		options.HealthcheckFunc = func(ctx context.Context, apiKey string) *healthcheck.Report {
 			return healthcheck.Run(ctx, &healthcheck.ReportOptions{
-				DB:        options.Database,
-				AccessURL: options.AccessURL,
-				DERPMap:   api.DERPMap(),
-				APIKey:    apiKey,
+				Database: healthcheck.DatabaseReportOptions{
+					DB:        options.Database,
+					Threshold: options.DeploymentValues.Healthcheck.ThresholdDatabase.Value(),
+				},
+				Websocket: healthcheck.WebsocketReportOptions{
+					AccessURL: options.AccessURL,
+					APIKey:    apiKey,
+				},
+				AccessURL: healthcheck.AccessURLReportOptions{
+					AccessURL: options.AccessURL,
+				},
+				DerpHealth: derphealth.ReportOptions{
+					DERPMap: api.DERPMap(),
+				},
 			})
 		}
 	}
 	if options.HealthcheckTimeout == 0 {
 		options.HealthcheckTimeout = 30 * time.Second
 	}
 	if options.HealthcheckRefresh == 0 {
-		options.HealthcheckRefresh = 10 * time.Minute
+		options.HealthcheckRefresh = options.DeploymentValues.Healthcheck.Refresh.Value()
 	}
 
 	var oidcAuthURLParams map[string]string

diff --git a/coderd/debug.go b/coderd/debug.go
@@ -32,20 +32,20 @@ func (api *API) debugCoordinator(rw http.ResponseWriter, r *http.Request) {
 // @Router /debug/health [get]
 func (api *API) debugDeploymentHealth(rw http.ResponseWriter, r *http.Request) {
 	apiKey := httpmw.APITokenFromRequest(r)
-	ctx, cancel := context.WithTimeout(r.Context(), api.HealthcheckTimeout)
+	ctx, cancel := context.WithTimeout(r.Context(), api.Options.HealthcheckTimeout)
 	defer cancel()
 
 	// Get cached report if it exists.
 	if report := api.healthCheckCache.Load(); report != nil {
-		if time.Since(report.Time) < api.HealthcheckRefresh {
+		if time.Since(report.Time) < api.Options.HealthcheckRefresh {
 			formatHealthcheck(ctx, rw, r, report)
 			return
 		}
 	}
 
 	resChan := api.healthCheckGroup.DoChan("", func() (*healthcheck.Report, error) {
 		// Create a new context not tied to the request.
-		ctx, cancel := context.WithTimeout(context.Background(), api.HealthcheckTimeout)
+		ctx, cancel := context.WithTimeout(context.Background(), api.Options.HealthcheckTimeout)
 		defer cancel()
 
 		report := api.HealthcheckFunc(ctx, apiKey)

diff --git a/coderd/debug_test.go b/coderd/debug_test.go
@@ -72,6 +72,51 @@ func TestDebugHealth(t *testing.T) {
 		require.Equal(t, http.StatusNotFound, res.StatusCode)
 	})
 
+	t.Run("Refresh", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			calls       = make(chan struct{})
+			callsDone   = make(chan struct{})
+			ctx, cancel = context.WithTimeout(context.Background(), testutil.WaitShort)
+			client      = coderdtest.New(t, &coderdtest.Options{
+				HealthcheckRefresh: time.Microsecond,
+				HealthcheckFunc: func(context.Context, string) *healthcheck.Report {
+					calls <- struct{}{}
+					return &healthcheck.Report{}
+				},
+			})
+			_ = coderdtest.CreateFirstUser(t, client)
+		)
+
+		defer cancel()
+
+		go func() {
+			defer close(callsDone)
+			<-calls
+			<-time.After(testutil.IntervalFast)
+			<-calls
+		}()
+
+		res, err := client.Request(ctx, "GET", "/api/v2/debug/health", nil)
+		require.NoError(t, err)
+		defer res.Body.Close()
+		_, _ = io.ReadAll(res.Body)
+		require.Equal(t, http.StatusOK, res.StatusCode)
+
+		res, err = client.Request(ctx, "GET", "/api/v2/debug/health", nil)
+		require.NoError(t, err)
+		defer res.Body.Close()
+		_, _ = io.ReadAll(res.Body)
+		require.Equal(t, http.StatusOK, res.StatusCode)
+
+		select {
+		case <-callsDone:
+		case <-ctx.Done():
+			t.Fatal("timed out waiting for calls to finish")
+		}
+	})
+
 	t.Run("Deduplicated", func(t *testing.T) {
 		t.Parallel()
 

diff --git a/coderd/healthcheck/database.go b/coderd/healthcheck/database.go
@@ -10,20 +10,30 @@ import (
 	"github.com/coder/coder/v2/coderd/database"
 )
 
+const (
+	DatabaseDefaultThreshold = 15 * time.Millisecond
+)
+
 // @typescript-generate DatabaseReport
 type DatabaseReport struct {
-	Healthy   bool    `json:"healthy"`
-	Reachable bool    `json:"reachable"`
-	Latency   string  `json:"latency"`
-	LatencyMs int     `json:"latency_ms"`
-	Error     *string `json:"error"`
+	Healthy     bool    `json:"healthy"`
+	Reachable   bool    `json:"reachable"`
+	Latency     string  `json:"latency"`
+	LatencyMS   int64   `json:"latency_ms"`
+	ThresholdMS int64   `json:"threshold_ms"`
+	Error       *string `json:"error"`
 }
 
 type DatabaseReportOptions struct {
-	DB database.Store
+	DB        database.Store
+	Threshold time.Duration
 }
 
 func (r *DatabaseReport) Run(ctx context.Context, opts *DatabaseReportOptions) {
+	r.ThresholdMS = opts.Threshold.Milliseconds()
+	if r.ThresholdMS == 0 {
+		r.ThresholdMS = DatabaseDefaultThreshold.Milliseconds()
+	}
 	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
 	defer cancel()
 
@@ -43,10 +53,8 @@ func (r *DatabaseReport) Run(ctx context.Context, opts *DatabaseReportOptions) {
 	// Take the median ping.
 	latency := pings[pingCount/2]
 	r.Latency = latency.String()
-	r.LatencyMs = int(latency.Milliseconds())
-	// Somewhat arbitrary, but if the latency is over 15ms, we consider it
-	// unhealthy.
-	if latency < 15*time.Millisecond {
+	r.LatencyMS = latency.Milliseconds()
+	if r.LatencyMS < r.ThresholdMS {
 		r.Healthy = true
 	}
 	r.Reachable = true

diff --git a/coderd/healthcheck/database_test.go b/coderd/healthcheck/database_test.go
@@ -36,7 +36,8 @@ func TestDatabase(t *testing.T) {
 		assert.True(t, report.Healthy)
 		assert.True(t, report.Reachable)
 		assert.Equal(t, ping.String(), report.Latency)
-		assert.Equal(t, int(ping.Milliseconds()), report.LatencyMs)
+		assert.Equal(t, ping.Milliseconds(), report.LatencyMS)
+		assert.Equal(t, healthcheck.DatabaseDefaultThreshold.Milliseconds(), report.ThresholdMS)
 		assert.Nil(t, report.Error)
 	})
 
@@ -59,6 +60,7 @@ func TestDatabase(t *testing.T) {
 		assert.False(t, report.Reachable)
 		assert.Zero(t, report.Latency)
 		require.NotNil(t, report.Error)
+		assert.Equal(t, healthcheck.DatabaseDefaultThreshold.Milliseconds(), report.ThresholdMS)
 		assert.Contains(t, *report.Error, err.Error())
 	})
 
@@ -83,7 +85,34 @@ func TestDatabase(t *testing.T) {
 		assert.True(t, report.Healthy)
 		assert.True(t, report.Reachable)
 		assert.Equal(t, time.Millisecond.String(), report.Latency)
-		assert.Equal(t, 1, report.LatencyMs)
+		assert.EqualValues(t, 1, report.LatencyMS)
+		assert.Equal(t, healthcheck.DatabaseDefaultThreshold.Milliseconds(), report.ThresholdMS)
+		assert.Nil(t, report.Error)
+	})
+
+	t.Run("Threshold", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			ctx, cancel = context.WithTimeout(context.Background(), testutil.WaitShort)
+			report      = healthcheck.DatabaseReport{}
+			db          = dbmock.NewMockStore(gomock.NewController(t))
+		)
+		defer cancel()
+
+		db.EXPECT().Ping(gomock.Any()).Return(time.Second, nil)
+		db.EXPECT().Ping(gomock.Any()).Return(time.Millisecond, nil)
+		db.EXPECT().Ping(gomock.Any()).Return(time.Second, nil)
+		db.EXPECT().Ping(gomock.Any()).Return(time.Millisecond, nil)
+		db.EXPECT().Ping(gomock.Any()).Return(time.Second, nil)
+
+		report.Run(ctx, &healthcheck.DatabaseReportOptions{DB: db, Threshold: time.Second})
+
+		assert.False(t, report.Healthy)
+		assert.True(t, report.Reachable)
+		assert.Equal(t, time.Second.String(), report.Latency)
+		assert.EqualValues(t, 1000, report.LatencyMS)
+		assert.Equal(t, time.Second.Milliseconds(), report.ThresholdMS)
 		assert.Nil(t, report.Error)
 	})
 }