feat(coderd): add DERP healthcheck

coder · coadler · Apr 3, 2023 · Mar 29, 2023 · Mar 31, 2023 · Mar 31, 2023
commit ca13285546ea29ffe36acd6a29a8ed3374451474
diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -33,6 +33,7 @@ import (
 	"tailscale.com/derp/derphttp"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
+	"tailscale.com/util/singleflight"
 
 	"cdr.dev/slog"
 	"github.com/coder/coder/buildinfo"
@@ -46,6 +47,7 @@ import (
 	"github.com/coder/coder/coderd/database/dbtype"
 	"github.com/coder/coder/coderd/gitauth"
 	"github.com/coder/coder/coderd/gitsshkey"
+	"github.com/coder/coder/coderd/healthcheck"
 	"github.com/coder/coder/coderd/httpapi"
 	"github.com/coder/coder/coderd/httpmw"
 	"github.com/coder/coder/coderd/metricscache"
@@ -123,7 +125,10 @@ type Options struct {
 	TemplateScheduleStore schedule.TemplateScheduleStore
 	// AppSigningKey denotes the symmetric key to use for signing app tickets.
 	// The key must be 64 bytes long.
-	AppSigningKey []byte
+	AppSigningKey      []byte
+	HealthcheckFunc    func(ctx context.Context) (*healthcheck.Report, error)
+	HealthcheckTimeout time.Duration
+	HealthcheckRefresh time.Duration
 
 	// APIRateLimit is the minutely throughput rate limit per user or ip.
 	// Setting a rate limit <0 will disable the rate limiter across the entire
@@ -235,6 +240,19 @@ func New(options *Options) *API {
 	if len(options.AppSigningKey) != 64 {
 		panic("coderd: AppSigningKey must be 64 bytes long")
 	}
+	if options.HealthcheckFunc == nil {
+		options.HealthcheckFunc = func(ctx context.Context) (*healthcheck.Report, error) {
+			return healthcheck.Run(ctx, &healthcheck.ReportOptions{
+				DERPMap: options.DERPMap.Clone(),
+			})
+		}
+	}
+	if options.HealthcheckTimeout == 0 {
+		options.HealthcheckTimeout = 30 * time.Second
+	}
+	if options.HealthcheckRefresh == 0 {
+		options.HealthcheckRefresh = 10 * time.Minute
+	}
 
 	siteCacheDir := options.CacheDir
 	if siteCacheDir != "" {
@@ -293,6 +311,7 @@ func New(options *Options) *API {
 		Auditor:               atomic.Pointer[audit.Auditor]{},
 		TemplateScheduleStore: atomic.Pointer[schedule.TemplateScheduleStore]{},
 		Experiments:           experiments,
+		healthCheckGroup:      &singleflight.Group[string, *healthcheck.Report]{},
 	}
 	if options.UpdateCheckOptions != nil {
 		api.updateChecker = updatecheck.New(
@@ -718,6 +737,7 @@ func New(options *Options) *API {
 			)
 
 			r.Get("/coordinator", api.debugCoordinator)
+			r.Get("/health", api.debugDeploymentHealth)
 		})
 	})
 
@@ -773,6 +793,8 @@ type API struct {
 	// Experiments contains the list of experiments currently enabled.
 	// This is used to gate features that are not yet ready for production.
 	Experiments codersdk.Experiments
+
+	healthCheckGroup *singleflight.Group[string, *healthcheck.Report]
 }
 
 // Close waits for all WebSocket connections to drain before returning.

diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go
@@ -60,6 +60,7 @@ import (
 	"github.com/coder/coder/coderd/database/dbtestutil"
 	"github.com/coder/coder/coderd/gitauth"
 	"github.com/coder/coder/coderd/gitsshkey"
+	"github.com/coder/coder/coderd/healthcheck"
 	"github.com/coder/coder/coderd/httpapi"
 	"github.com/coder/coder/coderd/httpmw"
 	"github.com/coder/coder/coderd/rbac"
@@ -105,6 +106,10 @@ type Options struct {
 	TrialGenerator        func(context.Context, string) error
 	TemplateScheduleStore schedule.TemplateScheduleStore
 
+	HealthcheckFunc    func(ctx context.Context) (*healthcheck.Report, error)
+	HealthcheckTimeout time.Duration
+	HealthcheckRefresh time.Duration
+
 	// All rate limits default to -1 (unlimited) in tests if not set.
 	APIRateLimit   int
 	LoginRateLimit int
@@ -335,6 +340,9 @@ func NewOptions(t *testing.T, options *Options) (func(http.Handler), context.Can
 			SwaggerEndpoint:             options.SwaggerEndpoint,
 			AppSigningKey:               AppSigningKey,
 			SSHConfig:                   options.ConfigSSH,
+			HealthcheckFunc:             options.HealthcheckFunc,
+			HealthcheckTimeout:          options.HealthcheckTimeout,
+			HealthcheckRefresh:          options.HealthcheckRefresh,
 		}
 }
 

diff --git a/coderd/debug.go b/coderd/debug.go
@@ -1,6 +1,14 @@
 package coderd
 
-import "net/http"
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/coder/coder/coderd/healthcheck"
+	"github.com/coder/coder/coderd/httpapi"
+	"github.com/coder/coder/codersdk"
+)
 
 // @Summary Debug Info Wireguard Coordinator
 // @ID debug-info-wireguard-coordinator
@@ -12,3 +20,37 @@ import "net/http"
 func (api *API) debugCoordinator(rw http.ResponseWriter, r *http.Request) {
 	(*api.TailnetCoordinator.Load()).ServeHTTPDebug(rw, r)
 }
+
+// @Summary Debug Info Deployment Health
+// @ID debug-info-deployment-health
+// @Security CoderSessionToken
+// @Produce text/html
+// @Produce json
+// @Tags Debug
+// @Success 200
+// @Router /debug/health [get]
+func (api *API) debugDeploymentHealth(rw http.ResponseWriter, r *http.Request) {
+	ctx, cancel := context.WithTimeout(r.Context(), api.HealthcheckTimeout)
+	defer cancel()
+
+	resChan := api.healthCheckGroup.DoChan("", func() (*healthcheck.Report, error) {
+		return api.HealthcheckFunc(ctx)
+	})
+
+	select {
+	case <-ctx.Done():
+		httpapi.Write(ctx, rw, http.StatusNotFound, codersdk.Response{
+			Message: "Healthcheck is in progress and did not complete in time. Try again in a few seconds.",
+		})
+		return
+	case res := <-resChan:
+		if time.Since(res.Val.Time) > api.HealthcheckRefresh {
+			api.healthCheckGroup.Forget("")
+			api.debugDeploymentHealth(rw, r)
+			return
+		}
+
+		httpapi.Write(ctx, rw, http.StatusOK, res.Val)
+		return
+	}
+}
diff --git a/coderd/debug_test.go b/coderd/debug_test.go
@@ -0,0 +1,71 @@
+package coderd_test
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httputil"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/coderd/coderdtest"
+	"github.com/coder/coder/coderd/healthcheck"
+	"github.com/coder/coder/testutil"
+)
+
+func TestDebug(t *testing.T) {
+	t.Parallel()
+	t.Run("Health/OK", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			ctx, cancel = context.WithTimeout(context.Background(), testutil.WaitShort)
+			client      = coderdtest.New(t, &coderdtest.Options{
+				HealthcheckFunc: func(context.Context) (*healthcheck.Report, error) {
+					return &healthcheck.Report{}, nil
+				},
+			})
+			_ = coderdtest.CreateFirstUser(t, client)
+		)
+		defer cancel()
+
+		res, err := client.Request(ctx, "GET", "/debug/health", nil)
+		require.NoError(t, err)
+		defer res.Body.Close()
+		require.Equal(t, http.StatusOK, res.StatusCode)
+
+	})
+
+	t.Run("Health/Timeout", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			ctx, cancel = context.WithTimeout(context.Background(), testutil.WaitShort)
+			client      = coderdtest.New(t, &coderdtest.Options{
+				HealthcheckTimeout: time.Microsecond,
+				HealthcheckFunc: func(context.Context) (*healthcheck.Report, error) {
+					t := time.NewTimer(time.Second)
+					defer t.Stop()
+
+					select {
+					case <-ctx.Done():
+						return nil, ctx.Err()
+					case <-t.C:
+						return &healthcheck.Report{}, nil
+					}
+				},
+			})
+			_ = coderdtest.CreateFirstUser(t, client)
+		)
+		defer cancel()
+
+		res, err := client.Request(ctx, "GET", "/api/v2/debug/health", nil)
+		require.NoError(t, err)
+		defer res.Body.Close()
+		dump, _ := httputil.DumpResponse(res, true)
+		fmt.Println(string(dump))
+		require.Equal(t, http.StatusNotFound, res.StatusCode)
+	})
+}
diff --git a/coderd/healthcheck/accessurl.go b/coderd/healthcheck/accessurl.go
@@ -0,0 +1,48 @@
+package healthcheck
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/url"
+
+	"golang.org/x/xerrors"
+)
+
+type AccessURLReport struct {
+	Reachable       bool
+	StatusCode      int
+	HealthzResponse string
+	Err             error
+}
+
+func (r *AccessURLReport) Run(ctx context.Context, accessURL *url.URL) {
+	accessURL, err := accessURL.Parse("/healthz")
+	if err != nil {
+		r.Err = xerrors.Errorf("parse healthz endpoint: %w", err)
+		return
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", accessURL.String(), nil)
+	if err != nil {
+		r.Err = xerrors.Errorf("create healthz request: %w", err)
+		return
+	}
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		r.Err = xerrors.Errorf("get healthz endpoint: %w", err)
+		return
+	}
+	defer res.Body.Close()
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		r.Err = xerrors.Errorf("read healthz response: %w", err)
+		return
+	}
+
+	r.Reachable = true
+	r.StatusCode = res.StatusCode
+	r.HealthzResponse = string(body)
+}