DERP servers partially unhealthy

mtojek · mtojek · commit aba2a9e620ea · 2023-11-22T19:43:42.000+01:00
diff --git a/coderd/healthcheck/derphealth/derp.go b/coderd/healthcheck/derphealth/derp.go
@@ -99,6 +99,8 @@ type ReportOptions struct {
 
 func (r *Report) Run(ctx context.Context, opts *ReportOptions) {
 	r.Healthy = true
+	r.Severity = health.SeverityOK
+
 	r.Regions = map[int]*RegionReport{}
 	r.Warnings = []string{}
 
@@ -150,6 +152,13 @@ func (r *Report) Run(ctx context.Context, opts *ReportOptions) {
 	r.NetcheckErr = convertError(netcheckErr)
 
 	wg.Wait()
+
+	// Review region reports and select the highest severity.
+	for _, regionReport := range r.Regions {
+		if regionReport.Severity.Value() > r.Severity.Value() {
+			r.Severity = regionReport.Severity
+		}
+	}
 }
 
 func (r *RegionReport) Run(ctx context.Context) {
@@ -203,14 +212,19 @@ func (r *RegionReport) Run(ctx context.Context) {
 	// Coder allows for 1 unhealthy node in the region, unless there is only 1 node.
 	if len(r.Region.Nodes) == 1 {
 		r.Healthy = healthyNodes == len(r.Region.Nodes)
-	} else if healthyNodes < len(r.Region.Nodes) {
+		r.Severity = r.NodeReports[0].Severity
+	} else if healthyNodes+1 == len(r.Region.Nodes) {
+		// r.Healthy = true (by default)
+		r.Severity = health.SeverityWarning
 		r.Warnings = append(r.Warnings, oneNodeUnhealthy)
-	}
+	} else if healthyNodes+1 < len(r.Region.Nodes) {
+		r.Healthy = false
 
-	// Review node reports and select the highest severing.
-	for _, nodeReport := range r.NodeReports {
-		if nodeReport.Severity.Value() > r.Severity.Value() {
-			r.Severity = nodeReport.Severity
+		// Review node reports and select the highest severity.
+		for _, nodeReport := range r.NodeReports {
+			if nodeReport.Severity.Value() > r.Severity.Value() {
+				r.Severity = nodeReport.Severity
+			}
 		}
 	}
 }
diff --git a/coderd/healthcheck/derphealth/derp_test.go b/coderd/healthcheck/derphealth/derp_test.go
@@ -18,6 +18,7 @@ import (
 	"tailscale.com/types/key"
 
 	"github.com/coder/coder/v2/coderd/healthcheck/derphealth"
+	"github.com/coder/coder/v2/coderd/healthcheck/health"
 	"github.com/coder/coder/v2/tailnet"
 	"github.com/coder/coder/v2/testutil"
 )
@@ -123,10 +124,13 @@ func TestDERP(t *testing.T) {
 		report.Run(ctx, opts)
 
 		assert.True(t, report.Healthy)
+		assert.Equal(t, health.SeverityWarning, report.Severity)
 		for _, region := range report.Regions {
 			assert.True(t, region.Healthy)
 			assert.True(t, region.NodeReports[0].Healthy)
+			assert.Equal(t, health.SeverityOK, region.NodeReports[0].Severity)
 			assert.False(t, region.NodeReports[1].Healthy)
+			assert.Equal(t, health.SeverityError, region.NodeReports[1].Severity)
 			assert.Len(t, region.Warnings, 1)
 		}
 	})
