Skip to content

feat: include health severity in reports #10817

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 21 commits into from
Nov 23, 2023
Merged

feat: include health severity in reports #10817

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
205a224
Severity definition
mtojek Nov 21, 2023
1bcad04
WIP
mtojek Nov 21, 2023
e1978f2
use model struct for base health summary
mtojek Nov 21, 2023
0e3bc17
The best I can do
mtojek Nov 21, 2023
f8af18c
fix imports
mtojek Nov 21, 2023
baae564
severity
mtojek Nov 21, 2023
3d17eb5
WIP
mtojek Nov 21, 2023
984d3e1
Merge branch 'main' into 9754-severity
mtojek Nov 22, 2023
b373cc2
Merge branch 'main' into 9754-severity
mtojek Nov 22, 2023
8dcbafd
WIP
mtojek Nov 22, 2023
d00802e
more rules
mtojek Nov 22, 2023
7b3945a
Fix: database
mtojek Nov 22, 2023
af2d0dc
WIP
mtojek Nov 22, 2023
6c59dc3
Reports
mtojek Nov 22, 2023
aba2a9e
DERP servers partially unhealthy
mtojek Nov 22, 2023
c15ea21
warning: failover to websockets
mtojek Nov 22, 2023
8d32194
Merge branch 'main' into 9754-severity
mtojek Nov 23, 2023
b77763a
Deprecation notice
mtojek Nov 23, 2023
5f6d7a6
Switch to unhealthy
mtojek Nov 23, 2023
1542296
simplify
mtojek Nov 23, 2023
140b5c6
Missing node report
mtojek Nov 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
DERP servers partially unhealthy
  • Loading branch information
@mtojek
mtojek committed Nov 22, 2023
commit aba2a9e620ea9bd6bf47b6aabf748f7a56293a3d
26 changes: 20 additions & 6 deletions coderd/healthcheck/derphealth/derp.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ type ReportOptions struct {

func (r *Report) Run(ctx context.Context, opts *ReportOptions) {
r.Healthy = true
r.Severity = health.SeverityOK

r.Regions = map[int]*RegionReport{}
r.Warnings = []string{}

Expand Down Expand Up @@ -150,6 +152,13 @@ func (r *Report) Run(ctx context.Context, opts *ReportOptions) {
r.NetcheckErr = convertError(netcheckErr)

wg.Wait()

// Review region reports and select the highest severity.
for _, regionReport := range r.Regions {
if regionReport.Severity.Value() > r.Severity.Value() {
r.Severity = regionReport.Severity
}
}
}

func (r *RegionReport) Run(ctx context.Context) {
Expand Down Expand Up @@ -203,14 +212,19 @@ func (r *RegionReport) Run(ctx context.Context) {
// Coder allows for 1 unhealthy node in the region, unless there is only 1 node.
if len(r.Region.Nodes) == 1 {
r.Healthy = healthyNodes == len(r.Region.Nodes)
} else if healthyNodes < len(r.Region.Nodes) {
r.Severity = r.NodeReports[0].Severity
} else if healthyNodes+1 == len(r.Region.Nodes) {
// r.Healthy = true (by default)
r.Severity = health.SeverityWarning
r.Warnings = append(r.Warnings, oneNodeUnhealthy)
}
} else if healthyNodes+1 < len(r.Region.Nodes) {
r.Healthy = false

// Review node reports and select the highest severing.
for _, nodeReport := range r.NodeReports {
if nodeReport.Severity.Value() > r.Severity.Value() {
r.Severity = nodeReport.Severity
// Review node reports and select the highest severity.
for _, nodeReport := range r.NodeReports {
if nodeReport.Severity.Value() > r.Severity.Value() {
r.Severity = nodeReport.Severity
}
}
}
}
Expand Down
4 changes: 4 additions & 0 deletions coderd/healthcheck/derphealth/derp_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"tailscale.com/types/key"

"github.com/coder/coder/v2/coderd/healthcheck/derphealth"
"github.com/coder/coder/v2/coderd/healthcheck/health"
"github.com/coder/coder/v2/tailnet"
"github.com/coder/coder/v2/testutil"
)
Expand Down Expand Up @@ -123,10 +124,13 @@ func TestDERP(t *testing.T) {
report.Run(ctx, opts)

assert.True(t, report.Healthy)
assert.Equal(t, health.SeverityWarning, report.Severity)
for _, region := range report.Regions {
assert.True(t, region.Healthy)
assert.True(t, region.NodeReports[0].Healthy)
assert.Equal(t, health.SeverityOK, region.NodeReports[0].Severity)
assert.False(t, region.NodeReports[1].Healthy)
assert.Equal(t, health.SeverityError, region.NodeReports[1].Severity)
assert.Len(t, region.Warnings, 1)
}
})
Expand Down