Skip to content

feat(coderd): add DERP healthcheck #6936

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Apr 3, 2023
Merged

feat(coderd): add DERP healthcheck #6936

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ca13285
feat(coderd): add DERP healthcheck
coadler Mar 29, 2023
98f37bd
fixup! feat(coderd): add DERP healthcheck
coadler Mar 31, 2023
94e00ff
fixup! feat(coderd): add DERP healthcheck
coadler Mar 31, 2023
7ef0217
remove access url report
coadler Mar 31, 2023
bf418dc
Merge branch 'main' into colin/healthcheck
coadler Apr 3, 2023
9f62a8c
add healthy bool
coadler Apr 3, 2023
f045e6e
fixup! add healthy bool
coadler Apr 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add healthy bool
  • Loading branch information
@coadler
coadler committed Apr 3, 2023
commit 9f62a8cbe598ae23270d01ccd249a25a1f15fee1
29 changes: 22 additions & 7 deletions coderd/healthcheck/derp.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ import (
)

type DERPReport struct {
mu sync.Mutex
mu sync.Mutex
Healthy bool `json:"healthy"`

Regions map[int]*DERPRegionReport `json:"regions"`

Expand All @@ -35,17 +36,19 @@ type DERPReport struct {
}

type DERPRegionReport struct {
mu sync.Mutex
Region *tailcfg.DERPRegion `json:"region"`
mu sync.Mutex
Healthy bool `json:"healthy"`

NodeReports []*DERPNodeReport `json:"node_reports"`
Region *tailcfg.DERPRegion `json:"region"`
NodeReports []*DERPNodeReport `json:"node_reports"`
}
type DERPNodeReport struct {
Node *tailcfg.DERPNode `json:"node"`

mu sync.Mutex
clientCounter int

Healthy bool `json:"healthy"`
Node *tailcfg.DERPNode `json:"node"`

CanExchangeMessages bool `json:"can_exchange_messages"`
RoundTripPing time.Duration `json:"round_trip_ping"`
UsesWebsocket bool `json:"uses_websocket"`
Expand All @@ -66,6 +69,7 @@ type DERPReportOptions struct {
}

func (r *DERPReport) Run(ctx context.Context, opts *DERPReportOptions) error {
r.Healthy = true
r.Regions = map[int]*DERPRegionReport{}

eg, ctx := errgroup.WithContext(ctx)
Expand All @@ -84,6 +88,9 @@ func (r *DERPReport) Run(ctx context.Context, opts *DERPReportOptions) error {

r.mu.Lock()
r.Regions[region.RegionID] = &regionReport
if !regionReport.Healthy {
r.Healthy = false
}
r.mu.Unlock()
return nil
})
Expand All @@ -108,14 +115,16 @@ func (r *DERPReport) Run(ctx context.Context, opts *DERPReportOptions) error {
}

func (r *DERPRegionReport) Run(ctx context.Context) error {
r.Healthy = true
r.NodeReports = []*DERPNodeReport{}
eg, ctx := errgroup.WithContext(ctx)

for _, node := range r.Region.Nodes {
node := node
eg.Go(func() error {
nodeReport := DERPNodeReport{
Node: node,
Node: node,
Healthy: true,
}

err := nodeReport.Run(ctx)
Expand All @@ -125,6 +134,9 @@ func (r *DERPRegionReport) Run(ctx context.Context) error {

r.mu.Lock()
r.NodeReports = append(r.NodeReports, &nodeReport)
if !nodeReport.Healthy {
r.Healthy = false
}
r.mu.Unlock()
return nil
})
Expand Down Expand Up @@ -159,6 +171,9 @@ func (r *DERPNodeReport) Run(ctx context.Context) error {
r.doExchangeMessage(ctx)
r.doSTUNTest(ctx)

if !r.CanExchangeMessages || r.UsesWebsocket || r.STUN.Error != nil {
r.Healthy = false
}
return nil
}

Expand Down
9 changes: 9 additions & 0 deletions coderd/healthcheck/derp_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,11 @@ func TestDERP(t *testing.T) {
err := report.Run(ctx, opts)
require.NoError(t, err)

assert.True(t, report.Healthy)
for _, region := range report.Regions {
assert.True(t, region.Healthy)
for _, node := range region.NodeReports {
assert.True(t, node.Healthy)
assert.True(t, node.CanExchangeMessages)
assert.Positive(t, node.RoundTripPing)
assert.Len(t, node.ClientLogs, 2)
Expand Down Expand Up @@ -96,8 +99,11 @@ func TestDERP(t *testing.T) {
err := report.Run(ctx, opts)
require.NoError(t, err)

assert.True(t, report.Healthy)
for _, region := range report.Regions {
assert.True(t, region.Healthy)
for _, node := range region.NodeReports {
assert.True(t, node.Healthy)
assert.True(t, node.CanExchangeMessages)
assert.Positive(t, node.RoundTripPing)
assert.Len(t, node.ClientLogs, 2)
Expand Down Expand Up @@ -156,8 +162,11 @@ func TestDERP(t *testing.T) {

report.Run(ctx, opts)

assert.False(t, report.Healthy)
for _, region := range report.Regions {
assert.False(t, region.Healthy)
for _, node := range region.NodeReports {
assert.False(t, node.Healthy)
assert.True(t, node.CanExchangeMessages)
assert.Positive(t, node.RoundTripPing)
assert.Len(t, node.ClientLogs, 2)
Expand Down
7 changes: 6 additions & 1 deletion coderd/healthcheck/healthcheck.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@ import (
)

type Report struct {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ideally there's an overall "Pass" or "Fail" grade, as well as a "Pass" or "Fail" for each of the sub-reports.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought about that but at least for DERP pass/fail felt more like a spectrum. You could have a derp server that could exchange messages but was listed as STUN but STUN wasn't working, etc

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Eh maybe "Fail" could just mean anything failed or is degraded.

Time time.Time `json:"time"`
// Time is the time the report was generated at.
Time time.Time `json:"time"`
// Healthy is true if the report returns no errors.
Healthy bool `json:"pass"`

DERP DERPReport `json:"derp"`

// TODO
Expand All @@ -33,5 +37,6 @@ func Run(ctx context.Context, opts *ReportOptions) (*Report, error) {
}

report.Time = time.Now()
report.Healthy = report.DERP.Healthy
return &report, nil
}