Skip to content

feat(coderd/healthcheck): add health check for proxy #10846

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 24 commits into from
Nov 24, 2023
Merged

feat(coderd/healthcheck): add health check for proxy #10846

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
a17ec4c
feat(coderd/healthcheck): add health check for proxy
johnstcn Nov 21, 2023
49b6e19
rm logger
johnstcn Nov 23, 2023
c7e202c
wire it up
johnstcn Nov 23, 2023
89bae7e
add severity, convert to table-driven tests
johnstcn Nov 23, 2023
bcdc08c
dbauthz
johnstcn Nov 23, 2023
340a276
make gen; make fmt
johnstcn Nov 23, 2023
d3478c8
Merge remote-tracking branch 'origin/main' into cj/workspaceproxy-hea…
johnstcn Nov 24, 2023
c746021
fix linter import shadowing complaint
johnstcn Nov 24, 2023
05d4b09
fix generated report name
johnstcn Nov 24, 2023
56be002
gen harder
johnstcn Nov 24, 2023
1112b1c
address comments
johnstcn Nov 24, 2023
fed35bd
Merge remote-tracking branch 'origin/main' into cj/workspaceproxy-hea…
johnstcn Nov 24, 2023
7f8760a
rename WorkspaceProxies -> workspace_proxies
johnstcn Nov 24, 2023
5f5c486
fixup! rename WorkspaceProxies -> workspace_proxies
johnstcn Nov 24, 2023
b9cbdd8
use severity value instead of direct comparison
johnstcn Nov 24, 2023
ae30089
more tests
johnstcn Nov 24, 2023
21f52bb
use errors.Join instead of multierror.Append
johnstcn Nov 24, 2023
79dc190
refactor to use interface instead of atomic.Pointers
johnstcn Nov 24, 2023
2530320
rm unnecessary authz
johnstcn Nov 24, 2023
f192488
fix typegen
johnstcn Nov 24, 2023
b45e9c5
Merge remote-tracking branch 'origin/main' into cj/workspaceproxy-hea…
johnstcn Nov 24, 2023
63fbafb
make typescript stop whingeing
johnstcn Nov 24, 2023
6de94bc
Merge remote-tracking branch 'origin/main' into cj/workspaceproxy-hea…
johnstcn Nov 24, 2023
b57cdeb
remove need for gosimple nolint
johnstcn Nov 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions coderd/apidoc/docs.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

26 changes: 26 additions & 0 deletions coderd/apidoc/swagger.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

22 changes: 18 additions & 4 deletions coderd/coderd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,10 +135,12 @@ type Options struct {
AccessControlStore *atomic.Pointer[dbauthz.AccessControlStore]
// AppSecurityKey is the crypto key used to sign and encrypt tokens related to
// workspace applications. It consists of both a signing and encryption key.
AppSecurityKey workspaceapps.SecurityKey
HealthcheckFunc func(ctx context.Context, apiKey string) *healthcheck.Report
HealthcheckTimeout time.Duration
HealthcheckRefresh time.Duration
AppSecurityKey workspaceapps.SecurityKey

HealthcheckFunc func(ctx context.Context, apiKey string) *healthcheck.Report
HealthcheckTimeout time.Duration
HealthcheckRefresh time.Duration
WorkspaceProxiesFetchUpdater *atomic.Pointer[healthcheck.WorkspaceProxiesFetchUpdater]

// OAuthSigningKey is the crypto key used to sign and encrypt state strings
// related to OAuth. This is a symmetric secret key using hmac to sign payloads.
Expand Down Expand Up @@ -396,6 +398,13 @@ func New(options *Options) *API {
*options.UpdateCheckOptions,
)
}

if options.WorkspaceProxiesFetchUpdater == nil {
options.WorkspaceProxiesFetchUpdater = &atomic.Pointer[healthcheck.WorkspaceProxiesFetchUpdater]{}
var wpfu healthcheck.WorkspaceProxiesFetchUpdater = &healthcheck.AGPLWorkspaceProxiesFetchUpdater{}
options.WorkspaceProxiesFetchUpdater.Store(&wpfu)
}

if options.HealthcheckFunc == nil {
options.HealthcheckFunc = func(ctx context.Context, apiKey string) *healthcheck.Report {
return healthcheck.Run(ctx, &healthcheck.ReportOptions{
Expand All @@ -413,9 +422,14 @@ func New(options *Options) *API {
DerpHealth: derphealth.ReportOptions{
DERPMap: api.DERPMap(),
},
WorkspaceProxy: healthcheck.WorkspaceProxyReportOptions{
CurrentVersion: buildinfo.Version(),
WorkspaceProxiesFetchUpdater: *(options.WorkspaceProxiesFetchUpdater).Load(),
},
})
}
}

if options.HealthcheckTimeout == 0 {
options.HealthcheckTimeout = 30 * time.Second
}
Expand Down
51 changes: 39 additions & 12 deletions coderd/healthcheck/healthcheck.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,19 @@ import (
)

const (
SectionDERP string = "DERP"
SectionAccessURL string = "AccessURL"
SectionWebsocket string = "Websocket"
SectionDatabase string = "Database"
SectionDERP string = "DERP"
SectionAccessURL string = "AccessURL"
SectionWebsocket string = "Websocket"
SectionDatabase string = "Database"
SectionWorkspaceProxy string = "WorkspaceProxy"
)

type Checker interface {
DERP(ctx context.Context, opts *derphealth.ReportOptions) derphealth.Report
AccessURL(ctx context.Context, opts *AccessURLReportOptions) AccessURLReport
Websocket(ctx context.Context, opts *WebsocketReportOptions) WebsocketReport
Database(ctx context.Context, opts *DatabaseReportOptions) DatabaseReport
WorkspaceProxy(ctx context.Context, opts *WorkspaceProxyReportOptions) WorkspaceProxyReport
}

// @typescript-generate Report
Expand All @@ -38,20 +40,22 @@ type Report struct {
// FailingSections is a list of sections that have failed their healthcheck.
FailingSections []string `json:"failing_sections"`

DERP derphealth.Report `json:"derp"`
AccessURL AccessURLReport `json:"access_url"`
Websocket WebsocketReport `json:"websocket"`
Database DatabaseReport `json:"database"`
DERP derphealth.Report `json:"derp"`
AccessURL AccessURLReport `json:"access_url"`
Websocket WebsocketReport `json:"websocket"`
Database DatabaseReport `json:"database"`
WorkspaceProxy WorkspaceProxyReport `json:"workspace_proxy"`

// The Coder version of the server that the report was generated on.
CoderVersion string `json:"coder_version"`
}

type ReportOptions struct {
AccessURL AccessURLReportOptions
Database DatabaseReportOptions
DerpHealth derphealth.ReportOptions
Websocket WebsocketReportOptions
AccessURL AccessURLReportOptions
Database DatabaseReportOptions
DerpHealth derphealth.ReportOptions
Websocket WebsocketReportOptions
WorkspaceProxy WorkspaceProxyReportOptions

Checker Checker
}
Expand All @@ -78,6 +82,11 @@ func (defaultChecker) Database(ctx context.Context, opts *DatabaseReportOptions)
return report
}

func (defaultChecker) WorkspaceProxy(ctx context.Context, opts *WorkspaceProxyReportOptions) (report WorkspaceProxyReport) {
report.Run(ctx, opts)
return report
}

func Run(ctx context.Context, opts *ReportOptions) *Report {
var (
wg sync.WaitGroup
Expand Down Expand Up @@ -136,6 +145,18 @@ func Run(ctx context.Context, opts *ReportOptions) *Report {
report.Database = opts.Checker.Database(ctx, &opts.Database)
}()

wg.Add(1)
go func() {
defer wg.Done()
defer func() {
if err := recover(); err != nil {
report.WorkspaceProxy.Error = ptr.Ref(fmt.Sprint(err))
}
}()

report.WorkspaceProxy = opts.Checker.WorkspaceProxy(ctx, &opts.WorkspaceProxy)
}()

report.CoderVersion = buildinfo.Version()
wg.Wait()

Expand All @@ -153,6 +174,9 @@ func Run(ctx context.Context, opts *ReportOptions) *Report {
if !report.Database.Healthy {
report.FailingSections = append(report.FailingSections, SectionDatabase)
}
if !report.WorkspaceProxy.Healthy {
report.FailingSections = append(report.FailingSections, SectionWorkspaceProxy)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

side thought: frontend does not use FailingSections at all. Maybe we should remove it for simplicity.

Copy link
Member Author

@johnstcn johnstcn Nov 24, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍 Will file a follow-up PR

Edit: #10854

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's beneficial to have a structure that can be navigated in a generic way.

for (let section of report.failing_sections) {
	console.log(report[section].reason);
}

Given a good design, it will "always works", even when new fields and structures are added. Granted, I'm not a proponent for the current design, just for the ability to do so.

}

report.Healthy = len(report.FailingSections) == 0

Expand All @@ -171,6 +195,9 @@ func Run(ctx context.Context, opts *ReportOptions) *Report {
if report.Database.Severity.Value() > report.Severity.Value() {
report.Severity = report.Database.Severity
}
if report.WorkspaceProxy.Severity.Value() > report.Severity.Value() {
report.Severity = report.WorkspaceProxy.Severity
}
return &report
}

Expand Down
75 changes: 69 additions & 6 deletions coderd/healthcheck/healthcheck_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,11 @@ import (
)

type testChecker struct {
DERPReport derphealth.Report
AccessURLReport healthcheck.AccessURLReport
WebsocketReport healthcheck.WebsocketReport
DatabaseReport healthcheck.DatabaseReport
DERPReport derphealth.Report
AccessURLReport healthcheck.AccessURLReport
WebsocketReport healthcheck.WebsocketReport
DatabaseReport healthcheck.DatabaseReport
WorkspaceProxyReport healthcheck.WorkspaceProxyReport
}

func (c *testChecker) DERP(context.Context, *derphealth.ReportOptions) derphealth.Report {
Expand All @@ -34,6 +35,10 @@ func (c *testChecker) Database(context.Context, *healthcheck.DatabaseReportOptio
return c.DatabaseReport
}

func (c *testChecker) WorkspaceProxy(context.Context, *healthcheck.WorkspaceProxyReportOptions) healthcheck.WorkspaceProxyReport {
return c.WorkspaceProxyReport
}

func TestHealthcheck(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -62,6 +67,10 @@ func TestHealthcheck(t *testing.T) {
Healthy: true,
Severity: health.SeverityOK,
},
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
Healthy: true,
Severity: health.SeverityOK,
},
},
healthy: true,
severity: health.SeverityOK,
Expand All @@ -85,6 +94,10 @@ func TestHealthcheck(t *testing.T) {
Healthy: true,
Severity: health.SeverityOK,
},
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
Healthy: true,
Severity: health.SeverityOK,
},
},
healthy: false,
severity: health.SeverityError,
Expand All @@ -109,6 +122,10 @@ func TestHealthcheck(t *testing.T) {
Healthy: true,
Severity: health.SeverityOK,
},
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
Healthy: true,
Severity: health.SeverityOK,
},
},
healthy: true,
severity: health.SeverityWarning,
Expand All @@ -132,6 +149,10 @@ func TestHealthcheck(t *testing.T) {
Healthy: true,
Severity: health.SeverityOK,
},
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
Healthy: true,
Severity: health.SeverityOK,
},
},
healthy: false,
severity: health.SeverityWarning,
Expand All @@ -155,6 +176,10 @@ func TestHealthcheck(t *testing.T) {
Healthy: true,
Severity: health.SeverityOK,
},
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
Healthy: true,
Severity: health.SeverityOK,
},
},
healthy: false,
severity: health.SeverityError,
Expand All @@ -178,12 +203,44 @@ func TestHealthcheck(t *testing.T) {
Healthy: false,
Severity: health.SeverityError,
},
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
Healthy: true,
Severity: health.SeverityOK,
},
},
healthy: false,
severity: health.SeverityError,
failingSections: []string{healthcheck.SectionDatabase},
}, {
name: "AllFail",
name: "ProxyFail",
checker: &testChecker{
DERPReport: derphealth.Report{
Healthy: true,
Severity: health.SeverityOK,
},
AccessURLReport: healthcheck.AccessURLReport{
Healthy: true,
Severity: health.SeverityOK,
},
WebsocketReport: healthcheck.WebsocketReport{
Healthy: true,
Severity: health.SeverityOK,
},
DatabaseReport: healthcheck.DatabaseReport{
Healthy: true,
Severity: health.SeverityOK,
},
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
Healthy: false,
Severity: health.SeverityError,
},
},
severity: health.SeverityError,
healthy: false,
failingSections: []string{healthcheck.SectionWorkspaceProxy},
}, {
name: "AllFail",
healthy: false,
checker: &testChecker{
DERPReport: derphealth.Report{
Healthy: false,
Expand All @@ -201,14 +258,18 @@ func TestHealthcheck(t *testing.T) {
Healthy: false,
Severity: health.SeverityError,
},
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
Healthy: false,
Severity: health.SeverityError,
},
},
healthy: false,
severity: health.SeverityError,
failingSections: []string{
healthcheck.SectionDERP,
healthcheck.SectionAccessURL,
healthcheck.SectionWebsocket,
healthcheck.SectionDatabase,
healthcheck.SectionWorkspaceProxy,
},
}} {
c := c
Expand All @@ -228,6 +289,8 @@ func TestHealthcheck(t *testing.T) {
assert.Equal(t, c.checker.AccessURLReport.Healthy, report.AccessURL.Healthy)
assert.Equal(t, c.checker.AccessURLReport.Severity, report.AccessURL.Severity)
assert.Equal(t, c.checker.WebsocketReport.Healthy, report.Websocket.Healthy)
assert.Equal(t, c.checker.WorkspaceProxyReport.Healthy, report.WorkspaceProxy.Healthy)
assert.Equal(t, c.checker.WorkspaceProxyReport.Warnings, report.WorkspaceProxy.Warnings)
assert.Equal(t, c.checker.WebsocketReport.Severity, report.Websocket.Severity)
assert.Equal(t, c.checker.DatabaseReport.Healthy, report.Database.Healthy)
assert.Equal(t, c.checker.DatabaseReport.Severity, report.Database.Severity)
Expand Down
Loading