Skip to content

Commit 411ce46

Browse files
johnstcnjohnstcn
authored
feat(coderd/healthcheck): add health check for proxy (#10846)
Adds a health check for workspace proxies: - Healthy iff all proxies are healthy and the same version, - Warning if some proxies are unhealthy, - Error if all proxies are unhealthy, or do not all have the same version.
1 parent b501046 commit 411ce46

File tree

15 files changed

+865
-35
lines changed

15 files changed

+865
-35
lines changed

‎coderd/apidoc/docs.go

+26
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/apidoc/swagger.json

+26
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/coderd.go

+18-4
Original file line numberDiff line numberDiff line change
@@ -135,10 +135,12 @@ type Options struct {
135135
AccessControlStore *atomic.Pointer[dbauthz.AccessControlStore]
136136
// AppSecurityKey is the crypto key used to sign and encrypt tokens related to
137137
// workspace applications. It consists of both a signing and encryption key.
138-
AppSecurityKey workspaceapps.SecurityKey
139-
HealthcheckFunc func(ctx context.Context, apiKey string) *healthcheck.Report
140-
HealthcheckTimeout time.Duration
141-
HealthcheckRefresh time.Duration
138+
AppSecurityKey workspaceapps.SecurityKey
139+
140+
HealthcheckFunc func(ctx context.Context, apiKey string) *healthcheck.Report
141+
HealthcheckTimeout time.Duration
142+
HealthcheckRefresh time.Duration
143+
WorkspaceProxiesFetchUpdater *atomic.Pointer[healthcheck.WorkspaceProxiesFetchUpdater]
142144

143145
// OAuthSigningKey is the crypto key used to sign and encrypt state strings
144146
// related to OAuth. This is a symmetric secret key using hmac to sign payloads.
@@ -396,6 +398,13 @@ func New(options *Options) *API {
396398
*options.UpdateCheckOptions,
397399
)
398400
}
401+
402+
if options.WorkspaceProxiesFetchUpdater == nil {
403+
options.WorkspaceProxiesFetchUpdater = &atomic.Pointer[healthcheck.WorkspaceProxiesFetchUpdater]{}
404+
var wpfu healthcheck.WorkspaceProxiesFetchUpdater = &healthcheck.AGPLWorkspaceProxiesFetchUpdater{}
405+
options.WorkspaceProxiesFetchUpdater.Store(&wpfu)
406+
}
407+
399408
if options.HealthcheckFunc == nil {
400409
options.HealthcheckFunc = func(ctx context.Context, apiKey string) *healthcheck.Report {
401410
return healthcheck.Run(ctx, &healthcheck.ReportOptions{
@@ -413,9 +422,14 @@ func New(options *Options) *API {
413422
DerpHealth: derphealth.ReportOptions{
414423
DERPMap: api.DERPMap(),
415424
},
425+
WorkspaceProxy: healthcheck.WorkspaceProxyReportOptions{
426+
CurrentVersion: buildinfo.Version(),
427+
WorkspaceProxiesFetchUpdater: *(options.WorkspaceProxiesFetchUpdater).Load(),
428+
},
416429
})
417430
}
418431
}
432+
419433
if options.HealthcheckTimeout == 0 {
420434
options.HealthcheckTimeout = 30 * time.Second
421435
}

‎coderd/healthcheck/healthcheck.go

+39-12
Original file line numberDiff line numberDiff line change
@@ -13,17 +13,19 @@ import (
1313
)
1414

1515
const (
16-
SectionDERP string = "DERP"
17-
SectionAccessURL string = "AccessURL"
18-
SectionWebsocket string = "Websocket"
19-
SectionDatabase string = "Database"
16+
SectionDERP string = "DERP"
17+
SectionAccessURL string = "AccessURL"
18+
SectionWebsocket string = "Websocket"
19+
SectionDatabase string = "Database"
20+
SectionWorkspaceProxy string = "WorkspaceProxy"
2021
)
2122

2223
type Checker interface {
2324
DERP(ctx context.Context, opts *derphealth.ReportOptions) derphealth.Report
2425
AccessURL(ctx context.Context, opts *AccessURLReportOptions) AccessURLReport
2526
Websocket(ctx context.Context, opts *WebsocketReportOptions) WebsocketReport
2627
Database(ctx context.Context, opts *DatabaseReportOptions) DatabaseReport
28+
WorkspaceProxy(ctx context.Context, opts *WorkspaceProxyReportOptions) WorkspaceProxyReport
2729
}
2830

2931
// @typescript-generate Report
@@ -38,20 +40,22 @@ type Report struct {
3840
// FailingSections is a list of sections that have failed their healthcheck.
3941
FailingSections []string `json:"failing_sections"`
4042

41-
DERP derphealth.Report `json:"derp"`
42-
AccessURL AccessURLReport `json:"access_url"`
43-
Websocket WebsocketReport `json:"websocket"`
44-
Database DatabaseReport `json:"database"`
43+
DERP derphealth.Report `json:"derp"`
44+
AccessURL AccessURLReport `json:"access_url"`
45+
Websocket WebsocketReport `json:"websocket"`
46+
Database DatabaseReport `json:"database"`
47+
WorkspaceProxy WorkspaceProxyReport `json:"workspace_proxy"`
4548

4649
// The Coder version of the server that the report was generated on.
4750
CoderVersion string `json:"coder_version"`
4851
}
4952

5053
type ReportOptions struct {
51-
AccessURL AccessURLReportOptions
52-
Database DatabaseReportOptions
53-
DerpHealth derphealth.ReportOptions
54-
Websocket WebsocketReportOptions
54+
AccessURL AccessURLReportOptions
55+
Database DatabaseReportOptions
56+
DerpHealth derphealth.ReportOptions
57+
Websocket WebsocketReportOptions
58+
WorkspaceProxy WorkspaceProxyReportOptions
5559

5660
Checker Checker
5761
}
@@ -78,6 +82,11 @@ func (defaultChecker) Database(ctx context.Context, opts *DatabaseReportOptions)
7882
return report
7983
}
8084

85+
func (defaultChecker) WorkspaceProxy(ctx context.Context, opts *WorkspaceProxyReportOptions) (report WorkspaceProxyReport) {
86+
report.Run(ctx, opts)
87+
return report
88+
}
89+
8190
func Run(ctx context.Context, opts *ReportOptions) *Report {
8291
var (
8392
wg sync.WaitGroup
@@ -136,6 +145,18 @@ func Run(ctx context.Context, opts *ReportOptions) *Report {
136145
report.Database = opts.Checker.Database(ctx, &opts.Database)
137146
}()
138147

148+
wg.Add(1)
149+
go func() {
150+
defer wg.Done()
151+
defer func() {
152+
if err := recover(); err != nil {
153+
report.WorkspaceProxy.Error = ptr.Ref(fmt.Sprint(err))
154+
}
155+
}()
156+
157+
report.WorkspaceProxy = opts.Checker.WorkspaceProxy(ctx, &opts.WorkspaceProxy)
158+
}()
159+
139160
report.CoderVersion = buildinfo.Version()
140161
wg.Wait()
141162

@@ -153,6 +174,9 @@ func Run(ctx context.Context, opts *ReportOptions) *Report {
153174
if !report.Database.Healthy {
154175
report.FailingSections = append(report.FailingSections, SectionDatabase)
155176
}
177+
if !report.WorkspaceProxy.Healthy {
178+
report.FailingSections = append(report.FailingSections, SectionWorkspaceProxy)
179+
}
156180

157181
report.Healthy = len(report.FailingSections) == 0
158182

@@ -171,6 +195,9 @@ func Run(ctx context.Context, opts *ReportOptions) *Report {
171195
if report.Database.Severity.Value() > report.Severity.Value() {
172196
report.Severity = report.Database.Severity
173197
}
198+
if report.WorkspaceProxy.Severity.Value() > report.Severity.Value() {
199+
report.Severity = report.WorkspaceProxy.Severity
200+
}
174201
return &report
175202
}
176203

‎coderd/healthcheck/healthcheck_test.go

+69-6
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,11 @@ import (
1212
)
1313

1414
type testChecker struct {
15-
DERPReport derphealth.Report
16-
AccessURLReport healthcheck.AccessURLReport
17-
WebsocketReport healthcheck.WebsocketReport
18-
DatabaseReport healthcheck.DatabaseReport
15+
DERPReport derphealth.Report
16+
AccessURLReport healthcheck.AccessURLReport
17+
WebsocketReport healthcheck.WebsocketReport
18+
DatabaseReport healthcheck.DatabaseReport
19+
WorkspaceProxyReport healthcheck.WorkspaceProxyReport
1920
}
2021

2122
func (c *testChecker) DERP(context.Context, *derphealth.ReportOptions) derphealth.Report {
@@ -34,6 +35,10 @@ func (c *testChecker) Database(context.Context, *healthcheck.DatabaseReportOptio
3435
return c.DatabaseReport
3536
}
3637

38+
func (c *testChecker) WorkspaceProxy(context.Context, *healthcheck.WorkspaceProxyReportOptions) healthcheck.WorkspaceProxyReport {
39+
return c.WorkspaceProxyReport
40+
}
41+
3742
func TestHealthcheck(t *testing.T) {
3843
t.Parallel()
3944

@@ -62,6 +67,10 @@ func TestHealthcheck(t *testing.T) {
6267
Healthy: true,
6368
Severity: health.SeverityOK,
6469
},
70+
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
71+
Healthy: true,
72+
Severity: health.SeverityOK,
73+
},
6574
},
6675
healthy: true,
6776
severity: health.SeverityOK,
@@ -85,6 +94,10 @@ func TestHealthcheck(t *testing.T) {
8594
Healthy: true,
8695
Severity: health.SeverityOK,
8796
},
97+
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
98+
Healthy: true,
99+
Severity: health.SeverityOK,
100+
},
88101
},
89102
healthy: false,
90103
severity: health.SeverityError,
@@ -109,6 +122,10 @@ func TestHealthcheck(t *testing.T) {
109122
Healthy: true,
110123
Severity: health.SeverityOK,
111124
},
125+
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
126+
Healthy: true,
127+
Severity: health.SeverityOK,
128+
},
112129
},
113130
healthy: true,
114131
severity: health.SeverityWarning,
@@ -132,6 +149,10 @@ func TestHealthcheck(t *testing.T) {
132149
Healthy: true,
133150
Severity: health.SeverityOK,
134151
},
152+
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
153+
Healthy: true,
154+
Severity: health.SeverityOK,
155+
},
135156
},
136157
healthy: false,
137158
severity: health.SeverityWarning,
@@ -155,6 +176,10 @@ func TestHealthcheck(t *testing.T) {
155176
Healthy: true,
156177
Severity: health.SeverityOK,
157178
},
179+
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
180+
Healthy: true,
181+
Severity: health.SeverityOK,
182+
},
158183
},
159184
healthy: false,
160185
severity: health.SeverityError,
@@ -178,12 +203,44 @@ func TestHealthcheck(t *testing.T) {
178203
Healthy: false,
179204
Severity: health.SeverityError,
180205
},
206+
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
207+
Healthy: true,
208+
Severity: health.SeverityOK,
209+
},
181210
},
182211
healthy: false,
183212
severity: health.SeverityError,
184213
failingSections: []string{healthcheck.SectionDatabase},
185214
}, {
186-
name: "AllFail",
215+
name: "ProxyFail",
216+
checker: &testChecker{
217+
DERPReport: derphealth.Report{
218+
Healthy: true,
219+
Severity: health.SeverityOK,
220+
},
221+
AccessURLReport: healthcheck.AccessURLReport{
222+
Healthy: true,
223+
Severity: health.SeverityOK,
224+
},
225+
WebsocketReport: healthcheck.WebsocketReport{
226+
Healthy: true,
227+
Severity: health.SeverityOK,
228+
},
229+
DatabaseReport: healthcheck.DatabaseReport{
230+
Healthy: true,
231+
Severity: health.SeverityOK,
232+
},
233+
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
234+
Healthy: false,
235+
Severity: health.SeverityError,
236+
},
237+
},
238+
severity: health.SeverityError,
239+
healthy: false,
240+
failingSections: []string{healthcheck.SectionWorkspaceProxy},
241+
}, {
242+
name: "AllFail",
243+
healthy: false,
187244
checker: &testChecker{
188245
DERPReport: derphealth.Report{
189246
Healthy: false,
@@ -201,14 +258,18 @@ func TestHealthcheck(t *testing.T) {
201258
Healthy: false,
202259
Severity: health.SeverityError,
203260
},
261+
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
262+
Healthy: false,
263+
Severity: health.SeverityError,
264+
},
204265
},
205-
healthy: false,
206266
severity: health.SeverityError,
207267
failingSections: []string{
208268
healthcheck.SectionDERP,
209269
healthcheck.SectionAccessURL,
210270
healthcheck.SectionWebsocket,
211271
healthcheck.SectionDatabase,
272+
healthcheck.SectionWorkspaceProxy,
212273
},
213274
}} {
214275
c := c
@@ -228,6 +289,8 @@ func TestHealthcheck(t *testing.T) {
228289
assert.Equal(t, c.checker.AccessURLReport.Healthy, report.AccessURL.Healthy)
229290
assert.Equal(t, c.checker.AccessURLReport.Severity, report.AccessURL.Severity)
230291
assert.Equal(t, c.checker.WebsocketReport.Healthy, report.Websocket.Healthy)
292+
assert.Equal(t, c.checker.WorkspaceProxyReport.Healthy, report.WorkspaceProxy.Healthy)
293+
assert.Equal(t, c.checker.WorkspaceProxyReport.Warnings, report.WorkspaceProxy.Warnings)
231294
assert.Equal(t, c.checker.WebsocketReport.Severity, report.Websocket.Severity)
232295
assert.Equal(t, c.checker.DatabaseReport.Healthy, report.Database.Healthy)
233296
assert.Equal(t, c.checker.DatabaseReport.Severity, report.Database.Severity)

0 commit comments

Comments
 (0)