Skip to content

Commit b7f4f3a

Browse files
EmyrkEmyrk
authored
chore: Implement workspace proxy going away (graceful shutdown) (coder#7459)
* chore: Implement workspace proxy going away When a workspace proxy shuts down, the health status of that proxy should immediately be updated. This is purely a courtesy and technically not required
1 parent a42a36a commit b7f4f3a

File tree

13 files changed

+202
-24
lines changed

13 files changed

+202
-24
lines changed

‎coderd/apidoc/docs.go

Lines changed: 30 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/apidoc/swagger.json

Lines changed: 26 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎codersdk/workspaceproxy.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@ import (
1515
type ProxyHealthStatus string
1616

1717
const (
18-
// ProxyReachable means the proxy access url is reachable and returns a healthy
18+
// ProxyHealthy means the proxy access url is reachable and returns a healthy
1919
// status code.
20-
ProxyReachable ProxyHealthStatus = "reachable"
20+
ProxyHealthy ProxyHealthStatus = "ok"
2121
// ProxyUnreachable means the proxy access url is not responding.
2222
ProxyUnreachable ProxyHealthStatus = "unreachable"
2323
// ProxyUnhealthy means the proxy access url is responding, but there is some

‎docs/api/enterprise.md

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1192,7 +1192,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaceproxies \
11921192
"errors": ["string"],
11931193
"warnings": ["string"]
11941194
},
1195-
"status": "reachable"
1195+
"status": "ok"
11961196
},
11971197
"updated_at": "2019-08-24T14:15:22Z",
11981198
"url": "string",
@@ -1234,7 +1234,7 @@ Status Code **200**
12341234

12351235
| Property | Value |
12361236
| -------- | -------------- |
1237-
| `status` | `reachable` |
1237+
| `status` | `ok` |
12381238
| `status` | `unreachable` |
12391239
| `status` | `unhealthy` |
12401240
| `status` | `unregistered` |
@@ -1289,7 +1289,7 @@ curl -X POST http://coder-server:8080/api/v2/workspaceproxies \
12891289
"errors": ["string"],
12901290
"warnings": ["string"]
12911291
},
1292-
"status": "reachable"
1292+
"status": "ok"
12931293
},
12941294
"updated_at": "2019-08-24T14:15:22Z",
12951295
"url": "string",
@@ -1342,7 +1342,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaceproxies/{workspaceproxy} \
13421342
"errors": ["string"],
13431343
"warnings": ["string"]
13441344
},
1345-
"status": "reachable"
1345+
"status": "ok"
13461346
},
13471347
"updated_at": "2019-08-24T14:15:22Z",
13481348
"url": "string",
@@ -1453,7 +1453,7 @@ curl -X PATCH http://coder-server:8080/api/v2/workspaceproxies/{workspaceproxy}
14531453
"errors": ["string"],
14541454
"warnings": ["string"]
14551455
},
1456-
"status": "reachable"
1456+
"status": "ok"
14571457
},
14581458
"updated_at": "2019-08-24T14:15:22Z",
14591459
"url": "string",

‎docs/api/schemas.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3476,7 +3476,7 @@ Parameter represents a set value for the scope.
34763476
## codersdk.ProxyHealthStatus
34773477

34783478
```json
3479-
"reachable"
3479+
"ok"
34803480
```
34813481

34823482
### Properties
@@ -3485,7 +3485,7 @@ Parameter represents a set value for the scope.
34853485

34863486
| Value |
34873487
| -------------- |
3488-
| `reachable` |
3488+
| `ok` |
34893489
| `unreachable` |
34903490
| `unhealthy` |
34913491
| `unregistered` |
@@ -5361,7 +5361,7 @@ Parameter represents a set value for the scope.
53615361
"errors": ["string"],
53625362
"warnings": ["string"]
53635363
},
5364-
"status": "reachable"
5364+
"status": "ok"
53655365
},
53665366
"updated_at": "2019-08-24T14:15:22Z",
53675367
"url": "string",
@@ -5393,7 +5393,7 @@ Parameter represents a set value for the scope.
53935393
"errors": ["string"],
53945394
"warnings": ["string"]
53955395
},
5396-
"status": "reachable"
5396+
"status": "ok"
53975397
}
53985398
```
53995399

‎enterprise/cli/proxyserver.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,7 @@ func (*RootCmd) proxyServer() *clibase.Cmd {
249249
if err != nil {
250250
return xerrors.Errorf("create workspace proxy: %w", err)
251251
}
252+
closers.Add(func() { _ = proxy.Close() })
252253

253254
shutdownConnsCtx, shutdownConns := context.WithCancel(ctx)
254255
defer shutdownConns()

‎enterprise/coderd/coderd.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ func New(ctx context.Context, options *Options) (*API, error) {
113113
)
114114
r.Post("/issue-signed-app-token", api.workspaceProxyIssueSignedAppToken)
115115
r.Post("/register", api.workspaceProxyRegister)
116+
r.Post("/goingaway", api.workspaceProxyGoingAway)
116117
})
117118
r.Route("/{workspaceproxy}", func(r chi.Router) {
118119
r.Use(
@@ -239,7 +240,7 @@ func New(ctx context.Context, options *Options) (*API, error) {
239240
if api.AGPL.Experiments.Enabled(codersdk.ExperimentMoons) {
240241
// Proxy health is a moon feature.
241242
api.ProxyHealth, err = proxyhealth.New(&proxyhealth.Options{
242-
Interval: time.Minute * 1,
243+
Interval: options.ProxyHealthInterval,
243244
DB: api.Database,
244245
Logger: options.Logger.Named("proxyhealth"),
245246
Client: api.HTTPClient,

‎enterprise/coderd/coderdenttest/coderdenttest.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ type Options struct {
4848
EntitlementsUpdateInterval time.Duration
4949
SCIMAPIKey []byte
5050
UserWorkspaceQuota int
51+
ProxyHealthInterval time.Duration
5152
}
5253

5354
// New constructs a codersdk client connected to an in-memory Enterprise API instance.
@@ -74,6 +75,7 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c
7475
Options: oop,
7576
EntitlementsUpdateInterval: options.EntitlementsUpdateInterval,
7677
Keys: Keys,
78+
ProxyHealthInterval: options.ProxyHealthInterval,
7779
})
7880
assert.NoError(t, err)
7981
setHandler(coderAPI.AGPL.RootHandler)

‎enterprise/coderd/workspaceproxy.go

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -68,11 +68,7 @@ func (api *API) regions(rw http.ResponseWriter, r *http.Request) {
6868
continue
6969
}
7070

71-
health, ok := proxyHealth[proxy.ID]
72-
if !ok {
73-
health.Status = proxyhealth.Unknown
74-
}
75-
71+
health := proxyHealth[proxy.ID]
7672
regions = append(regions, codersdk.Region{
7773
ID: proxy.ID,
7874
Name: proxy.Name,
@@ -423,7 +419,7 @@ func (api *API) workspaceProxyRegister(rw http.ResponseWriter, r *http.Request)
423419
// Log: api.Logger,
424420
// Request: r,
425421
// Action: database.AuditActionWrite,
426-
//})
422+
// })
427423
)
428424
// aReq.Old = proxy
429425
// defer commitAudit()
@@ -473,6 +469,33 @@ func (api *API) workspaceProxyRegister(rw http.ResponseWriter, r *http.Request)
473469
go api.forceWorkspaceProxyHealthUpdate(api.ctx)
474470
}
475471

472+
// workspaceProxyGoingAway is used to tell coderd that the workspace proxy is
473+
// shutting down and going away. The main purpose of this function is for the
474+
// health status of the workspace proxy to be more quickly updated when we know
475+
// that the proxy is going to be unhealthy. This does not delete the workspace
476+
// or cause any other side effects.
477+
// If the workspace proxy comes back online, even without a register, it will
478+
// be found healthy again by the normal checks.
479+
// @Summary Workspace proxy going away
480+
// @ID workspace-proxy-going-away
481+
// @Security CoderSessionToken
482+
// @Produce json
483+
// @Tags Enterprise
484+
// @Success 201 {object} codersdk.Response
485+
// @Router /workspaceproxies/me/goingaway [post]
486+
// @x-apidocgen {"skip": true}
487+
func (api *API) workspaceProxyGoingAway(rw http.ResponseWriter, r *http.Request) {
488+
ctx := r.Context()
489+
490+
// Force a health update to happen immediately. The proxy should
491+
// not return a successful response if it is going away.
492+
go api.forceWorkspaceProxyHealthUpdate(api.ctx)
493+
494+
httpapi.Write(ctx, rw, http.StatusOK, codersdk.Response{
495+
Message: "OK",
496+
})
497+
}
498+
476499
// reconnectingPTYSignedToken issues a signed app token for use when connecting
477500
// to the reconnecting PTY websocket on an external workspace proxy. This is set
478501
// by the client as a query parameter when connecting.
@@ -588,6 +611,9 @@ func convertProxies(p []database.WorkspaceProxy, statuses map[uuid.UUID]proxyhea
588611
}
589612

590613
func convertProxy(p database.WorkspaceProxy, status proxyhealth.ProxyStatus) codersdk.WorkspaceProxy {
614+
if status.Status == "" {
615+
status.Status = proxyhealth.Unknown
616+
}
591617
return codersdk.WorkspaceProxy{
592618
ID: p.ID,
593619
Name: p.Name,

‎enterprise/coderd/workspaceproxy_test.go

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"net/http/httputil"
88
"net/url"
99
"testing"
10+
"time"
1011

1112
"github.com/google/uuid"
1213
"github.com/moby/moby/pkg/namesgenerator"
@@ -172,6 +173,69 @@ func TestRegions(t *testing.T) {
172173
require.Error(t, err)
173174
require.Empty(t, regions)
174175
})
176+
177+
t.Run("GoingAway", func(t *testing.T) {
178+
t.Parallel()
179+
180+
dv := coderdtest.DeploymentValues(t)
181+
dv.Experiments = []string{
182+
string(codersdk.ExperimentMoons),
183+
"*",
184+
}
185+
186+
db, pubsub := dbtestutil.NewDB(t)
187+
188+
ctx := testutil.Context(t, testutil.WaitLong)
189+
190+
client, closer, api := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
191+
Options: &coderdtest.Options{
192+
AppHostname: appHostname,
193+
Database: db,
194+
Pubsub: pubsub,
195+
DeploymentValues: dv,
196+
},
197+
// The interval is set to 1 hour so the proxy health
198+
// check will never happen manually. All checks will be
199+
// forced updates.
200+
ProxyHealthInterval: time.Hour,
201+
})
202+
t.Cleanup(func() {
203+
_ = closer.Close()
204+
})
205+
_ = coderdtest.CreateFirstUser(t, client)
206+
_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
207+
Features: license.Features{
208+
codersdk.FeatureWorkspaceProxy: 1,
209+
},
210+
})
211+
212+
const proxyName = "testproxy"
213+
proxy := coderdenttest.NewWorkspaceProxy(t, api, client, &coderdenttest.ProxyOptions{
214+
Name: proxyName,
215+
})
216+
_ = proxy
217+
218+
require.Eventuallyf(t, func() bool {
219+
proxy, err := client.WorkspaceProxyByName(ctx, proxyName)
220+
if err != nil {
221+
// We are testing the going away, not the initial healthy.
222+
// Just force an update to change this to healthy.
223+
_ = api.ProxyHealth.ForceUpdate(ctx)
224+
return false
225+
}
226+
return proxy.Status.Status == codersdk.ProxyHealthy
227+
}, testutil.WaitShort, testutil.IntervalFast, "proxy never became healthy")
228+
229+
_ = proxy.Close()
230+
// The proxy should tell the primary on close that is is no longer healthy.
231+
require.Eventuallyf(t, func() bool {
232+
proxy, err := client.WorkspaceProxyByName(ctx, proxyName)
233+
if err != nil {
234+
return false
235+
}
236+
return proxy.Status.Status == codersdk.ProxyUnhealthy
237+
}, testutil.WaitShort, testutil.IntervalFast, "proxy never became unhealthy after close")
238+
})
175239
}
176240

177241
func TestWorkspaceProxyCRUD(t *testing.T) {

0 commit comments

Comments
 (0)