Skip to content

Commit 0016b02

Browse files
deansheatherdeansheather
authored
chore: add test for workspace proxy derp meshing (#12220)
- Reworks the proxy registration loop into a struct (so I can add a `RegisterNow` method) - Changes the proxy registration loop interval to 15s (previously 30s) - Adds test which tests bidirectional DERP meshing on all possible paths between 6 workspace proxy replicas Related to coder/customers#438
1 parent 5c6974e commit 0016b02

File tree

5 files changed

+396
-109
lines changed

5 files changed

+396
-109
lines changed

‎enterprise/coderd/coderdenttest/proxytest.go

+35-14
Original file line numberDiff line numberDiff line change
@@ -38,15 +38,29 @@ type ProxyOptions struct {
3838
// ProxyURL is optional
3939
ProxyURL *url.URL
4040

41+
// Token is optional. If specified, a new workspace proxy region will not be
42+
// created, and the proxy will become a replica of the existing proxy
43+
// region.
44+
Token string
45+
4146
// FlushStats is optional
4247
FlushStats chan chan<- struct{}
4348
}
4449

45-
// NewWorkspaceProxy will configure a wsproxy.Server with the given options.
46-
// The new wsproxy will register itself with the given coderd.API instance.
47-
// The first user owner client is required to create the wsproxy on the coderd
48-
// api server.
49-
func NewWorkspaceProxy(t *testing.T, coderdAPI *coderd.API, owner *codersdk.Client, options *ProxyOptions) *wsproxy.Server {
50+
type WorkspaceProxy struct {
51+
*wsproxy.Server
52+
53+
ServerURL *url.URL
54+
}
55+
56+
// NewWorkspaceProxyReplica will configure a wsproxy.Server with the given
57+
// options. The new wsproxy replica will register itself with the given
58+
// coderd.API instance.
59+
//
60+
// If a token is not provided, a new workspace proxy region is created using the
61+
// owner client. If a token is provided, the proxy will become a replica of the
62+
// existing proxy region.
63+
func NewWorkspaceProxyReplica(t *testing.T, coderdAPI *coderd.API, owner *codersdk.Client, options *ProxyOptions) WorkspaceProxy {
5064
ctx, cancelFunc := context.WithCancel(context.Background())
5165
t.Cleanup(cancelFunc)
5266

@@ -107,11 +121,15 @@ func NewWorkspaceProxy(t *testing.T, coderdAPI *coderd.API, owner *codersdk.Clie
107121
options.Name = namesgenerator.GetRandomName(1)
108122
}
109123

110-
proxyRes, err := owner.CreateWorkspaceProxy(ctx, codersdk.CreateWorkspaceProxyRequest{
111-
Name: options.Name,
112-
Icon: "/emojis/flag.png",
113-
})
114-
require.NoError(t, err, "failed to create workspace proxy")
124+
token := options.Token
125+
if token == "" {
126+
proxyRes, err := owner.CreateWorkspaceProxy(ctx, codersdk.CreateWorkspaceProxyRequest{
127+
Name: options.Name,
128+
Icon: "/emojis/flag.png",
129+
})
130+
require.NoError(t, err, "failed to create workspace proxy")
131+
token = proxyRes.ProxyToken
132+
}
115133

116134
// Inherit collector options from coderd, but keep the wsproxy reporter.
117135
statsCollectorOptions := coderdAPI.Options.WorkspaceAppsStatsCollectorOptions
@@ -121,7 +139,7 @@ func NewWorkspaceProxy(t *testing.T, coderdAPI *coderd.API, owner *codersdk.Clie
121139
}
122140

123141
wssrv, err := wsproxy.New(ctx, &wsproxy.Options{
124-
Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug),
142+
Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug).With(slog.F("server_url", serverURL.String())),
125143
Experiments: options.Experiments,
126144
DashboardURL: coderdAPI.AccessURL,
127145
AccessURL: accessURL,
@@ -131,14 +149,14 @@ func NewWorkspaceProxy(t *testing.T, coderdAPI *coderd.API, owner *codersdk.Clie
131149
Tracing: coderdAPI.TracerProvider,
132150
APIRateLimit: coderdAPI.APIRateLimit,
133151
SecureAuthCookie: coderdAPI.SecureAuthCookie,
134-
ProxySessionToken: proxyRes.ProxyToken,
152+
ProxySessionToken: token,
135153
DisablePathApps: options.DisablePathApps,
136154
// We need a new registry to not conflict with the coderd internal
137155
// proxy metrics.
138156
PrometheusRegistry: prometheus.NewRegistry(),
139157
DERPEnabled: !options.DerpDisabled,
140158
DERPOnly: options.DerpOnly,
141-
DERPServerRelayAddress: accessURL.String(),
159+
DERPServerRelayAddress: serverURL.String(),
142160
StatsCollectorOptions: statsCollectorOptions,
143161
})
144162
require.NoError(t, err)
@@ -151,5 +169,8 @@ func NewWorkspaceProxy(t *testing.T, coderdAPI *coderd.API, owner *codersdk.Clie
151169
handler = wssrv.Handler
152170
mutex.Unlock()
153171

154-
return wssrv
172+
return WorkspaceProxy{
173+
Server: wssrv,
174+
ServerURL: serverURL,
175+
}
155176
}

‎enterprise/coderd/workspaceproxy_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ func TestRegions(t *testing.T) {
9999
require.NoError(t, err)
100100

101101
const proxyName = "hello"
102-
_ = coderdenttest.NewWorkspaceProxy(t, api, client, &coderdenttest.ProxyOptions{
102+
_ = coderdenttest.NewWorkspaceProxyReplica(t, api, client, &coderdenttest.ProxyOptions{
103103
Name: proxyName,
104104
AppHostname: appHostname + ".proxy",
105105
})
@@ -734,7 +734,7 @@ func TestReconnectingPTYSignedToken(t *testing.T) {
734734
proxyURL, err := url.Parse(fmt.Sprintf("https://%s.com", namesgenerator.GetRandomName(1)))
735735
require.NoError(t, err)
736736

737-
_ = coderdenttest.NewWorkspaceProxy(t, api, client, &coderdenttest.ProxyOptions{
737+
_ = coderdenttest.NewWorkspaceProxyReplica(t, api, client, &coderdenttest.ProxyOptions{
738738
Name: namesgenerator.GetRandomName(1),
739739
ProxyURL: proxyURL,
740740
AppHostname: "*.sub.example.com",

‎enterprise/wsproxy/wsproxy.go

+14-12
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ type Server struct {
128128
ctx context.Context
129129
cancel context.CancelFunc
130130
derpCloseFunc func()
131-
registerDone <-chan struct{}
131+
registerLoop *wsproxysdk.RegisterWorkspaceProxyLoop
132132
}
133133

134134
// New creates a new workspace proxy server. This requires a primary coderd
@@ -210,7 +210,7 @@ func New(ctx context.Context, opts *Options) (*Server, error) {
210210
// goroutine to periodically re-register.
211211
replicaID := uuid.New()
212212
osHostname := cliutil.Hostname()
213-
regResp, registerDone, err := client.RegisterWorkspaceProxyLoop(ctx, wsproxysdk.RegisterWorkspaceProxyLoopOpts{
213+
registerLoop, regResp, err := client.RegisterWorkspaceProxyLoop(ctx, wsproxysdk.RegisterWorkspaceProxyLoopOpts{
214214
Logger: opts.Logger,
215215
Request: wsproxysdk.RegisterWorkspaceProxyRequest{
216216
AccessURL: opts.AccessURL.String(),
@@ -230,12 +230,13 @@ func New(ctx context.Context, opts *Options) (*Server, error) {
230230
if err != nil {
231231
return nil, xerrors.Errorf("register proxy: %w", err)
232232
}
233-
s.registerDone = registerDone
234-
err = s.handleRegister(ctx, regResp)
233+
s.registerLoop = registerLoop
234+
235+
derpServer.SetMeshKey(regResp.DERPMeshKey)
236+
err = s.handleRegister(regResp)
235237
if err != nil {
236238
return nil, xerrors.Errorf("handle register: %w", err)
237239
}
238-
derpServer.SetMeshKey(regResp.DERPMeshKey)
239240

240241
secKey, err := workspaceapps.KeyFromString(regResp.AppSecurityKey)
241242
if err != nil {
@@ -409,16 +410,16 @@ func New(ctx context.Context, opts *Options) (*Server, error) {
409410
return s, nil
410411
}
411412

413+
func (s *Server) RegisterNow() error {
414+
_, err := s.registerLoop.RegisterNow()
415+
return err
416+
}
417+
412418
func (s *Server) Close() error {
413419
s.cancel()
414420

415421
var err error
416-
registerDoneWaitTicker := time.NewTicker(11 * time.Second) // the attempt timeout is 10s
417-
select {
418-
case <-registerDoneWaitTicker.C:
419-
err = multierror.Append(err, xerrors.New("timed out waiting for registerDone"))
420-
case <-s.registerDone:
421-
}
422+
s.registerLoop.Close()
422423
s.derpCloseFunc()
423424
appServerErr := s.AppServer.Close()
424425
if appServerErr != nil {
@@ -437,11 +438,12 @@ func (*Server) mutateRegister(_ *wsproxysdk.RegisterWorkspaceProxyRequest) {
437438
// package in the primary and update req.ReplicaError accordingly.
438439
}
439440

440-
func (s *Server) handleRegister(_ context.Context, res wsproxysdk.RegisterWorkspaceProxyResponse) error {
441+
func (s *Server) handleRegister(res wsproxysdk.RegisterWorkspaceProxyResponse) error {
441442
addresses := make([]string, len(res.SiblingReplicas))
442443
for i, replica := range res.SiblingReplicas {
443444
addresses[i] = replica.RelayAddress
444445
}
446+
s.Logger.Debug(s.ctx, "setting DERP mesh sibling addresses", slog.F("addresses", addresses))
445447
s.derpMesh.SetAddresses(addresses, false)
446448

447449
s.latestDERPMap.Store(res.DERPMap)

0 commit comments

Comments
 (0)