Skip to content

Commit eae4c3a

Browse files
deansheatherdeansheather
committed
chore: add derpserver to proxy, add proxies to derpmap
1 parent bb0a38b commit eae4c3a

File tree

9 files changed

+139
-34
lines changed

9 files changed

+139
-34
lines changed

‎cli/server.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -759,7 +759,7 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
759759

760760
if cfg.Prometheus.Enable {
761761
// Agent metrics require reference to the tailnet coordinator, so must be initiated after Coder API.
762-
closeAgentsFunc, err := prometheusmetrics.Agents(ctx, logger, options.PrometheusRegistry, coderAPI.Database, &coderAPI.TailnetCoordinator, options.DERPMap, coderAPI.Options.AgentInactiveDisconnectTimeout, 0)
762+
closeAgentsFunc, err := prometheusmetrics.Agents(ctx, logger, options.PrometheusRegistry, coderAPI.Database, &coderAPI.TailnetCoordinator, coderAPI.DERPMap, coderAPI.Options.AgentInactiveDisconnectTimeout, 0)
763763
if err != nil {
764764
return xerrors.Errorf("register agents prometheus metric: %w", err)
765765
}

‎coderd/coderd.go

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,11 @@ import (
3535
"tailscale.com/types/key"
3636
"tailscale.com/util/singleflight"
3737

38-
"cdr.dev/slog"
39-
40-
"github.com/coder/coder/buildinfo"
41-
"github.com/coder/coder/codersdk/agentsdk"
42-
4338
// Used for swagger docs.
4439
_ "github.com/coder/coder/coderd/apidoc"
40+
41+
"cdr.dev/slog"
42+
"github.com/coder/coder/buildinfo"
4543
"github.com/coder/coder/coderd/audit"
4644
"github.com/coder/coder/coderd/awsidentity"
4745
"github.com/coder/coder/coderd/database"
@@ -63,6 +61,7 @@ import (
6361
"github.com/coder/coder/coderd/workspaceapps"
6462
"github.com/coder/coder/coderd/wsconncache"
6563
"github.com/coder/coder/codersdk"
64+
"github.com/coder/coder/codersdk/agentsdk"
6665
"github.com/coder/coder/provisionerd/proto"
6766
"github.com/coder/coder/provisionersdk"
6867
"github.com/coder/coder/site"
@@ -251,14 +250,6 @@ func New(options *Options) *API {
251250
v := schedule.NewAGPLTemplateScheduleStore()
252251
options.TemplateScheduleStore.Store(&v)
253252
}
254-
if options.HealthcheckFunc == nil {
255-
options.HealthcheckFunc = func(ctx context.Context) (*healthcheck.Report, error) {
256-
return healthcheck.Run(ctx, &healthcheck.ReportOptions{
257-
AccessURL: options.AccessURL,
258-
DERPMap: options.DERPMap.Clone(),
259-
})
260-
}
261-
}
262253
if options.HealthcheckTimeout == 0 {
263254
options.HealthcheckTimeout = 30 * time.Second
264255
}
@@ -325,6 +316,14 @@ func New(options *Options) *API {
325316
Experiments: experiments,
326317
healthCheckGroup: &singleflight.Group[string, *healthcheck.Report]{},
327318
}
319+
if options.HealthcheckFunc == nil {
320+
options.HealthcheckFunc = func(ctx context.Context) (*healthcheck.Report, error) {
321+
return healthcheck.Run(ctx, &healthcheck.ReportOptions{
322+
AccessURL: options.AccessURL,
323+
DERPMap: api.DERPMap().Clone(),
324+
})
325+
}
326+
}
328327
if options.UpdateCheckOptions != nil {
329328
api.updateChecker = updatecheck.New(
330329
options.Database,
@@ -814,6 +813,7 @@ type API struct {
814813
TailnetCoordinator atomic.Pointer[tailnet.Coordinator]
815814
QuotaCommitter atomic.Pointer[proto.QuotaCommitter]
816815
TemplateScheduleStore *atomic.Pointer[schedule.TemplateScheduleStore]
816+
DERPMapper atomic.Pointer[func(derpMap *tailcfg.DERPMap) *tailcfg.DERPMap]
817817

818818
HTTPAuth *HTTPAuthorizer
819819

@@ -954,6 +954,15 @@ func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context, debounce ti
954954
return proto.NewDRPCProvisionerDaemonClient(clientSession), nil
955955
}
956956

957+
func (api *API) DERPMap() *tailcfg.DERPMap {
958+
fn := api.DERPMapper.Load()
959+
if fn != nil {
960+
return (*fn)(api.Options.DERPMap)
961+
}
962+
963+
return api.Options.DERPMap
964+
}
965+
957966
// nolint:revive
958967
func initExperiments(log slog.Logger, raw []string) codersdk.Experiments {
959968
exps := make([]codersdk.Experiment, 0, len(raw))

‎coderd/prometheusmetrics/prometheusmetrics.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ func Workspaces(ctx context.Context, registerer prometheus.Registerer, db databa
136136
}
137137

138138
// Agents tracks the total number of workspaces with labels on status.
139-
func Agents(ctx context.Context, logger slog.Logger, registerer prometheus.Registerer, db database.Store, coordinator *atomic.Pointer[tailnet.Coordinator], derpMap *tailcfg.DERPMap, agentInactiveDisconnectTimeout, duration time.Duration) (func(), error) {
139+
func Agents(ctx context.Context, logger slog.Logger, registerer prometheus.Registerer, db database.Store, coordinator *atomic.Pointer[tailnet.Coordinator], derpMapFn func() *tailcfg.DERPMap, agentInactiveDisconnectTimeout, duration time.Duration) (func(), error) {
140140
if duration == 0 {
141141
duration = 1 * time.Minute
142142
}
@@ -215,6 +215,7 @@ func Agents(ctx context.Context, logger slog.Logger, registerer prometheus.Regis
215215

216216
logger.Debug(ctx, "Agent metrics collection is starting")
217217
timer := prometheus.NewTimer(metricsCollectorAgents)
218+
derpMap := derpMapFn()
218219

219220
workspaceRows, err := db.GetWorkspaces(ctx, database.GetWorkspacesParams{
220221
AgentInactiveDisconnectTimeoutSeconds: int64(agentInactiveDisconnectTimeout.Seconds()),

‎coderd/prometheusmetrics/prometheusmetrics_test.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/prometheus/client_golang/prometheus"
1616
"github.com/stretchr/testify/assert"
1717
"github.com/stretchr/testify/require"
18+
"tailscale.com/tailcfg"
1819

1920
"cdr.dev/slog"
2021
"cdr.dev/slog/sloggers/slogtest"
@@ -303,6 +304,9 @@ func TestAgents(t *testing.T) {
303304
coordinatorPtr := atomic.Pointer[tailnet.Coordinator]{}
304305
coordinatorPtr.Store(&coordinator)
305306
derpMap := tailnettest.RunDERPAndSTUN(t)
307+
derpMapFn := func() *tailcfg.DERPMap {
308+
return derpMap
309+
}
306310
agentInactiveDisconnectTimeout := 1 * time.Hour // don't need to focus on this value in tests
307311
registry := prometheus.NewRegistry()
308312

@@ -312,7 +316,7 @@ func TestAgents(t *testing.T) {
312316
// when
313317
closeFunc, err := prometheusmetrics.Agents(ctx, slogtest.Make(t, &slogtest.Options{
314318
IgnoreErrors: true,
315-
}), registry, db, &coordinatorPtr, derpMap, agentInactiveDisconnectTimeout, time.Millisecond)
319+
}), registry, db, &coordinatorPtr, derpMapFn, agentInactiveDisconnectTimeout, time.Millisecond)
316320
require.NoError(t, err)
317321
t.Cleanup(closeFunc)
318322

‎coderd/provisionerjobs.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,7 @@ func (api *API) provisionerJobResources(rw http.ResponseWriter, r *http.Request,
261261
}
262262

263263
apiAgent, err := convertWorkspaceAgent(
264-
api.DERPMap, *api.TailnetCoordinator.Load(), agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout,
264+
api.DERPMap(), *api.TailnetCoordinator.Load(), agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout,
265265
api.DeploymentValues.AgentFallbackTroubleshootingURL.String(),
266266
)
267267
if err != nil {

‎coderd/workspaceagents.go

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ func (api *API) workspaceAgent(rw http.ResponseWriter, r *http.Request) {
6363
return
6464
}
6565
apiAgent, err := convertWorkspaceAgent(
66-
api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout,
66+
api.DERPMap(), *api.TailnetCoordinator.Load(), workspaceAgent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout,
6767
api.DeploymentValues.AgentFallbackTroubleshootingURL.String(),
6868
)
6969
if err != nil {
@@ -88,7 +88,7 @@ func (api *API) workspaceAgentManifest(rw http.ResponseWriter, r *http.Request)
8888
ctx := r.Context()
8989
workspaceAgent := httpmw.WorkspaceAgent(r)
9090
apiAgent, err := convertWorkspaceAgent(
91-
api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout,
91+
api.DERPMap(), *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout,
9292
api.DeploymentValues.AgentFallbackTroubleshootingURL.String(),
9393
)
9494
if err != nil {
@@ -162,7 +162,7 @@ func (api *API) workspaceAgentManifest(rw http.ResponseWriter, r *http.Request)
162162

163163
httpapi.Write(ctx, rw, http.StatusOK, agentsdk.Manifest{
164164
Apps: convertApps(dbApps),
165-
DERPMap: api.DERPMap,
165+
DERPMap: api.DERPMap(),
166166
GitAuthConfigs: len(api.GitAuthConfigs),
167167
EnvironmentVariables: apiAgent.EnvironmentVariables,
168168
StartupScript: apiAgent.StartupScript,
@@ -190,7 +190,7 @@ func (api *API) postWorkspaceAgentStartup(rw http.ResponseWriter, r *http.Reques
190190
ctx := r.Context()
191191
workspaceAgent := httpmw.WorkspaceAgent(r)
192192
apiAgent, err := convertWorkspaceAgent(
193-
api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout,
193+
api.DERPMap(), *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout,
194194
api.DeploymentValues.AgentFallbackTroubleshootingURL.String(),
195195
)
196196
if err != nil {
@@ -567,7 +567,7 @@ func (api *API) workspaceAgentListeningPorts(rw http.ResponseWriter, r *http.Req
567567
workspaceAgent := httpmw.WorkspaceAgentParam(r)
568568

569569
apiAgent, err := convertWorkspaceAgent(
570-
api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout,
570+
api.DERPMap(), *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout,
571571
api.DeploymentValues.AgentFallbackTroubleshootingURL.String(),
572572
)
573573
if err != nil {
@@ -663,7 +663,7 @@ func (api *API) dialWorkspaceAgentTailnet(agentID uuid.UUID) (*codersdk.Workspac
663663
clientConn, serverConn := net.Pipe()
664664
conn, err := tailnet.NewConn(&tailnet.Options{
665665
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)},
666-
DERPMap: api.DERPMap,
666+
DERPMap: api.DERPMap(),
667667
Logger: api.Logger.Named("tailnet"),
668668
})
669669
if err != nil {
@@ -732,7 +732,7 @@ func (api *API) workspaceAgentConnection(rw http.ResponseWriter, r *http.Request
732732
ctx := r.Context()
733733

734734
httpapi.Write(ctx, rw, http.StatusOK, codersdk.WorkspaceAgentConnectionInfo{
735-
DERPMap: api.DERPMap,
735+
DERPMap: api.DERPMap(),
736736
})
737737
}
738738

‎coderd/workspacebuilds.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1126,7 +1126,7 @@ func (api *API) convertWorkspaceBuild(
11261126
for _, agent := range agents {
11271127
apps := appsByAgentID[agent.ID]
11281128
apiAgent, err := convertWorkspaceAgent(
1129-
api.DERPMap, *api.TailnetCoordinator.Load(), agent, convertApps(apps), api.AgentInactiveDisconnectTimeout,
1129+
api.DERPMap(), *api.TailnetCoordinator.Load(), agent, convertApps(apps), api.AgentInactiveDisconnectTimeout,
11301130
api.DeploymentValues.AgentFallbackTroubleshootingURL.String(),
11311131
)
11321132
if err != nil {

‎enterprise/coderd/coderd.go

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,15 @@ import (
55
"crypto/ed25519"
66
"crypto/tls"
77
"crypto/x509"
8+
"fmt"
89
"net/http"
10+
"net/url"
11+
"strconv"
912
"sync"
1013
"time"
1114

1215
"golang.org/x/xerrors"
16+
"tailscale.com/tailcfg"
1317

1418
"github.com/cenkalti/backoff/v4"
1519
"github.com/go-chi/chi/v5"
@@ -429,11 +433,82 @@ func (api *API) updateEntitlements(ctx context.Context) error {
429433
}
430434
}
431435

436+
if changed, enabled := featureChanged(codersdk.FeatureWorkspaceProxy); changed {
437+
if enabled {
438+
fn := derpMapper(api.ProxyHealth)
439+
api.AGPL.DERPMapper.Store(&fn)
440+
} else {
441+
api.AGPL.DERPMapper.Store(nil)
442+
}
443+
}
444+
432445
api.entitlements = entitlements
433446

434447
return nil
435448
}
436449

450+
func derpMapper(proxyHealth *proxyhealth.ProxyHealth) func(*tailcfg.DERPMap) *tailcfg.DERPMap {
451+
return func(derpMap *tailcfg.DERPMap) *tailcfg.DERPMap {
452+
derpMap = derpMap.Clone()
453+
454+
// Add all healthy proxies to the DERP map.
455+
// TODO: @dean proxies should be able to disable DERP and report that
456+
// when they register, and this should respect that.
457+
statusMap := proxyHealth.HealthStatus()
458+
for _, status := range statusMap {
459+
// TODO: @dean region ID should be constant and unique for each
460+
// proxy. Make the proxies report these values (from a flag like the
461+
// primary) when they register.
462+
const (
463+
regionID = -999
464+
regionCode = "proxy"
465+
)
466+
467+
if status.Status != proxyhealth.Healthy {
468+
// Only add healthy proxies to the DERP map.
469+
continue
470+
}
471+
472+
u, err := url.Parse(status.Proxy.Url)
473+
if err != nil {
474+
// Not really any need to log, the proxy should be unreachable
475+
// anyways and filtered out by the above condition.
476+
continue
477+
}
478+
port := u.Port()
479+
if port == "" {
480+
port = "80"
481+
if u.Scheme == "https" {
482+
port = "443"
483+
}
484+
}
485+
portInt, err := strconv.Atoi(port)
486+
if err != nil {
487+
// Not really any need to log, the proxy should be unreachable
488+
// anyways and filtered out by the above condition.
489+
continue
490+
}
491+
492+
derpMap.Regions[regionID] = &tailcfg.DERPRegion{
493+
EmbeddedRelay: true,
494+
RegionID: regionID,
495+
RegionCode: regionCode,
496+
RegionName: status.Proxy.Name,
497+
Nodes: []*tailcfg.DERPNode{{
498+
Name: fmt.Sprintf("%db", regionID),
499+
RegionID: regionID,
500+
HostName: u.Hostname(),
501+
DERPPort: portInt,
502+
STUNPort: -1,
503+
ForceHTTP: u.Scheme == "http",
504+
}},
505+
}
506+
}
507+
508+
return derpMap
509+
}
510+
}
511+
437512
// @Summary Get entitlements
438513
// @ID get-entitlements
439514
// @Security CoderSessionToken

0 commit comments

Comments
 (0)