chore: add derpserver to proxy, add proxies to derpmap

coder · deansheather · Jul 26, 2023 · Apr 27, 2023 · May 1, 2023 · May 2, 2023
commit eae4c3a5965cabe673bb9f43e6b231cf6865fee3
diff --git a/cli/server.go b/cli/server.go
@@ -759,7 +759,7 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 
 			if cfg.Prometheus.Enable {
 				// Agent metrics require reference to the tailnet coordinator, so must be initiated after Coder API.
-				closeAgentsFunc, err := prometheusmetrics.Agents(ctx, logger, options.PrometheusRegistry, coderAPI.Database, &coderAPI.TailnetCoordinator, options.DERPMap, coderAPI.Options.AgentInactiveDisconnectTimeout, 0)
+				closeAgentsFunc, err := prometheusmetrics.Agents(ctx, logger, options.PrometheusRegistry, coderAPI.Database, &coderAPI.TailnetCoordinator, coderAPI.DERPMap, coderAPI.Options.AgentInactiveDisconnectTimeout, 0)
 				if err != nil {
 					return xerrors.Errorf("register agents prometheus metric: %w", err)
 				}

diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -35,13 +35,11 @@ import (
 	"tailscale.com/types/key"
 	"tailscale.com/util/singleflight"
 
-	"cdr.dev/slog"
-
-	"github.com/coder/coder/buildinfo"
-	"github.com/coder/coder/codersdk/agentsdk"
-
 	// Used for swagger docs.
 	_ "github.com/coder/coder/coderd/apidoc"
+
+	"cdr.dev/slog"
+	"github.com/coder/coder/buildinfo"
 	"github.com/coder/coder/coderd/audit"
 	"github.com/coder/coder/coderd/awsidentity"
 	"github.com/coder/coder/coderd/database"
@@ -63,6 +61,7 @@ import (
 	"github.com/coder/coder/coderd/workspaceapps"
 	"github.com/coder/coder/coderd/wsconncache"
 	"github.com/coder/coder/codersdk"
+	"github.com/coder/coder/codersdk/agentsdk"
 	"github.com/coder/coder/provisionerd/proto"
 	"github.com/coder/coder/provisionersdk"
 	"github.com/coder/coder/site"
@@ -251,14 +250,6 @@ func New(options *Options) *API {
 		v := schedule.NewAGPLTemplateScheduleStore()
 		options.TemplateScheduleStore.Store(&v)
 	}
-	if options.HealthcheckFunc == nil {
-		options.HealthcheckFunc = func(ctx context.Context) (*healthcheck.Report, error) {
-			return healthcheck.Run(ctx, &healthcheck.ReportOptions{
-				AccessURL: options.AccessURL,
-				DERPMap:   options.DERPMap.Clone(),
-			})
-		}
-	}
 	if options.HealthcheckTimeout == 0 {
 		options.HealthcheckTimeout = 30 * time.Second
 	}
@@ -325,6 +316,14 @@ func New(options *Options) *API {
 		Experiments:           experiments,
 		healthCheckGroup:      &singleflight.Group[string, *healthcheck.Report]{},
 	}
+	if options.HealthcheckFunc == nil {
+		options.HealthcheckFunc = func(ctx context.Context) (*healthcheck.Report, error) {
+			return healthcheck.Run(ctx, &healthcheck.ReportOptions{
+				AccessURL: options.AccessURL,
+				DERPMap:   api.DERPMap().Clone(),
+			})
+		}
+	}
 	if options.UpdateCheckOptions != nil {
 		api.updateChecker = updatecheck.New(
 			options.Database,
@@ -814,6 +813,7 @@ type API struct {
 	TailnetCoordinator                atomic.Pointer[tailnet.Coordinator]
 	QuotaCommitter                    atomic.Pointer[proto.QuotaCommitter]
 	TemplateScheduleStore             *atomic.Pointer[schedule.TemplateScheduleStore]
+	DERPMapper                        atomic.Pointer[func(derpMap *tailcfg.DERPMap) *tailcfg.DERPMap]
 
 	HTTPAuth *HTTPAuthorizer
 
@@ -954,6 +954,15 @@ func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context, debounce ti
 	return proto.NewDRPCProvisionerDaemonClient(clientSession), nil
 }
 
+func (api *API) DERPMap() *tailcfg.DERPMap {
+	fn := api.DERPMapper.Load()
+	if fn != nil {
+		return (*fn)(api.Options.DERPMap)
+	}
+
+	return api.Options.DERPMap
+}
+
 // nolint:revive
 func initExperiments(log slog.Logger, raw []string) codersdk.Experiments {
 	exps := make([]codersdk.Experiment, 0, len(raw))

diff --git a/coderd/prometheusmetrics/prometheusmetrics.go b/coderd/prometheusmetrics/prometheusmetrics.go
@@ -136,7 +136,7 @@ func Workspaces(ctx context.Context, registerer prometheus.Registerer, db databa
 }
 
 // Agents tracks the total number of workspaces with labels on status.
-func Agents(ctx context.Context, logger slog.Logger, registerer prometheus.Registerer, db database.Store, coordinator *atomic.Pointer[tailnet.Coordinator], derpMap *tailcfg.DERPMap, agentInactiveDisconnectTimeout, duration time.Duration) (func(), error) {
+func Agents(ctx context.Context, logger slog.Logger, registerer prometheus.Registerer, db database.Store, coordinator *atomic.Pointer[tailnet.Coordinator], derpMapFn func() *tailcfg.DERPMap, agentInactiveDisconnectTimeout, duration time.Duration) (func(), error) {
 	if duration == 0 {
 		duration = 1 * time.Minute
 	}
@@ -215,6 +215,7 @@ func Agents(ctx context.Context, logger slog.Logger, registerer prometheus.Regis
 
 			logger.Debug(ctx, "Agent metrics collection is starting")
 			timer := prometheus.NewTimer(metricsCollectorAgents)
+			derpMap := derpMapFn()
 
 			workspaceRows, err := db.GetWorkspaces(ctx, database.GetWorkspacesParams{
 				AgentInactiveDisconnectTimeoutSeconds: int64(agentInactiveDisconnectTimeout.Seconds()),

diff --git a/coderd/prometheusmetrics/prometheusmetrics_test.go b/coderd/prometheusmetrics/prometheusmetrics_test.go
@@ -15,6 +15,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"tailscale.com/tailcfg"
 
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/slogtest"
@@ -303,6 +304,9 @@ func TestAgents(t *testing.T) {
 	coordinatorPtr := atomic.Pointer[tailnet.Coordinator]{}
 	coordinatorPtr.Store(&coordinator)
 	derpMap := tailnettest.RunDERPAndSTUN(t)
+	derpMapFn := func() *tailcfg.DERPMap {
+		return derpMap
+	}
 	agentInactiveDisconnectTimeout := 1 * time.Hour // don't need to focus on this value in tests
 	registry := prometheus.NewRegistry()
 
@@ -312,7 +316,7 @@ func TestAgents(t *testing.T) {
 	// when
 	closeFunc, err := prometheusmetrics.Agents(ctx, slogtest.Make(t, &slogtest.Options{
 		IgnoreErrors: true,
-	}), registry, db, &coordinatorPtr, derpMap, agentInactiveDisconnectTimeout, time.Millisecond)
+	}), registry, db, &coordinatorPtr, derpMapFn, agentInactiveDisconnectTimeout, time.Millisecond)
 	require.NoError(t, err)
 	t.Cleanup(closeFunc)
 

diff --git a/coderd/provisionerjobs.go b/coderd/provisionerjobs.go
@@ -261,7 +261,7 @@ func (api *API) provisionerJobResources(rw http.ResponseWriter, r *http.Request,
 			}
 
 			apiAgent, err := convertWorkspaceAgent(
-				api.DERPMap, *api.TailnetCoordinator.Load(), agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout,
+				api.DERPMap(), *api.TailnetCoordinator.Load(), agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout,
 				api.DeploymentValues.AgentFallbackTroubleshootingURL.String(),
 			)
 			if err != nil {

diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go
@@ -63,7 +63,7 @@ func (api *API) workspaceAgent(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 	apiAgent, err := convertWorkspaceAgent(
-		api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout,
+		api.DERPMap(), *api.TailnetCoordinator.Load(), workspaceAgent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout,
 		api.DeploymentValues.AgentFallbackTroubleshootingURL.String(),
 	)
 	if err != nil {
@@ -88,7 +88,7 @@ func (api *API) workspaceAgentManifest(rw http.ResponseWriter, r *http.Request)
 	ctx := r.Context()
 	workspaceAgent := httpmw.WorkspaceAgent(r)
 	apiAgent, err := convertWorkspaceAgent(
-		api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout,
+		api.DERPMap(), *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout,
 		api.DeploymentValues.AgentFallbackTroubleshootingURL.String(),
 	)
 	if err != nil {
@@ -162,7 +162,7 @@ func (api *API) workspaceAgentManifest(rw http.ResponseWriter, r *http.Request)
 
 	httpapi.Write(ctx, rw, http.StatusOK, agentsdk.Manifest{
 		Apps:                  convertApps(dbApps),
-		DERPMap:               api.DERPMap,
+		DERPMap:               api.DERPMap(),
 		GitAuthConfigs:        len(api.GitAuthConfigs),
 		EnvironmentVariables:  apiAgent.EnvironmentVariables,
 		StartupScript:         apiAgent.StartupScript,
@@ -190,7 +190,7 @@ func (api *API) postWorkspaceAgentStartup(rw http.ResponseWriter, r *http.Reques
 	ctx := r.Context()
 	workspaceAgent := httpmw.WorkspaceAgent(r)
 	apiAgent, err := convertWorkspaceAgent(
-		api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout,
+		api.DERPMap(), *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout,
 		api.DeploymentValues.AgentFallbackTroubleshootingURL.String(),
 	)
 	if err != nil {
@@ -567,7 +567,7 @@ func (api *API) workspaceAgentListeningPorts(rw http.ResponseWriter, r *http.Req
 	workspaceAgent := httpmw.WorkspaceAgentParam(r)
 
 	apiAgent, err := convertWorkspaceAgent(
-		api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout,
+		api.DERPMap(), *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout,
 		api.DeploymentValues.AgentFallbackTroubleshootingURL.String(),
 	)
 	if err != nil {
@@ -663,7 +663,7 @@ func (api *API) dialWorkspaceAgentTailnet(agentID uuid.UUID) (*codersdk.Workspac
 	clientConn, serverConn := net.Pipe()
 	conn, err := tailnet.NewConn(&tailnet.Options{
 		Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)},
-		DERPMap:   api.DERPMap,
+		DERPMap:   api.DERPMap(),
 		Logger:    api.Logger.Named("tailnet"),
 	})
 	if err != nil {
@@ -732,7 +732,7 @@ func (api *API) workspaceAgentConnection(rw http.ResponseWriter, r *http.Request
 	ctx := r.Context()
 
 	httpapi.Write(ctx, rw, http.StatusOK, codersdk.WorkspaceAgentConnectionInfo{
-		DERPMap: api.DERPMap,
+		DERPMap: api.DERPMap(),
 	})
 }
 

diff --git a/coderd/workspacebuilds.go b/coderd/workspacebuilds.go
@@ -1126,7 +1126,7 @@ func (api *API) convertWorkspaceBuild(
 		for _, agent := range agents {
 			apps := appsByAgentID[agent.ID]
 			apiAgent, err := convertWorkspaceAgent(
-				api.DERPMap, *api.TailnetCoordinator.Load(), agent, convertApps(apps), api.AgentInactiveDisconnectTimeout,
+				api.DERPMap(), *api.TailnetCoordinator.Load(), agent, convertApps(apps), api.AgentInactiveDisconnectTimeout,
 				api.DeploymentValues.AgentFallbackTroubleshootingURL.String(),
 			)
 			if err != nil {

diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go
@@ -5,11 +5,15 @@ import (
 	"crypto/ed25519"
 	"crypto/tls"
 	"crypto/x509"
+	"fmt"
 	"net/http"
+	"net/url"
+	"strconv"
 	"sync"
 	"time"
 
 	"golang.org/x/xerrors"
+	"tailscale.com/tailcfg"
 
 	"github.com/cenkalti/backoff/v4"
 	"github.com/go-chi/chi/v5"
@@ -429,11 +433,82 @@ func (api *API) updateEntitlements(ctx context.Context) error {
 		}
 	}
 
+	if changed, enabled := featureChanged(codersdk.FeatureWorkspaceProxy); changed {
+		if enabled {
+			fn := derpMapper(api.ProxyHealth)
+			api.AGPL.DERPMapper.Store(&fn)
+		} else {
+			api.AGPL.DERPMapper.Store(nil)
+		}
+	}
+
 	api.entitlements = entitlements
 
 	return nil
 }
 
+func derpMapper(proxyHealth *proxyhealth.ProxyHealth) func(*tailcfg.DERPMap) *tailcfg.DERPMap {
+	return func(derpMap *tailcfg.DERPMap) *tailcfg.DERPMap {
+		derpMap = derpMap.Clone()
+
+		// Add all healthy proxies to the DERP map.
+		// TODO: @dean proxies should be able to disable DERP and report that
+		// when they register, and this should respect that.
+		statusMap := proxyHealth.HealthStatus()
+		for _, status := range statusMap {
+			// TODO: @dean region ID should be constant and unique for each
+			// proxy. Make the proxies report these values (from a flag like the
+			// primary) when they register.
+			const (
+				regionID   = -999
+				regionCode = "proxy"
+			)
+
+			if status.Status != proxyhealth.Healthy {
+				// Only add healthy proxies to the DERP map.
+				continue
+			}
+
+			u, err := url.Parse(status.Proxy.Url)
+			if err != nil {
+				// Not really any need to log, the proxy should be unreachable
+				// anyways and filtered out by the above condition.
+				continue
+			}
+			port := u.Port()
+			if port == "" {
+				port = "80"
+				if u.Scheme == "https" {
+					port = "443"
+				}
+			}
+			portInt, err := strconv.Atoi(port)
+			if err != nil {
+				// Not really any need to log, the proxy should be unreachable
+				// anyways and filtered out by the above condition.
+				continue
+			}
+
+			derpMap.Regions[regionID] = &tailcfg.DERPRegion{
+				EmbeddedRelay: true,
+				RegionID:      regionID,
+				RegionCode:    regionCode,
+				RegionName:    status.Proxy.Name,
+				Nodes: []*tailcfg.DERPNode{{
+					Name:      fmt.Sprintf("%db", regionID),
+					RegionID:  regionID,
+					HostName:  u.Hostname(),
+					DERPPort:  portInt,
+					STUNPort:  -1,
+					ForceHTTP: u.Scheme == "http",
+				}},
+			}
+		}
+
+		return derpMap
+	}
+}
+
 // @Summary Get entitlements
 // @ID get-entitlements
 // @Security CoderSessionToken