coder · coadler · Sep 22, 2022 · Sep 23, 2022 · Sep 23, 2022 · Sep 23, 2022
diff --git a/agent/agent_test.go b/agent/agent_test.go
@@ -560,7 +560,7 @@ func setupAgent(t *testing.T, metadata codersdk.WorkspaceAgentMetadata, ptyTimeo
 	if metadata.DERPMap == nil {
 		metadata.DERPMap = tailnettest.RunDERPAndSTUN(t)
 	}
-	coordinator := tailnet.NewCoordinator()
+	coordinator := tailnet.NewMemoryCoordinator()
 	agentID := uuid.New()
 	statsCh := make(chan *codersdk.AgentStats)
 	closer := agent.New(agent.Options{

diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -76,7 +76,7 @@ type Options struct {
 	TracerProvider       trace.TracerProvider
 	AutoImportTemplates  []AutoImportTemplate
 
-	TailnetCoordinator *tailnet.Coordinator
+	TailnetCoordinator tailnet.Coordinator
 	DERPMap            *tailcfg.DERPMap
 
 	MetricsCacheRefreshInterval time.Duration
@@ -119,7 +119,7 @@ func New(options *Options) *API {
 		options.PrometheusRegistry = prometheus.NewRegistry()
 	}
 	if options.TailnetCoordinator == nil {
-		options.TailnetCoordinator = tailnet.NewCoordinator()
+		options.TailnetCoordinator = tailnet.NewMemoryCoordinator()
 	}
 	if options.Auditor == nil {
 		options.Auditor = audit.NewNop()
@@ -159,6 +159,7 @@ func New(options *Options) *API {
 	api.Auditor.Store(&options.Auditor)
 	api.WorkspaceQuotaEnforcer.Store(&options.WorkspaceQuotaEnforcer)
 	api.workspaceAgentCache = wsconncache.New(api.dialWorkspaceAgentTailnet, 0)
+	api.TailnetCoordinator.Store(&options.TailnetCoordinator)
 	api.derpServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger))
 	oauthConfigs := &httpmw.OAuth2Configs{
 		Github: options.GithubOAuth2Config,
@@ -531,6 +532,7 @@ type API struct {
 	Auditor                           atomic.Pointer[audit.Auditor]
 	WorkspaceClientCoordinateOverride atomic.Pointer[func(rw http.ResponseWriter) bool]
 	WorkspaceQuotaEnforcer            atomic.Pointer[workspacequota.Enforcer]
+	TailnetCoordinator                atomic.Pointer[tailnet.Coordinator]
 	HTTPAuth                          *HTTPAuthorizer
 
 	// APIHandler serves "/api/v2"

diff --git a/coderd/database/pubsub_memory.go b/coderd/database/pubsub_memory.go
@@ -47,8 +47,9 @@ func (m *memoryPubsub) Publish(event string, message []byte) error {
 		return nil
 	}
 	for _, listener := range listeners {
-		listener(context.Background(), message)
+		go listener(context.Background(), message)
 	}
+
 	return nil
 }
 

diff --git a/coderd/provisionerjobs.go b/coderd/provisionerjobs.go
@@ -270,7 +270,7 @@ func (api *API) provisionerJobResources(rw http.ResponseWriter, r *http.Request,
 				}
 			}
 
-			apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout)
+			apiAgent, err := convertWorkspaceAgent(api.DERPMap, *api.TailnetCoordinator.Load(), agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout)
 			if err != nil {
 				httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
 					Message: "Internal error reading job agent.",

diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go
@@ -49,7 +49,7 @@ func (api *API) workspaceAgent(rw http.ResponseWriter, r *http.Request) {
 		})
 		return
 	}
-	apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, workspaceAgent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout)
+	apiAgent, err := convertWorkspaceAgent(api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout)
 	if err != nil {
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
 			Message: "Internal error reading workspace agent.",
@@ -78,7 +78,7 @@ func (api *API) workspaceAgentApps(rw http.ResponseWriter, r *http.Request) {
 func (api *API) workspaceAgentMetadata(rw http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 	workspaceAgent := httpmw.WorkspaceAgent(r)
-	apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout)
+	apiAgent, err := convertWorkspaceAgent(api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout)
 	if err != nil {
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
 			Message: "Internal error reading workspace agent.",
@@ -98,7 +98,7 @@ func (api *API) workspaceAgentMetadata(rw http.ResponseWriter, r *http.Request)
 func (api *API) postWorkspaceAgentVersion(rw http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 	workspaceAgent := httpmw.WorkspaceAgent(r)
-	apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout)
+	apiAgent, err := convertWorkspaceAgent(api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout)
 	if err != nil {
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
 			Message: "Internal error reading workspace agent.",
@@ -152,7 +152,7 @@ func (api *API) workspaceAgentPTY(rw http.ResponseWriter, r *http.Request) {
 		httpapi.ResourceNotFound(rw)
 		return
 	}
-	apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout)
+	apiAgent, err := convertWorkspaceAgent(api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout)
 	if err != nil {
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
 			Message: "Internal error reading workspace agent.",
@@ -229,7 +229,7 @@ func (api *API) workspaceAgentListeningPorts(rw http.ResponseWriter, r *http.Req
 		return
 	}
 
-	apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout)
+	apiAgent, err := convertWorkspaceAgent(api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout)
 	if err != nil {
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
 			Message: "Internal error reading workspace agent.",
@@ -376,7 +376,7 @@ func (api *API) dialWorkspaceAgentTailnet(r *http.Request, agentID uuid.UUID) (*
 	})
 	conn.SetNodeCallback(sendNodes)
 	go func() {
-		err := api.TailnetCoordinator.ServeClient(serverConn, uuid.New(), agentID)
+		err := (*api.TailnetCoordinator.Load()).ServeClient(serverConn, uuid.New(), agentID)
 		if err != nil {
 			_ = conn.Close()
 		}
@@ -514,7 +514,7 @@ func (api *API) workspaceAgentCoordinate(rw http.ResponseWriter, r *http.Request
 	closeChan := make(chan struct{})
 	go func() {
 		defer close(closeChan)
-		err := api.TailnetCoordinator.ServeAgent(wsNetConn, workspaceAgent.ID)
+		err := (*api.TailnetCoordinator.Load()).ServeAgent(wsNetConn, workspaceAgent.ID)
 		if err != nil {
 			_ = conn.Close(websocket.StatusInternalError, err.Error())
 			return
@@ -583,7 +583,7 @@ func (api *API) workspaceAgentClientCoordinate(rw http.ResponseWriter, r *http.R
 	go httpapi.Heartbeat(ctx, conn)
 
 	defer conn.Close(websocket.StatusNormalClosure, "")
-	err = api.TailnetCoordinator.ServeClient(websocket.NetConn(ctx, conn, websocket.MessageBinary), uuid.New(), workspaceAgent.ID)
+	err = (*api.TailnetCoordinator.Load()).ServeClient(websocket.NetConn(ctx, conn, websocket.MessageBinary), uuid.New(), workspaceAgent.ID)
 	if err != nil {
 		_ = conn.Close(websocket.StatusInternalError, err.Error())
 		return
@@ -610,7 +610,7 @@ func convertApps(dbApps []database.WorkspaceApp) []codersdk.WorkspaceApp {
 	return apps
 }
 
-func convertWorkspaceAgent(derpMap *tailcfg.DERPMap, coordinator *tailnet.Coordinator, dbAgent database.WorkspaceAgent, apps []codersdk.WorkspaceApp, agentInactiveDisconnectTimeout time.Duration) (codersdk.WorkspaceAgent, error) {
+func convertWorkspaceAgent(derpMap *tailcfg.DERPMap, coordinator tailnet.Coordinator, dbAgent database.WorkspaceAgent, apps []codersdk.WorkspaceApp, agentInactiveDisconnectTimeout time.Duration) (codersdk.WorkspaceAgent, error) {
 	var envs map[string]string
 	if dbAgent.EnvironmentVariables.Valid {
 		err := json.Unmarshal(dbAgent.EnvironmentVariables.RawMessage, &envs)

diff --git a/coderd/workspacebuilds.go b/coderd/workspacebuilds.go
@@ -844,7 +844,7 @@ func (api *API) convertWorkspaceBuild(
 		apiAgents := make([]codersdk.WorkspaceAgent, 0)
 		for _, agent := range agents {
 			apps := appsByAgentID[agent.ID]
-			apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, agent, convertApps(apps), api.AgentInactiveDisconnectTimeout)
+			apiAgent, err := convertWorkspaceAgent(api.DERPMap, *api.TailnetCoordinator.Load(), agent, convertApps(apps), api.AgentInactiveDisconnectTimeout)
 			if err != nil {
 				return codersdk.WorkspaceBuild{}, xerrors.Errorf("converting workspace agent: %w", err)
 			}

diff --git a/coderd/wsconncache/wsconncache_test.go b/coderd/wsconncache/wsconncache_test.go
@@ -143,7 +143,7 @@ func TestCache(t *testing.T) {
 func setupAgent(t *testing.T, metadata codersdk.WorkspaceAgentMetadata, ptyTimeout time.Duration) *codersdk.AgentConn {
 	metadata.DERPMap = tailnettest.RunDERPAndSTUN(t)
 
-	coordinator := tailnet.NewCoordinator()
+	coordinator := tailnet.NewMemoryCoordinator()
 	agentID := uuid.New()
 	closer := agent.New(agent.Options{
 		FetchMetadata: func(ctx context.Context) (codersdk.WorkspaceAgentMetadata, error) {

diff --git a/codersdk/features.go b/codersdk/features.go
@@ -15,12 +15,13 @@ const (
 )
 
 const (
-	FeatureUserLimit      = "user_limit"
-	FeatureAuditLog       = "audit_log"
-	FeatureBrowserOnly    = "browser_only"
-	FeatureSCIM           = "scim"
-	FeatureWorkspaceQuota = "workspace_quota"
-	FeatureTemplateRBAC   = "template_rbac"
+	FeatureUserLimit        = "user_limit"
+	FeatureAuditLog         = "audit_log"
+	FeatureBrowserOnly      = "browser_only"
+	FeatureSCIM             = "scim"
+	FeatureWorkspaceQuota   = "workspace_quota"
+	FeatureTemplateRBAC     = "template_rbac"
+	FeatureHighAvailability = "high_availability"
 )
 
 var FeatureNames = []string{
@@ -30,6 +31,7 @@ var FeatureNames = []string{
 	FeatureSCIM,
 	FeatureWorkspaceQuota,
 	FeatureTemplateRBAC,
+	FeatureHighAvailability,
 }
 
 type Feature struct {

diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go
@@ -21,7 +21,6 @@ import (
 	"tailscale.com/tailcfg"
 
 	"cdr.dev/slog"
-
 	"github.com/coder/coder/tailnet"
 	"github.com/coder/retry"
 )

diff --git a/enterprise/cli/features_test.go b/enterprise/cli/features_test.go
@@ -71,6 +71,8 @@ func TestFeaturesList(t *testing.T) {
 			entitlements.Features[codersdk.FeatureTemplateRBAC].Entitlement)
 		assert.Equal(t, codersdk.EntitlementNotEntitled,
 			entitlements.Features[codersdk.FeatureSCIM].Entitlement)
+		assert.Equal(t, codersdk.EntitlementNotEntitled,
+			entitlements.Features[codersdk.FeatureHighAvailability].Entitlement)
 		assert.False(t, entitlements.HasLicense)
 		assert.False(t, entitlements.Experimental)
 	})

diff --git a/enterprise/coderd/authorize_test.go b/enterprise/coderd/authorize_test.go
@@ -28,7 +28,7 @@ func TestCheckACLPermissions(t *testing.T) {
 	// Create adminClient, member, and org adminClient
 	adminUser := coderdtest.CreateFirstUser(t, adminClient)
 	_ = coderdenttest.AddLicense(t, adminClient, coderdenttest.LicenseOptions{
-		TemplateRBACEnabled: true,
+		TemplateRBAC: true,
 	})
 
 	memberClient := coderdtest.CreateAnotherUser(t, adminClient, adminUser.OrganizationID)

diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go
@@ -23,6 +23,8 @@ import (
 	"github.com/coder/coder/enterprise/audit"
 	"github.com/coder/coder/enterprise/audit/backends"
 	"github.com/coder/coder/enterprise/coderd/license"
+	"github.com/coder/coder/enterprise/tailnet"
+	agpltailnet "github.com/coder/coder/tailnet"
 )
 
 // New constructs an Enterprise coderd API instance.
@@ -203,6 +205,30 @@ func (api *API) updateEntitlements(ctx context.Context) error {
 		api.AGPL.WorkspaceQuotaEnforcer.Store(&enforcer)
 	}
 
+	if changed, enabled := featureChanged(codersdk.FeatureHighAvailability); changed {
+		coordinator := agpltailnet.NewMemoryCoordinator()
+		if enabled {
+			haCoordinator, err := tailnet.NewHACoordinator(api.Logger, api.Pubsub)
+			if err != nil {
+				api.Logger.Error(ctx, "unable to setup HA tailnet coordinator", slog.Error(err))
+				// If we try to setup the HA coordinator and it fails, nothing
+				// is actually changing.
+				changed = false
+			} else {
+				coordinator = haCoordinator
+			}
+		}
+
+		// Recheck changed in case the HA coordinator failed to set up.
+		if changed {
+			oldCoordinator := *api.AGPL.TailnetCoordinator.Swap(&coordinator)
+			err := oldCoordinator.Close()
+			if err != nil {
+				api.Logger.Error(ctx, "close old tailnet coordinator", slog.Error(err))
+			}
+		}
+	}
+
 	api.entitlements = entitlements
 
 	return nil

diff --git a/enterprise/coderd/coderd_test.go b/enterprise/coderd/coderd_test.go
@@ -41,9 +41,9 @@ func TestEntitlements(t *testing.T) {
 		})
 		_ = coderdtest.CreateFirstUser(t, client)
 		coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			UserLimit:           100,
-			AuditLog:            true,
-			TemplateRBACEnabled: true,
+			UserLimit:    100,
+			AuditLog:     true,
+			TemplateRBAC: true,
 		})
 		res, err := client.Entitlements(context.Background())
 		require.NoError(t, err)

diff --git a/enterprise/coderd/coderdenttest/coderdenttest.go b/enterprise/coderd/coderdenttest/coderdenttest.go
@@ -87,18 +87,19 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c
 }
 
 type LicenseOptions struct {
-	AccountType         string
-	AccountID           string
-	Trial               bool
-	AllFeatures         bool
-	GraceAt             time.Time
-	ExpiresAt           time.Time
-	UserLimit           int64
-	AuditLog            bool
-	BrowserOnly         bool
-	SCIM                bool
-	WorkspaceQuota      bool
-	TemplateRBACEnabled bool
+	AccountType      string
+	AccountID        string
+	Trial            bool
+	AllFeatures      bool
+	GraceAt          time.Time
+	ExpiresAt        time.Time
+	UserLimit        int64
+	AuditLog         bool
+	BrowserOnly      bool
+	SCIM             bool
+	WorkspaceQuota   bool
+	TemplateRBAC     bool
+	HighAvailability bool
 }
 
 // AddLicense generates a new license with the options provided and inserts it.
@@ -134,9 +135,13 @@ func GenerateLicense(t *testing.T, options LicenseOptions) string {
 	if options.WorkspaceQuota {
 		workspaceQuota = 1
 	}
+	highAvailability := int64(0)
+	if options.HighAvailability {
+		highAvailability = 1
+	}
 
 	rbacEnabled := int64(0)
-	if options.TemplateRBACEnabled {
+	if options.TemplateRBAC {
 		rbacEnabled = 1
 	}
 
@@ -154,12 +159,13 @@ func GenerateLicense(t *testing.T, options LicenseOptions) string {
 		Version:        license.CurrentVersion,
 		AllFeatures:    options.AllFeatures,
 		Features: license.Features{
-			UserLimit:      options.UserLimit,
-			AuditLog:       auditLog,
-			BrowserOnly:    browserOnly,
-			SCIM:           scim,
-			WorkspaceQuota: workspaceQuota,
-			TemplateRBAC:   rbacEnabled,
+			UserLimit:        options.UserLimit,
+			AuditLog:         auditLog,
+			BrowserOnly:      browserOnly,
+			SCIM:             scim,
+			WorkspaceQuota:   workspaceQuota,
+			HighAvailability: highAvailability,
+			TemplateRBAC:     rbacEnabled,
 		},
 	}
 	tok := jwt.NewWithClaims(jwt.SigningMethodEdDSA, c)

diff --git a/enterprise/coderd/coderdenttest/coderdenttest_test.go b/enterprise/coderd/coderdenttest/coderdenttest_test.go
@@ -33,7 +33,7 @@ func TestAuthorizeAllEndpoints(t *testing.T) {
 	ctx, _ := testutil.Context(t)
 	admin := coderdtest.CreateFirstUser(t, client)
 	license := coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-		TemplateRBACEnabled: true,
+		TemplateRBAC: true,
 	})
 	group, err := client.CreateGroup(ctx, admin.OrganizationID, codersdk.CreateGroupRequest{
 		Name: "testgroup",