From 35b2fed6860da12707266979f14286f73fe1468a Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Thu, 22 Sep 2022 17:40:59 -0500 Subject: [PATCH 01/79] feat: HA tailnet coordinator --- agent/agent_test.go | 2 +- coderd/coderd.go | 4 +- coderd/database/pubsub_memory.go | 3 +- coderd/workspaceagents.go | 2 +- coderd/wsconncache/wsconncache_test.go | 2 +- codersdk/workspaceagents.go | 1 - enterprise/tailnet/coordinator.go | 426 +++++++++++++++++++++++++ enterprise/tailnet/coordinator_test.go | 267 ++++++++++++++++ tailnet/coordinator.go | 203 +++++++----- tailnet/coordinator_test.go | 10 +- 10 files changed, 834 insertions(+), 86 deletions(-) create mode 100644 enterprise/tailnet/coordinator.go create mode 100644 enterprise/tailnet/coordinator_test.go diff --git a/agent/agent_test.go b/agent/agent_test.go index afed644f78e5e..d6ff21cdcd33d 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -572,7 +572,7 @@ func setupAgent(t *testing.T, metadata agent.Metadata, ptyTimeout time.Duration) if metadata.DERPMap == nil { metadata.DERPMap = tailnettest.RunDERPAndSTUN(t) } - coordinator := tailnet.NewCoordinator() + coordinator := tailnet.NewMemoryCoordinator() agentID := uuid.New() statsCh := make(chan *agent.Stats) closer := agent.New(agent.Options{ diff --git a/coderd/coderd.go b/coderd/coderd.go index 25ac1afec2f36..f183e4d9b9ab7 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -74,7 +74,7 @@ type Options struct { TracerProvider trace.TracerProvider AutoImportTemplates []AutoImportTemplate - TailnetCoordinator *tailnet.Coordinator + TailnetCoordinator tailnet.Coordinator DERPMap *tailcfg.DERPMap MetricsCacheRefreshInterval time.Duration @@ -121,7 +121,7 @@ func New(options *Options) *API { options.PrometheusRegistry = prometheus.NewRegistry() } if options.TailnetCoordinator == nil { - options.TailnetCoordinator = tailnet.NewCoordinator() + options.TailnetCoordinator = tailnet.NewMemoryCoordinator() } if options.Auditor == nil { options.Auditor = audit.NewNop() diff --git a/coderd/database/pubsub_memory.go b/coderd/database/pubsub_memory.go index 148d2f57b129f..de5a940414d6c 100644 --- a/coderd/database/pubsub_memory.go +++ b/coderd/database/pubsub_memory.go @@ -47,8 +47,9 @@ func (m *memoryPubsub) Publish(event string, message []byte) error { return nil } for _, listener := range listeners { - listener(context.Background(), message) + go listener(context.Background(), message) } + return nil } diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 6167790fb8bb7..dd777913c452d 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -447,7 +447,7 @@ func convertApps(dbApps []database.WorkspaceApp) []codersdk.WorkspaceApp { return apps } -func convertWorkspaceAgent(derpMap *tailcfg.DERPMap, coordinator *tailnet.Coordinator, dbAgent database.WorkspaceAgent, apps []codersdk.WorkspaceApp, agentInactiveDisconnectTimeout time.Duration) (codersdk.WorkspaceAgent, error) { +func convertWorkspaceAgent(derpMap *tailcfg.DERPMap, coordinator tailnet.Coordinator, dbAgent database.WorkspaceAgent, apps []codersdk.WorkspaceApp, agentInactiveDisconnectTimeout time.Duration) (codersdk.WorkspaceAgent, error) { var envs map[string]string if dbAgent.EnvironmentVariables.Valid { err := json.Unmarshal(dbAgent.EnvironmentVariables.RawMessage, &envs) diff --git a/coderd/wsconncache/wsconncache_test.go b/coderd/wsconncache/wsconncache_test.go index a9ea85a2492ac..e4c7d58413110 100644 --- a/coderd/wsconncache/wsconncache_test.go +++ b/coderd/wsconncache/wsconncache_test.go @@ -142,7 +142,7 @@ func TestCache(t *testing.T) { func setupAgent(t *testing.T, metadata agent.Metadata, ptyTimeout time.Duration) *agent.Conn { metadata.DERPMap = tailnettest.RunDERPAndSTUN(t) - coordinator := tailnet.NewCoordinator() + coordinator := tailnet.NewMemoryCoordinator() agentID := uuid.New() closer := agent.New(agent.Options{ FetchMetadata: func(ctx context.Context) (agent.Metadata, error) { diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index 46d8ead8d2d6d..72e9767713c7c 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -20,7 +20,6 @@ import ( "tailscale.com/tailcfg" "cdr.dev/slog" - "github.com/coder/coder/agent" "github.com/coder/coder/tailnet" "github.com/coder/retry" diff --git a/enterprise/tailnet/coordinator.go b/enterprise/tailnet/coordinator.go new file mode 100644 index 0000000000000..8824f584d60da --- /dev/null +++ b/enterprise/tailnet/coordinator.go @@ -0,0 +1,426 @@ +package tailnet + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "sync" + "time" + + "github.com/google/uuid" + "golang.org/x/xerrors" + + "cdr.dev/slog" + + "github.com/coder/coder/coderd/database" + agpl "github.com/coder/coder/tailnet" +) + +func NewHACoordinator(logger slog.Logger, pubsub database.Pubsub) (agpl.Coordinator, error) { + coord := &haCoordinator{ + id: uuid.New(), + log: logger, + pubsub: pubsub, + close: make(chan struct{}), + nodes: map[uuid.UUID]*agpl.Node{}, + agentSockets: map[uuid.UUID]net.Conn{}, + agentToConnectionSockets: map[uuid.UUID]map[uuid.UUID]net.Conn{}, + } + + if err := coord.runPubsub(); err != nil { + return nil, xerrors.Errorf("run coordinator pubsub: %w", err) + } + + return coord, nil +} + +type haCoordinator struct { + id uuid.UUID + log slog.Logger + mutex sync.RWMutex + pubsub database.Pubsub + close chan struct{} + + // nodes maps agent and connection IDs their respective node. + nodes map[uuid.UUID]*agpl.Node + // agentSockets maps agent IDs to their open websocket. + agentSockets map[uuid.UUID]net.Conn + // agentToConnectionSockets maps agent IDs to connection IDs of conns that + // are subscribed to updates for that agent. + agentToConnectionSockets map[uuid.UUID]map[uuid.UUID]net.Conn +} + +// Node returns an in-memory node by ID. +func (c *haCoordinator) Node(id uuid.UUID) *agpl.Node { + c.mutex.RLock() + defer c.mutex.RUnlock() + node := c.nodes[id] + return node +} + +// ServeClient accepts a WebSocket connection that wants to connect to an agent +// with the specified ID. +func (c *haCoordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID) error { + c.mutex.Lock() + // When a new connection is requested, we update it with the latest + // node of the agent. This allows the connection to establish. + node, ok := c.nodes[agent] + if ok { + data, err := json.Marshal([]*agpl.Node{node}) + if err != nil { + c.mutex.Unlock() + return xerrors.Errorf("marshal node: %w", err) + } + _, err = conn.Write(data) + if err != nil { + c.mutex.Unlock() + return xerrors.Errorf("write nodes: %w", err) + } + } + + connectionSockets, ok := c.agentToConnectionSockets[agent] + if !ok { + connectionSockets = map[uuid.UUID]net.Conn{} + c.agentToConnectionSockets[agent] = connectionSockets + } + + // Insert this connection into a map so the agent can publish node updates. + connectionSockets[id] = conn + c.mutex.Unlock() + + defer func() { + c.mutex.Lock() + defer c.mutex.Unlock() + // Clean all traces of this connection from the map. + delete(c.nodes, id) + connectionSockets, ok := c.agentToConnectionSockets[agent] + if !ok { + return + } + delete(connectionSockets, id) + if len(connectionSockets) != 0 { + return + } + delete(c.agentToConnectionSockets, agent) + }() + + decoder := json.NewDecoder(conn) + // Indefinitely handle messages from the client websocket. + for { + err := c.handleNextClientMessage(id, agent, decoder) + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return xerrors.Errorf("handle next client message: %w", err) + } + } +} + +func (c *haCoordinator) handleNextClientMessage(id, agent uuid.UUID, decoder *json.Decoder) error { + var node agpl.Node + err := decoder.Decode(&node) + if err != nil { + return xerrors.Errorf("read json: %w", err) + } + + c.mutex.Lock() + defer c.mutex.Unlock() + + // Update the node of this client in our in-memory map. If an agent entirely + // shuts down and reconnects, it needs to be aware of all clients attempting + // to establish connections. + c.nodes[id] = &node + + // Write the new node from this client to the actively connected agent. + err = c.writeNodeToAgent(agent, &node) + if err != nil { + return xerrors.Errorf("write node to agent: %w", err) + } + + return nil +} + +func (c *haCoordinator) writeNodeToAgent(agent uuid.UUID, node *agpl.Node) error { + agentSocket, ok := c.agentSockets[agent] + if !ok { + // If we don't own the agent locally, send it over pubsub to a node that + // owns the agent. + err := c.publishNodeToAgent(agent, node) + if err != nil { + return xerrors.Errorf("publish node to agent") + } + return nil + } + + // Write the new node from this client to the actively + // connected agent. + data, err := json.Marshal([]*agpl.Node{node}) + if err != nil { + return xerrors.Errorf("marshal nodes: %w", err) + } + + _, err = agentSocket.Write(data) + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return xerrors.Errorf("write json: %w", err) + } + return nil +} + +// ServeAgent accepts a WebSocket connection to an agent that listens to +// incoming connections and publishes node updates. +func (c *haCoordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { + c.mutex.Lock() + sockets, ok := c.agentToConnectionSockets[id] + if ok { + // Publish all nodes that want to connect to the + // desired agent ID. + nodes := make([]*agpl.Node, 0, len(sockets)) + for targetID := range sockets { + node, ok := c.nodes[targetID] + if !ok { + continue + } + nodes = append(nodes, node) + } + data, err := json.Marshal(nodes) + if err != nil { + c.mutex.Unlock() + return xerrors.Errorf("marshal json: %w", err) + } + _, err = conn.Write(data) + if err != nil { + c.mutex.Unlock() + return xerrors.Errorf("write nodes: %w", err) + } + } + + // If an old agent socket is connected, we close it + // to avoid any leaks. This shouldn't ever occur because + // we expect one agent to be running. + oldAgentSocket, ok := c.agentSockets[id] + if ok { + _ = oldAgentSocket.Close() + } + c.agentSockets[id] = conn + c.mutex.Unlock() + defer func() { + c.mutex.Lock() + defer c.mutex.Unlock() + delete(c.agentSockets, id) + delete(c.nodes, id) + }() + + decoder := json.NewDecoder(conn) + for { + err := c.hangleAgentUpdate(id, decoder, false) + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return xerrors.Errorf("handle next agent message: %w", err) + } + } +} + +func (c *haCoordinator) hangleAgentUpdate(id uuid.UUID, decoder *json.Decoder, fromPubsub bool) error { + var node agpl.Node + err := decoder.Decode(&node) + if err != nil { + return xerrors.Errorf("read json: %w", err) + } + + c.mutex.Lock() + defer c.mutex.Unlock() + + c.nodes[id] = &node + + // Don't send the agent back over pubsub if that's where we received it from! + if !fromPubsub { + err = c.publishAgentToNodes(id, &node) + if err != nil { + return xerrors.Errorf("publish agent to nodes: %w", err) + } + } + + connectionSockets, ok := c.agentToConnectionSockets[id] + if !ok { + return nil + } + + data, err := json.Marshal([]*agpl.Node{&node}) + if err != nil { + return xerrors.Errorf("marshal nodes: %w", err) + } + + // Publish the new node to every listening socket. + var wg sync.WaitGroup + wg.Add(len(connectionSockets)) + for _, connectionSocket := range connectionSockets { + connectionSocket := connectionSocket + go func() { + _ = connectionSocket.SetWriteDeadline(time.Now().Add(5 * time.Second)) + _, _ = connectionSocket.Write(data) + wg.Done() + }() + } + + wg.Wait() + return nil +} + +func (c *haCoordinator) Close() error { + close(c.close) + return nil +} + +func (c *haCoordinator) publishNodeToAgent(recipient uuid.UUID, node *agpl.Node) error { + msg, err := c.formatCallMeMaybe(recipient, node) + if err != nil { + return xerrors.Errorf("format publish message: %w", err) + } + + fmt.Println("publishing callmemaybe", c.id.String()) + err = c.pubsub.Publish("wireguard_peers", msg) + if err != nil { + return xerrors.Errorf("publish message: %w", err) + } + + return nil +} + +func (c *haCoordinator) publishAgentToNodes(id uuid.UUID, node *agpl.Node) error { + msg, err := c.formatAgentUpdate(id, node) + if err != nil { + return xerrors.Errorf("format publish message: %w", err) + } + + fmt.Println("publishing agentupdate", c.id.String()) + err = c.pubsub.Publish("wireguard_peers", msg) + if err != nil { + return xerrors.Errorf("publish message: %w", err) + } + + return nil +} + +func (c *haCoordinator) runPubsub() error { + cancelSub, err := c.pubsub.Subscribe("wireguard_peers", func(ctx context.Context, message []byte) { + sp := bytes.Split(message, []byte("|")) + if len(sp) != 4 { + c.log.Error(ctx, "invalid wireguard peer message", slog.F("msg", string(message))) + return + } + + var ( + coordinatorID = sp[0] + eventType = sp[1] + agentID = sp[2] + nodeJSON = sp[3] + ) + + sender, err := uuid.ParseBytes(coordinatorID) + if err != nil { + c.log.Error(ctx, "invalid sender id", slog.F("id", string(coordinatorID)), slog.F("msg", string(message))) + return + } + + // We sent this message! + if sender == c.id { + return + } + + switch string(eventType) { + case "callmemaybe": + agentUUID, err := uuid.ParseBytes(agentID) + if err != nil { + c.log.Error(ctx, "invalid agent id", slog.F("id", string(agentID))) + return + } + + fmt.Println("got callmemaybe", agentUUID.String()) + c.mutex.Lock() + defer c.mutex.Unlock() + + fmt.Println("process callmemaybe", agentUUID.String()) + agentSocket, ok := c.agentSockets[agentUUID] + if !ok { + fmt.Println("no socket") + return + } + + // We get a single node over pubsub, so turn into an array. + _, err = agentSocket.Write(bytes.Join([][]byte{[]byte("["), nodeJSON, []byte("]")}, []byte{})) + if err != nil { + if errors.Is(err, io.EOF) { + return + } + c.log.Error(ctx, "send callmemaybe to agent", slog.Error(err)) + return + } + fmt.Println("success callmemaybe", agentUUID.String()) + + case "agentupdate": + agentUUID, err := uuid.ParseBytes(agentID) + if err != nil { + c.log.Error(ctx, "invalid agent id", slog.F("id", string(agentID))) + } + + decoder := json.NewDecoder(bytes.NewReader(nodeJSON)) + err = c.hangleAgentUpdate(agentUUID, decoder, true) + if err != nil { + c.log.Error(ctx, "handle agent update", slog.Error(err)) + } + + default: + c.log.Error(ctx, "unknown peer event", slog.F("name", string(eventType))) + } + }) + if err != nil { + return xerrors.Errorf("subscribe wireguard peers") + } + + go func() { + defer cancelSub() + <-c.close + }() + + return nil +} + +// format: |callmemaybe|| +func (c *haCoordinator) formatCallMeMaybe(recipient uuid.UUID, node *agpl.Node) ([]byte, error) { + buf := bytes.Buffer{} + + buf.WriteString(c.id.String() + "|") + buf.WriteString("callmemaybe|") + buf.WriteString(recipient.String() + "|") + err := json.NewEncoder(&buf).Encode(node) + if err != nil { + return nil, xerrors.Errorf("encode node: %w", err) + } + + return buf.Bytes(), nil +} + +// format: |agentupdate|| +func (c *haCoordinator) formatAgentUpdate(id uuid.UUID, node *agpl.Node) ([]byte, error) { + buf := bytes.Buffer{} + + buf.WriteString(c.id.String() + "|") + buf.WriteString("agentupdate|") + buf.WriteString(id.String() + "|") + err := json.NewEncoder(&buf).Encode(node) + if err != nil { + return nil, xerrors.Errorf("encode node: %w", err) + } + + return buf.Bytes(), nil +} diff --git a/enterprise/tailnet/coordinator_test.go b/enterprise/tailnet/coordinator_test.go new file mode 100644 index 0000000000000..48fce5bfd0f6f --- /dev/null +++ b/enterprise/tailnet/coordinator_test.go @@ -0,0 +1,267 @@ +package tailnet_test + +import ( + "fmt" + "net" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "cdr.dev/slog/sloggers/slogtest" + + "github.com/coder/coder/coderd/database" + "github.com/coder/coder/enterprise/tailnet" + agpl "github.com/coder/coder/tailnet" + "github.com/coder/coder/testutil" +) + +func TestCoordinatorSingle(t *testing.T) { + t.Parallel() + t.Run("ClientWithoutAgent", func(t *testing.T) { + t.Parallel() + coordinator, err := tailnet.NewHACoordinator(slogtest.Make(t, nil), database.NewPubsubInMemory()) + require.NoError(t, err) + defer coordinator.Close() + + client, server := net.Pipe() + sendNode, errChan := agpl.ServeCoordinator(client, func(node []*agpl.Node) error { + return nil + }) + id := uuid.New() + closeChan := make(chan struct{}) + go func() { + err := coordinator.ServeClient(server, id, uuid.New()) + assert.NoError(t, err) + close(closeChan) + }() + sendNode(&agpl.Node{}) + require.Eventually(t, func() bool { + return coordinator.Node(id) != nil + }, testutil.WaitShort, testutil.IntervalFast) + + err = client.Close() + require.NoError(t, err) + <-errChan + <-closeChan + }) + + t.Run("AgentWithoutClients", func(t *testing.T) { + t.Parallel() + coordinator, err := tailnet.NewHACoordinator(slogtest.Make(t, nil), database.NewPubsubInMemory()) + require.NoError(t, err) + defer coordinator.Close() + + client, server := net.Pipe() + sendNode, errChan := agpl.ServeCoordinator(client, func(node []*agpl.Node) error { + return nil + }) + id := uuid.New() + closeChan := make(chan struct{}) + go func() { + err := coordinator.ServeAgent(server, id) + assert.NoError(t, err) + close(closeChan) + }() + sendNode(&agpl.Node{}) + require.Eventually(t, func() bool { + return coordinator.Node(id) != nil + }, testutil.WaitShort, testutil.IntervalFast) + err = client.Close() + require.NoError(t, err) + <-errChan + <-closeChan + }) + + t.Run("AgentWithClient", func(t *testing.T) { + t.Parallel() + + coordinator, err := tailnet.NewHACoordinator(slogtest.Make(t, nil), database.NewPubsubInMemory()) + require.NoError(t, err) + defer coordinator.Close() + + agentWS, agentServerWS := net.Pipe() + defer agentWS.Close() + agentNodeChan := make(chan []*agpl.Node) + sendAgentNode, agentErrChan := agpl.ServeCoordinator(agentWS, func(nodes []*agpl.Node) error { + agentNodeChan <- nodes + return nil + }) + agentID := uuid.New() + closeAgentChan := make(chan struct{}) + go func() { + err := coordinator.ServeAgent(agentServerWS, agentID) + assert.NoError(t, err) + close(closeAgentChan) + }() + sendAgentNode(&agpl.Node{}) + require.Eventually(t, func() bool { + return coordinator.Node(agentID) != nil + }, testutil.WaitShort, testutil.IntervalFast) + + clientWS, clientServerWS := net.Pipe() + defer clientWS.Close() + defer clientServerWS.Close() + clientNodeChan := make(chan []*agpl.Node) + sendClientNode, clientErrChan := agpl.ServeCoordinator(clientWS, func(nodes []*agpl.Node) error { + clientNodeChan <- nodes + return nil + }) + clientID := uuid.New() + closeClientChan := make(chan struct{}) + go func() { + err := coordinator.ServeClient(clientServerWS, clientID, agentID) + assert.NoError(t, err) + close(closeClientChan) + }() + agentNodes := <-clientNodeChan + require.Len(t, agentNodes, 1) + sendClientNode(&agpl.Node{}) + clientNodes := <-agentNodeChan + require.Len(t, clientNodes, 1) + + // Ensure an update to the agent node reaches the client! + sendAgentNode(&agpl.Node{}) + agentNodes = <-clientNodeChan + require.Len(t, agentNodes, 1) + + // Close the agent WebSocket so a new one can connect. + err = agentWS.Close() + require.NoError(t, err) + <-agentErrChan + <-closeAgentChan + + // Create a new agent connection. This is to simulate a reconnect! + agentWS, agentServerWS = net.Pipe() + defer agentWS.Close() + agentNodeChan = make(chan []*agpl.Node) + _, agentErrChan = agpl.ServeCoordinator(agentWS, func(nodes []*agpl.Node) error { + agentNodeChan <- nodes + return nil + }) + closeAgentChan = make(chan struct{}) + go func() { + err := coordinator.ServeAgent(agentServerWS, agentID) + assert.NoError(t, err) + close(closeAgentChan) + }() + // Ensure the existing listening client sends it's node immediately! + clientNodes = <-agentNodeChan + require.Len(t, clientNodes, 1) + + err = agentWS.Close() + require.NoError(t, err) + <-agentErrChan + <-closeAgentChan + + err = clientWS.Close() + require.NoError(t, err) + <-clientErrChan + <-closeClientChan + }) +} + +func TestCoordinatorHA(t *testing.T) { + t.Parallel() + + t.Run("AgentWithClient", func(t *testing.T) { + t.Parallel() + + pubsub := database.NewPubsubInMemory() + + coordinator1, err := tailnet.NewHACoordinator(slogtest.Make(t, nil), pubsub) + require.NoError(t, err) + defer coordinator1.Close() + + coordinator2, err := tailnet.NewHACoordinator(slogtest.Make(t, nil), pubsub) + require.NoError(t, err) + defer coordinator2.Close() + + agentWS, agentServerWS := net.Pipe() + defer agentWS.Close() + agentNodeChan := make(chan []*agpl.Node) + sendAgentNode, agentErrChan := agpl.ServeCoordinator(agentWS, func(nodes []*agpl.Node) error { + fmt.Println("got agent node") + agentNodeChan <- nodes + fmt.Println("sent agent node") + return nil + }) + agentID := uuid.New() + closeAgentChan := make(chan struct{}) + go func() { + err := coordinator1.ServeAgent(agentServerWS, agentID) + assert.NoError(t, err) + close(closeAgentChan) + }() + sendAgentNode(&agpl.Node{}) + require.Eventually(t, func() bool { + return coordinator1.Node(agentID) != nil + }, testutil.WaitShort, testutil.IntervalFast) + + clientWS, clientServerWS := net.Pipe() + defer clientWS.Close() + defer clientServerWS.Close() + clientNodeChan := make(chan []*agpl.Node) + sendClientNode, clientErrChan := agpl.ServeCoordinator(clientWS, func(nodes []*agpl.Node) error { + fmt.Println("got client node") + clientNodeChan <- nodes + fmt.Println("sent client node") + return nil + }) + clientID := uuid.New() + closeClientChan := make(chan struct{}) + go func() { + err := coordinator2.ServeClient(clientServerWS, clientID, agentID) + assert.NoError(t, err) + close(closeClientChan) + }() + agentNodes := <-clientNodeChan + require.Len(t, agentNodes, 1) + sendClientNode(&agpl.Node{}) + _ = sendClientNode + clientNodes := <-agentNodeChan + require.Len(t, clientNodes, 1) + + // Ensure an update to the agent node reaches the client! + sendAgentNode(&agpl.Node{}) + agentNodes = <-clientNodeChan + require.Len(t, agentNodes, 1) + + // Close the agent WebSocket so a new one can connect. + require.NoError(t, agentWS.Close()) + require.NoError(t, agentServerWS.Close()) + <-agentErrChan + <-closeAgentChan + + // Create a new agent connection. This is to simulate a reconnect! + agentWS, agentServerWS = net.Pipe() + defer agentWS.Close() + agentNodeChan = make(chan []*agpl.Node) + _, agentErrChan = agpl.ServeCoordinator(agentWS, func(nodes []*agpl.Node) error { + fmt.Println("got agent node") + agentNodeChan <- nodes + fmt.Println("sent agent node") + return nil + }) + closeAgentChan = make(chan struct{}) + go func() { + err := coordinator1.ServeAgent(agentServerWS, agentID) + assert.NoError(t, err) + close(closeAgentChan) + }() + // Ensure the existing listening client sends it's node immediately! + clientNodes = <-agentNodeChan + require.Len(t, clientNodes, 1) + + err = agentWS.Close() + require.NoError(t, err) + <-agentErrChan + <-closeAgentChan + + err = clientWS.Close() + require.NoError(t, err) + <-clientErrChan + <-closeClientChan + }) +} diff --git a/tailnet/coordinator.go b/tailnet/coordinator.go index 95209d56559ff..af6a5fee58288 100644 --- a/tailnet/coordinator.go +++ b/tailnet/coordinator.go @@ -7,6 +7,7 @@ import ( "net" "net/netip" "sync" + "time" "github.com/google/uuid" "golang.org/x/xerrors" @@ -14,6 +15,24 @@ import ( "tailscale.com/types/key" ) +// Coordinator exchanges nodes with agents to establish connections. +// ┌──────────────────┐ ┌────────────────────┐ ┌───────────────────┐ ┌──────────────────┐ +// │tailnet.Coordinate├──►│tailnet.AcceptClient│◄─►│tailnet.AcceptAgent│◄──┤tailnet.Coordinate│ +// └──────────────────┘ └────────────────────┘ └───────────────────┘ └──────────────────┘ +// Coordinators have different guarantees for HA support. +type Coordinator interface { + // Node returns an in-memory node by ID. + Node(id uuid.UUID) *Node + // ServeClient accepts a WebSocket connection that wants to connect to an agent + // with the specified ID. + ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID) error + // ServeAgent accepts a WebSocket connection to an agent that listens to + // incoming connections and publishes node updates. + ServeAgent(conn net.Conn, id uuid.UUID) error + // Close closes the coordinator. + Close() error +} + // Node represents a node in the network. type Node struct { ID tailcfg.NodeID `json:"id"` @@ -64,44 +83,46 @@ func ServeCoordinator(conn net.Conn, updateNodes func(node []*Node) error) (func }, errChan } -// NewCoordinator constructs a new in-memory connection coordinator. -func NewCoordinator() *Coordinator { - return &Coordinator{ +// NewMemoryCoordinator constructs a new in-memory connection coordinator. This +// coordinator is incompatible with multiple Coder replicas as all node data is +// in-memory. +func NewMemoryCoordinator() Coordinator { + return &memoryCoordinator{ nodes: map[uuid.UUID]*Node{}, agentSockets: map[uuid.UUID]net.Conn{}, agentToConnectionSockets: map[uuid.UUID]map[uuid.UUID]net.Conn{}, } } -// Coordinator exchanges nodes with agents to establish connections. +// MemoryCoordinator exchanges nodes with agents to establish connections. // ┌──────────────────┐ ┌────────────────────┐ ┌───────────────────┐ ┌──────────────────┐ // │tailnet.Coordinate├──►│tailnet.AcceptClient│◄─►│tailnet.AcceptAgent│◄──┤tailnet.Coordinate│ // └──────────────────┘ └────────────────────┘ └───────────────────┘ └──────────────────┘ // This coordinator is incompatible with multiple Coder // replicas as all node data is in-memory. -type Coordinator struct { +type memoryCoordinator struct { mutex sync.Mutex - // Maps agent and connection IDs to a node. + // nodes maps agent and connection IDs their respective node. nodes map[uuid.UUID]*Node - // Maps agent ID to an open socket. + // agentSockets maps agent IDs to their open websocket. agentSockets map[uuid.UUID]net.Conn - // Maps agent ID to connection ID for sending - // new node data as it comes in! + // agentToConnectionSockets maps agent IDs to connection IDs of conns that + // are subscribed to updates for that agent. agentToConnectionSockets map[uuid.UUID]map[uuid.UUID]net.Conn } // Node returns an in-memory node by ID. -func (c *Coordinator) Node(id uuid.UUID) *Node { +func (c *memoryCoordinator) Node(id uuid.UUID) *Node { c.mutex.Lock() defer c.mutex.Unlock() node := c.nodes[id] return node } -// ServeClient accepts a WebSocket connection that wants to -// connect to an agent with the specified ID. -func (c *Coordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID) error { +// ServeClient accepts a WebSocket connection that wants to connect to an agent +// with the specified ID. +func (c *memoryCoordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID) error { c.mutex.Lock() // When a new connection is requested, we update it with the latest // node of the agent. This allows the connection to establish. @@ -145,48 +166,67 @@ func (c *Coordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID) decoder := json.NewDecoder(conn) for { - var node Node - err := decoder.Decode(&node) - if errors.Is(err, io.EOF) { - return nil - } - if err != nil { - return xerrors.Errorf("read json: %w", err) - } - c.mutex.Lock() - // Update the node of this client in our in-memory map. - // If an agent entirely shuts down and reconnects, it - // needs to be aware of all clients attempting to - // establish connections. - c.nodes[id] = &node - agentSocket, ok := c.agentSockets[agent] - if !ok { - c.mutex.Unlock() - continue - } - // Write the new node from this client to the actively - // connected agent. - data, err := json.Marshal([]*Node{&node}) + err := c.handleNextClientMessage(id, agent, decoder) if err != nil { - c.mutex.Unlock() - return xerrors.Errorf("marshal nodes: %w", err) + if errors.Is(err, io.EOF) { + return nil + } + return xerrors.Errorf("handle next client message: %w", err) } - _, err = agentSocket.Write(data) + } +} + +func (c *memoryCoordinator) handleNextClientMessage(id, agent uuid.UUID, decoder *json.Decoder) error { + var node Node + err := decoder.Decode(&node) + if err != nil { + return xerrors.Errorf("read json: %w", err) + } + + c.mutex.Lock() + defer c.mutex.Unlock() + + // Update the node of this client in our in-memory map. If an agent + // entirely shuts down and reconnects, it needs to be aware of all clients + // attempting to establish connections. + c.nodes[id] = &node + + // Write the new node from this client to the actively + // connected agent. + err = c.writeNodeToAgent(agent, &node) + if err != nil { + return xerrors.Errorf("write node to agent: %w", err) + } + + return nil +} + +func (c *memoryCoordinator) writeNodeToAgent(agent uuid.UUID, node *Node) error { + agentSocket, ok := c.agentSockets[agent] + if !ok { + return nil + } + + // Write the new node from this client to the actively + // connected agent. + data, err := json.Marshal([]*Node{node}) + if err != nil { + return xerrors.Errorf("marshal nodes: %w", err) + } + + _, err = agentSocket.Write(data) + if err != nil { if errors.Is(err, io.EOF) { - c.mutex.Unlock() return nil } - if err != nil { - c.mutex.Unlock() - return xerrors.Errorf("write json: %w", err) - } - c.mutex.Unlock() + return xerrors.Errorf("write json: %w", err) } + return nil } // ServeAgent accepts a WebSocket connection to an agent that // listens to incoming connections and publishes node updates. -func (c *Coordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { +func (c *memoryCoordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { c.mutex.Lock() sockets, ok := c.agentToConnectionSockets[id] if ok { @@ -230,36 +270,51 @@ func (c *Coordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { decoder := json.NewDecoder(conn) for { - var node Node - err := decoder.Decode(&node) - if errors.Is(err, io.EOF) { - return nil - } - if err != nil { - return xerrors.Errorf("read json: %w", err) - } - c.mutex.Lock() - c.nodes[id] = &node - connectionSockets, ok := c.agentToConnectionSockets[id] - if !ok { - c.mutex.Unlock() - continue - } - data, err := json.Marshal([]*Node{&node}) + err := c.handleNextAgentMessage(id, decoder) if err != nil { - return xerrors.Errorf("marshal nodes: %w", err) - } - // Publish the new node to every listening socket. - var wg sync.WaitGroup - wg.Add(len(connectionSockets)) - for _, connectionSocket := range connectionSockets { - connectionSocket := connectionSocket - go func() { - _, _ = connectionSocket.Write(data) - wg.Done() - }() + if errors.Is(err, io.EOF) { + return nil + } + return xerrors.Errorf("handle next agent message: %w", err) } - wg.Wait() - c.mutex.Unlock() } } + +func (c *memoryCoordinator) handleNextAgentMessage(id uuid.UUID, decoder *json.Decoder) error { + var node Node + err := decoder.Decode(&node) + if err != nil { + return xerrors.Errorf("read json: %w", err) + } + + c.mutex.Lock() + defer c.mutex.Unlock() + + c.nodes[id] = &node + connectionSockets, ok := c.agentToConnectionSockets[id] + if !ok { + return nil + } + + data, err := json.Marshal([]*Node{&node}) + if err != nil { + return xerrors.Errorf("marshal nodes: %w", err) + } + + // Publish the new node to every listening socket. + var wg sync.WaitGroup + wg.Add(len(connectionSockets)) + for _, connectionSocket := range connectionSockets { + connectionSocket := connectionSocket + go func() { + _ = connectionSocket.SetWriteDeadline(time.Now().Add(5 * time.Second)) + _, _ = connectionSocket.Write(data) + wg.Done() + }() + } + + wg.Wait() + return nil +} + +func (*memoryCoordinator) Close() error { return nil } diff --git a/tailnet/coordinator_test.go b/tailnet/coordinator_test.go index f3fdab88d5ef8..e0ed44420ede2 100644 --- a/tailnet/coordinator_test.go +++ b/tailnet/coordinator_test.go @@ -16,7 +16,7 @@ func TestCoordinator(t *testing.T) { t.Parallel() t.Run("ClientWithoutAgent", func(t *testing.T) { t.Parallel() - coordinator := tailnet.NewCoordinator() + coordinator := tailnet.NewMemoryCoordinator() client, server := net.Pipe() sendNode, errChan := tailnet.ServeCoordinator(client, func(node []*tailnet.Node) error { return nil @@ -32,15 +32,15 @@ func TestCoordinator(t *testing.T) { require.Eventually(t, func() bool { return coordinator.Node(id) != nil }, testutil.WaitShort, testutil.IntervalFast) - err := client.Close() - require.NoError(t, err) + require.NoError(t, client.Close()) + require.NoError(t, server.Close()) <-errChan <-closeChan }) t.Run("AgentWithoutClients", func(t *testing.T) { t.Parallel() - coordinator := tailnet.NewCoordinator() + coordinator := tailnet.NewMemoryCoordinator() client, server := net.Pipe() sendNode, errChan := tailnet.ServeCoordinator(client, func(node []*tailnet.Node) error { return nil @@ -64,7 +64,7 @@ func TestCoordinator(t *testing.T) { t.Run("AgentWithClient", func(t *testing.T) { t.Parallel() - coordinator := tailnet.NewCoordinator() + coordinator := tailnet.NewMemoryCoordinator() agentWS, agentServerWS := net.Pipe() defer agentWS.Close() From 68a812b134d43b3777d7173fdeadc503eea9ad4e Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Fri, 23 Sep 2022 13:26:25 -0500 Subject: [PATCH 02/79] fixup! feat: HA tailnet coordinator --- enterprise/tailnet/coordinator.go | 132 +++++++++++++++++++++--------- 1 file changed, 92 insertions(+), 40 deletions(-) diff --git a/enterprise/tailnet/coordinator.go b/enterprise/tailnet/coordinator.go index 8824f584d60da..6999fa7157d48 100644 --- a/enterprise/tailnet/coordinator.go +++ b/enterprise/tailnet/coordinator.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "io" "net" "sync" @@ -150,7 +149,7 @@ func (c *haCoordinator) writeNodeToAgent(agent uuid.UUID, node *agpl.Node) error if !ok { // If we don't own the agent locally, send it over pubsub to a node that // owns the agent. - err := c.publishNodeToAgent(agent, node) + err := c.publishNodesToAgent(agent, []*agpl.Node{node}) if err != nil { return xerrors.Errorf("publish node to agent") } @@ -178,18 +177,15 @@ func (c *haCoordinator) writeNodeToAgent(agent uuid.UUID, node *agpl.Node) error // incoming connections and publishes node updates. func (c *haCoordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { c.mutex.Lock() - sockets, ok := c.agentToConnectionSockets[id] - if ok { - // Publish all nodes that want to connect to the - // desired agent ID. - nodes := make([]*agpl.Node, 0, len(sockets)) - for targetID := range sockets { - node, ok := c.nodes[targetID] - if !ok { - continue - } - nodes = append(nodes, node) - } + + // Tell clients on other instances to send a callmemaybe to us. + err := c.publishAgentHello(id) + if err != nil { + return xerrors.Errorf("publish agent hello: %w", err) + } + + nodes := c.nodesSubscribedToAgent(id) + if len(nodes) > 0 { data, err := json.Marshal(nodes) if err != nil { c.mutex.Unlock() @@ -220,21 +216,46 @@ func (c *haCoordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { decoder := json.NewDecoder(conn) for { - err := c.hangleAgentUpdate(id, decoder, false) + node, err := c.hangleAgentUpdate(id, decoder) if err != nil { if errors.Is(err, io.EOF) { return nil } return xerrors.Errorf("handle next agent message: %w", err) } + + err = c.publishAgentToNodes(id, node) + if err != nil { + return xerrors.Errorf("publish agent to nodes: %w", err) + } } } -func (c *haCoordinator) hangleAgentUpdate(id uuid.UUID, decoder *json.Decoder, fromPubsub bool) error { +func (c *haCoordinator) nodesSubscribedToAgent(agentID uuid.UUID) []*agpl.Node { + sockets, ok := c.agentToConnectionSockets[agentID] + if !ok { + return nil + } + + // Publish all nodes that want to connect to the + // desired agent ID. + nodes := make([]*agpl.Node, 0, len(sockets)) + for targetID := range sockets { + node, ok := c.nodes[targetID] + if !ok { + continue + } + nodes = append(nodes, node) + } + + return nodes +} + +func (c *haCoordinator) hangleAgentUpdate(id uuid.UUID, decoder *json.Decoder) (*agpl.Node, error) { var node agpl.Node err := decoder.Decode(&node) if err != nil { - return xerrors.Errorf("read json: %w", err) + return nil, xerrors.Errorf("read json: %w", err) } c.mutex.Lock() @@ -242,22 +263,14 @@ func (c *haCoordinator) hangleAgentUpdate(id uuid.UUID, decoder *json.Decoder, f c.nodes[id] = &node - // Don't send the agent back over pubsub if that's where we received it from! - if !fromPubsub { - err = c.publishAgentToNodes(id, &node) - if err != nil { - return xerrors.Errorf("publish agent to nodes: %w", err) - } - } - connectionSockets, ok := c.agentToConnectionSockets[id] if !ok { - return nil + return &node, nil } data, err := json.Marshal([]*agpl.Node{&node}) if err != nil { - return xerrors.Errorf("marshal nodes: %w", err) + return nil, xerrors.Errorf("marshal nodes: %w", err) } // Publish the new node to every listening socket. @@ -273,7 +286,7 @@ func (c *haCoordinator) hangleAgentUpdate(id uuid.UUID, decoder *json.Decoder, f } wg.Wait() - return nil + return &node, nil } func (c *haCoordinator) Close() error { @@ -281,13 +294,26 @@ func (c *haCoordinator) Close() error { return nil } -func (c *haCoordinator) publishNodeToAgent(recipient uuid.UUID, node *agpl.Node) error { - msg, err := c.formatCallMeMaybe(recipient, node) +func (c *haCoordinator) publishNodesToAgent(recipient uuid.UUID, nodes []*agpl.Node) error { + msg, err := c.formatCallMeMaybe(recipient, nodes) + if err != nil { + return xerrors.Errorf("format publish message: %w", err) + } + + err = c.pubsub.Publish("wireguard_peers", msg) + if err != nil { + return xerrors.Errorf("publish message: %w", err) + } + + return nil +} + +func (c *haCoordinator) publishAgentHello(id uuid.UUID) error { + msg, err := c.formatAgentHello(id) if err != nil { return xerrors.Errorf("format publish message: %w", err) } - fmt.Println("publishing callmemaybe", c.id.String()) err = c.pubsub.Publish("wireguard_peers", msg) if err != nil { return xerrors.Errorf("publish message: %w", err) @@ -302,7 +328,6 @@ func (c *haCoordinator) publishAgentToNodes(id uuid.UUID, node *agpl.Node) error return xerrors.Errorf("format publish message: %w", err) } - fmt.Println("publishing agentupdate", c.id.String()) err = c.pubsub.Publish("wireguard_peers", msg) if err != nil { return xerrors.Errorf("publish message: %w", err) @@ -345,19 +370,16 @@ func (c *haCoordinator) runPubsub() error { return } - fmt.Println("got callmemaybe", agentUUID.String()) c.mutex.Lock() defer c.mutex.Unlock() - fmt.Println("process callmemaybe", agentUUID.String()) agentSocket, ok := c.agentSockets[agentUUID] if !ok { - fmt.Println("no socket") return } // We get a single node over pubsub, so turn into an array. - _, err = agentSocket.Write(bytes.Join([][]byte{[]byte("["), nodeJSON, []byte("]")}, []byte{})) + _, err = agentSocket.Write(nodeJSON) if err != nil { if errors.Is(err, io.EOF) { return @@ -365,18 +387,37 @@ func (c *haCoordinator) runPubsub() error { c.log.Error(ctx, "send callmemaybe to agent", slog.Error(err)) return } - fmt.Println("success callmemaybe", agentUUID.String()) + + case "agenthello": + agentUUID, err := uuid.ParseBytes(agentID) + if err != nil { + c.log.Error(ctx, "invalid agent id", slog.F("id", string(agentID))) + return + } + + c.mutex.Lock() + nodes := c.nodesSubscribedToAgent(agentUUID) + c.mutex.Unlock() + if len(nodes) > 0 { + err := c.publishNodesToAgent(agentUUID, nodes) + if err != nil { + c.log.Error(ctx, "publish nodes to agent", slog.Error(err)) + return + } + } case "agentupdate": agentUUID, err := uuid.ParseBytes(agentID) if err != nil { c.log.Error(ctx, "invalid agent id", slog.F("id", string(agentID))) + return } decoder := json.NewDecoder(bytes.NewReader(nodeJSON)) - err = c.hangleAgentUpdate(agentUUID, decoder, true) + _, err = c.hangleAgentUpdate(agentUUID, decoder) if err != nil { c.log.Error(ctx, "handle agent update", slog.Error(err)) + return } default: @@ -396,13 +437,13 @@ func (c *haCoordinator) runPubsub() error { } // format: |callmemaybe|| -func (c *haCoordinator) formatCallMeMaybe(recipient uuid.UUID, node *agpl.Node) ([]byte, error) { +func (c *haCoordinator) formatCallMeMaybe(recipient uuid.UUID, nodes []*agpl.Node) ([]byte, error) { buf := bytes.Buffer{} buf.WriteString(c.id.String() + "|") buf.WriteString("callmemaybe|") buf.WriteString(recipient.String() + "|") - err := json.NewEncoder(&buf).Encode(node) + err := json.NewEncoder(&buf).Encode(nodes) if err != nil { return nil, xerrors.Errorf("encode node: %w", err) } @@ -410,6 +451,17 @@ func (c *haCoordinator) formatCallMeMaybe(recipient uuid.UUID, node *agpl.Node) return buf.Bytes(), nil } +// format: |agenthello|| +func (c *haCoordinator) formatAgentHello(id uuid.UUID) ([]byte, error) { + buf := bytes.Buffer{} + + buf.WriteString(c.id.String() + "|") + buf.WriteString("agenthello|") + buf.WriteString(id.String() + "|") + + return buf.Bytes(), nil +} + // format: |agentupdate|| func (c *haCoordinator) formatAgentUpdate(id uuid.UUID, node *agpl.Node) ([]byte, error) { buf := bytes.Buffer{} From 774c5dafe3cb41a9eca1819531f073fd8ff9c9b9 Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Fri, 23 Sep 2022 13:29:40 -0500 Subject: [PATCH 03/79] fixup! feat: HA tailnet coordinator --- enterprise/tailnet/coordinator.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/enterprise/tailnet/coordinator.go b/enterprise/tailnet/coordinator.go index 6999fa7157d48..61b4bd5759ace 100644 --- a/enterprise/tailnet/coordinator.go +++ b/enterprise/tailnet/coordinator.go @@ -184,6 +184,7 @@ func (c *haCoordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { return xerrors.Errorf("publish agent hello: %w", err) } + // Publish all nodes on this instance that want to connect to this agent. nodes := c.nodesSubscribedToAgent(id) if len(nodes) > 0 { data, err := json.Marshal(nodes) @@ -237,8 +238,6 @@ func (c *haCoordinator) nodesSubscribedToAgent(agentID uuid.UUID) []*agpl.Node { return nil } - // Publish all nodes that want to connect to the - // desired agent ID. nodes := make([]*agpl.Node, 0, len(sockets)) for targetID := range sockets { node, ok := c.nodes[targetID] From bd82c5e36c79c080954b38255c8d198a0f0b925f Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Fri, 23 Sep 2022 13:30:58 -0500 Subject: [PATCH 04/79] remove printlns --- enterprise/tailnet/coordinator_test.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/enterprise/tailnet/coordinator_test.go b/enterprise/tailnet/coordinator_test.go index 48fce5bfd0f6f..4889cd1c8ba60 100644 --- a/enterprise/tailnet/coordinator_test.go +++ b/enterprise/tailnet/coordinator_test.go @@ -1,7 +1,6 @@ package tailnet_test import ( - "fmt" "net" "testing" @@ -182,9 +181,7 @@ func TestCoordinatorHA(t *testing.T) { defer agentWS.Close() agentNodeChan := make(chan []*agpl.Node) sendAgentNode, agentErrChan := agpl.ServeCoordinator(agentWS, func(nodes []*agpl.Node) error { - fmt.Println("got agent node") agentNodeChan <- nodes - fmt.Println("sent agent node") return nil }) agentID := uuid.New() @@ -204,9 +201,7 @@ func TestCoordinatorHA(t *testing.T) { defer clientServerWS.Close() clientNodeChan := make(chan []*agpl.Node) sendClientNode, clientErrChan := agpl.ServeCoordinator(clientWS, func(nodes []*agpl.Node) error { - fmt.Println("got client node") clientNodeChan <- nodes - fmt.Println("sent client node") return nil }) clientID := uuid.New() @@ -239,9 +234,7 @@ func TestCoordinatorHA(t *testing.T) { defer agentWS.Close() agentNodeChan = make(chan []*agpl.Node) _, agentErrChan = agpl.ServeCoordinator(agentWS, func(nodes []*agpl.Node) error { - fmt.Println("got agent node") agentNodeChan <- nodes - fmt.Println("sent agent node") return nil }) closeAgentChan = make(chan struct{}) From fbad8d075ddfb47d99c9cd7f2d1696ded78266ed Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Fri, 7 Oct 2022 11:49:19 -0500 Subject: [PATCH 05/79] close all connections on coordinator --- codersdk/features.go | 12 ++++--- enterprise/coderd/coderd.go | 8 +++++ enterprise/coderd/license/license.go | 25 +++++++++++---- enterprise/tailnet/coordinator.go | 30 ++++++++++++++++- tailnet/coordinator.go | 48 ++++++++++++++++++++++++++-- 5 files changed, 109 insertions(+), 14 deletions(-) diff --git a/codersdk/features.go b/codersdk/features.go index fe8673ef028fd..6884f44087629 100644 --- a/codersdk/features.go +++ b/codersdk/features.go @@ -15,11 +15,12 @@ const ( ) const ( - FeatureUserLimit = "user_limit" - FeatureAuditLog = "audit_log" - FeatureBrowserOnly = "browser_only" - FeatureSCIM = "scim" - FeatureWorkspaceQuota = "workspace_quota" + FeatureUserLimit = "user_limit" + FeatureAuditLog = "audit_log" + FeatureBrowserOnly = "browser_only" + FeatureSCIM = "scim" + FeatureWorkspaceQuota = "workspace_quota" + FeatureHighAvailability = "high_availability" ) var FeatureNames = []string{ @@ -28,6 +29,7 @@ var FeatureNames = []string{ FeatureBrowserOnly, FeatureSCIM, FeatureWorkspaceQuota, + FeatureHighAvailability, } type Feature struct { diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 11cceef98f0db..a6595e8bd6554 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -170,6 +170,14 @@ func (api *API) updateEntitlements(ctx context.Context) error { api.AGPL.WorkspaceQuotaEnforcer.Store(&enforcer) } + if changed, enabled := featureChanged(codersdk.FeatureHighAvailability); changed { + enforcer := workspacequota.NewNop() + if enabled { + enforcer = NewEnforcer(api.Options.UserWorkspaceQuota) + } + api.AGPL.WorkspaceQuotaEnforcer.Store(&enforcer) + } + api.entitlements = entitlements return nil diff --git a/enterprise/coderd/license/license.go b/enterprise/coderd/license/license.go index 55a62eee17eee..84d28dfcccb21 100644 --- a/enterprise/coderd/license/license.go +++ b/enterprise/coderd/license/license.go @@ -17,7 +17,13 @@ import ( ) // Entitlements processes licenses to return whether features are enabled or not. -func Entitlements(ctx context.Context, db database.Store, logger slog.Logger, keys map[string]ed25519.PublicKey, enablements map[string]bool) (codersdk.Entitlements, error) { +func Entitlements( + ctx context.Context, + db database.Store, + logger slog.Logger, + keys map[string]ed25519.PublicKey, + enablements map[string]bool, +) (codersdk.Entitlements, error) { now := time.Now() // Default all entitlements to be disabled. entitlements := codersdk.Entitlements{ @@ -96,6 +102,12 @@ func Entitlements(ctx context.Context, db database.Store, logger slog.Logger, ke Enabled: enablements[codersdk.FeatureWorkspaceQuota], } } + if claims.Features.HighAvailability > 0 { + entitlements.Features[codersdk.FeatureHighAvailability] = codersdk.Feature{ + Entitlement: entitlement, + Enabled: enablements[codersdk.FeatureHighAvailability], + } + } if claims.AllFeatures { allFeatures = true } @@ -165,11 +177,12 @@ var ( ) type Features struct { - UserLimit int64 `json:"user_limit"` - AuditLog int64 `json:"audit_log"` - BrowserOnly int64 `json:"browser_only"` - SCIM int64 `json:"scim"` - WorkspaceQuota int64 `json:"workspace_quota"` + UserLimit int64 `json:"user_limit"` + AuditLog int64 `json:"audit_log"` + BrowserOnly int64 `json:"browser_only"` + SCIM int64 `json:"scim"` + WorkspaceQuota int64 `json:"workspace_quota"` + HighAvailability int64 `json:"high_availability"` } type Claims struct { diff --git a/enterprise/tailnet/coordinator.go b/enterprise/tailnet/coordinator.go index 61b4bd5759ace..6bf2327507165 100644 --- a/enterprise/tailnet/coordinator.go +++ b/enterprise/tailnet/coordinator.go @@ -14,7 +14,6 @@ import ( "golang.org/x/xerrors" "cdr.dev/slog" - "github.com/coder/coder/coderd/database" agpl "github.com/coder/coder/tailnet" ) @@ -288,8 +287,37 @@ func (c *haCoordinator) hangleAgentUpdate(id uuid.UUID, decoder *json.Decoder) ( return &node, nil } +// Close closes all of the open connections in the coordinator and stops the +// coordinator from accepting new connections. func (c *haCoordinator) Close() error { + c.mutex.Lock() + defer c.mutex.Unlock() + close(c.close) + + wg := sync.WaitGroup{} + + wg.Add(len(c.agentSockets)) + for _, socket := range c.agentSockets { + socket := socket + go func() { + _ = socket.Close() + wg.Done() + }() + } + + for _, connMap := range c.agentToConnectionSockets { + wg.Add(len(connMap)) + for _, socket := range connMap { + socket := socket + go func() { + _ = socket.Close() + wg.Done() + }() + } + } + + wg.Wait() return nil } diff --git a/tailnet/coordinator.go b/tailnet/coordinator.go index d79ffa34a5a3b..150a323bcfe52 100644 --- a/tailnet/coordinator.go +++ b/tailnet/coordinator.go @@ -99,6 +99,7 @@ func ServeCoordinator(conn net.Conn, updateNodes func(node []*Node) error) (func // in-memory. func NewMemoryCoordinator() Coordinator { return &memoryCoordinator{ + closed: false, nodes: map[uuid.UUID]*Node{}, agentSockets: map[uuid.UUID]net.Conn{}, agentToConnectionSockets: map[uuid.UUID]map[uuid.UUID]net.Conn{}, @@ -112,7 +113,8 @@ func NewMemoryCoordinator() Coordinator { // This coordinator is incompatible with multiple Coder // replicas as all node data is in-memory. type memoryCoordinator struct { - mutex sync.Mutex + mutex sync.Mutex + closed bool // nodes maps agent and connection IDs their respective node. nodes map[uuid.UUID]*Node @@ -135,6 +137,11 @@ func (c *memoryCoordinator) Node(id uuid.UUID) *Node { // with the specified ID. func (c *memoryCoordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID) error { c.mutex.Lock() + + if c.closed { + return xerrors.New("coordinator is closed") + } + // When a new connection is requested, we update it with the latest // node of the agent. This allows the connection to establish. node, ok := c.nodes[agent] @@ -229,6 +236,11 @@ func (c *memoryCoordinator) handleNextClientMessage(id, agent uuid.UUID, decoder // listens to incoming connections and publishes node updates. func (c *memoryCoordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { c.mutex.Lock() + + if c.closed { + return xerrors.New("coordinator is closed") + } + sockets, ok := c.agentToConnectionSockets[id] if ok { // Publish all nodes that want to connect to the @@ -320,4 +332,36 @@ func (c *memoryCoordinator) handleNextAgentMessage(id uuid.UUID, decoder *json.D return nil } -func (*memoryCoordinator) Close() error { return nil } +// Close closes all of the open connections in the coordinator and stops the +// coordinator from accepting new connections. +func (c *memoryCoordinator) Close() error { + c.mutex.Lock() + defer c.mutex.Unlock() + + c.closed = true + + wg := sync.WaitGroup{} + + wg.Add(len(c.agentSockets)) + for _, socket := range c.agentSockets { + socket := socket + go func() { + _ = socket.Close() + wg.Done() + }() + } + + for _, connMap := range c.agentToConnectionSockets { + wg.Add(len(connMap)) + for _, socket := range connMap { + socket := socket + go func() { + _ = socket.Close() + wg.Done() + }() + } + } + + wg.Wait() + return nil +} From 46803aa38ba2d4189f687bda248f01bf933bf18e Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Fri, 7 Oct 2022 12:22:44 -0500 Subject: [PATCH 06/79] impelement high availability feature --- coderd/coderd.go | 2 ++ coderd/provisionerjobs.go | 2 +- coderd/workspaceagents.go | 16 ++++++++-------- coderd/workspacebuilds.go | 2 +- enterprise/coderd/coderd.go | 24 +++++++++++++++++++++--- 5 files changed, 33 insertions(+), 13 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index 58686ae66fbcd..f3cdab0caea04 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -158,6 +158,7 @@ func New(options *Options) *API { api.Auditor.Store(&options.Auditor) api.WorkspaceQuotaEnforcer.Store(&options.WorkspaceQuotaEnforcer) api.workspaceAgentCache = wsconncache.New(api.dialWorkspaceAgentTailnet, 0) + api.TailnetCoordinator.Store(&options.TailnetCoordinator) api.derpServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger)) oauthConfigs := &httpmw.OAuth2Configs{ Github: options.GithubOAuth2Config, @@ -525,6 +526,7 @@ type API struct { Auditor atomic.Pointer[audit.Auditor] WorkspaceClientCoordinateOverride atomic.Pointer[func(rw http.ResponseWriter) bool] WorkspaceQuotaEnforcer atomic.Pointer[workspacequota.Enforcer] + TailnetCoordinator atomic.Pointer[tailnet.Coordinator] HTTPAuth *HTTPAuthorizer // APIHandler serves "/api/v2" diff --git a/coderd/provisionerjobs.go b/coderd/provisionerjobs.go index 56a825ea09a3a..68802df04e5ec 100644 --- a/coderd/provisionerjobs.go +++ b/coderd/provisionerjobs.go @@ -270,7 +270,7 @@ func (api *API) provisionerJobResources(rw http.ResponseWriter, r *http.Request, } } - apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.DERPMap, *api.TailnetCoordinator.Load(), agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading job agent.", diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 247915db99592..29943c8701ec8 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -48,7 +48,7 @@ func (api *API) workspaceAgent(rw http.ResponseWriter, r *http.Request) { }) return } - apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, workspaceAgent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -77,7 +77,7 @@ func (api *API) workspaceAgentApps(rw http.ResponseWriter, r *http.Request) { func (api *API) workspaceAgentMetadata(rw http.ResponseWriter, r *http.Request) { ctx := r.Context() workspaceAgent := httpmw.WorkspaceAgent(r) - apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -97,7 +97,7 @@ func (api *API) workspaceAgentMetadata(rw http.ResponseWriter, r *http.Request) func (api *API) postWorkspaceAgentVersion(rw http.ResponseWriter, r *http.Request) { ctx := r.Context() workspaceAgent := httpmw.WorkspaceAgent(r) - apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -151,7 +151,7 @@ func (api *API) workspaceAgentPTY(rw http.ResponseWriter, r *http.Request) { httpapi.ResourceNotFound(rw) return } - apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -228,7 +228,7 @@ func (api *API) workspaceAgentListeningPorts(rw http.ResponseWriter, r *http.Req return } - apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.DERPMap, *api.TailnetCoordinator.Load(), workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -322,7 +322,7 @@ func (api *API) dialWorkspaceAgentTailnet(r *http.Request, agentID uuid.UUID) (* }) conn.SetNodeCallback(sendNodes) go func() { - err := api.TailnetCoordinator.ServeClient(serverConn, uuid.New(), agentID) + err := (*api.TailnetCoordinator.Load()).ServeClient(serverConn, uuid.New(), agentID) if err != nil { _ = conn.Close() } @@ -460,7 +460,7 @@ func (api *API) workspaceAgentCoordinate(rw http.ResponseWriter, r *http.Request closeChan := make(chan struct{}) go func() { defer close(closeChan) - err := api.TailnetCoordinator.ServeAgent(wsNetConn, workspaceAgent.ID) + err := (*api.TailnetCoordinator.Load()).ServeAgent(wsNetConn, workspaceAgent.ID) if err != nil { _ = conn.Close(websocket.StatusInternalError, err.Error()) return @@ -529,7 +529,7 @@ func (api *API) workspaceAgentClientCoordinate(rw http.ResponseWriter, r *http.R go httpapi.Heartbeat(ctx, conn) defer conn.Close(websocket.StatusNormalClosure, "") - err = api.TailnetCoordinator.ServeClient(websocket.NetConn(ctx, conn, websocket.MessageBinary), uuid.New(), workspaceAgent.ID) + err = (*api.TailnetCoordinator.Load()).ServeClient(websocket.NetConn(ctx, conn, websocket.MessageBinary), uuid.New(), workspaceAgent.ID) if err != nil { _ = conn.Close(websocket.StatusInternalError, err.Error()) return diff --git a/coderd/workspacebuilds.go b/coderd/workspacebuilds.go index 6ece8d379b153..88e162fa7db94 100644 --- a/coderd/workspacebuilds.go +++ b/coderd/workspacebuilds.go @@ -831,7 +831,7 @@ func (api *API) convertWorkspaceBuild( apiAgents := make([]codersdk.WorkspaceAgent, 0) for _, agent := range agents { apps := appsByAgentID[agent.ID] - apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, agent, convertApps(apps), api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.DERPMap, *api.TailnetCoordinator.Load(), agent, convertApps(apps), api.AgentInactiveDisconnectTimeout) if err != nil { return codersdk.WorkspaceBuild{}, xerrors.Errorf("converting workspace agent: %w", err) } diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index a6595e8bd6554..8eddcf42e325b 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -22,6 +22,8 @@ import ( "github.com/coder/coder/enterprise/audit" "github.com/coder/coder/enterprise/audit/backends" "github.com/coder/coder/enterprise/coderd/license" + "github.com/coder/coder/enterprise/tailnet" + agpltailnet "github.com/coder/coder/tailnet" ) // New constructs an Enterprise coderd API instance. @@ -171,11 +173,27 @@ func (api *API) updateEntitlements(ctx context.Context) error { } if changed, enabled := featureChanged(codersdk.FeatureHighAvailability); changed { - enforcer := workspacequota.NewNop() + coordinator := agpltailnet.NewMemoryCoordinator() if enabled { - enforcer = NewEnforcer(api.Options.UserWorkspaceQuota) + haCoordinator, err := tailnet.NewHACoordinator(api.Logger, api.Pubsub) + if err != nil { + api.Logger.Error(ctx, "unable to setup HA tailnet coordinator", slog.Error(err)) + // If we try to setup the HA coordinator and it fails, nothing + // is actually changing. + changed = false + } else { + coordinator = haCoordinator + } + } + + // Recheck changed in case the HA coordinator failed to set up. + if changed { + oldCoordinator := *api.AGPL.TailnetCoordinator.Swap(&coordinator) + err := oldCoordinator.Close() + if err != nil { + api.Logger.Error(ctx, "unable to setup HA tailnet coordinator", slog.Error(err)) + } } - api.AGPL.WorkspaceQuotaEnforcer.Store(&enforcer) } api.entitlements = entitlements From d38391e9f6ff27351e33017540efcc21f3dcc7d8 Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Fri, 7 Oct 2022 12:45:29 -0500 Subject: [PATCH 07/79] fixup! impelement high availability feature --- enterprise/coderd/coderd.go | 2 +- .../coderd/coderdenttest/coderdenttest.go | 23 ++++++++++--------- enterprise/coderd/license/license_test.go | 9 ++++---- enterprise/coderd/licenses_test.go | 22 ++++++++++-------- 4 files changed, 30 insertions(+), 26 deletions(-) diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 8eddcf42e325b..d52596c547027 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -191,7 +191,7 @@ func (api *API) updateEntitlements(ctx context.Context) error { oldCoordinator := *api.AGPL.TailnetCoordinator.Swap(&coordinator) err := oldCoordinator.Close() if err != nil { - api.Logger.Error(ctx, "unable to setup HA tailnet coordinator", slog.Error(err)) + api.Logger.Error(ctx, "close old tailnet coordinator", slog.Error(err)) } } } diff --git a/enterprise/coderd/coderdenttest/coderdenttest.go b/enterprise/coderd/coderdenttest/coderdenttest.go index 90d09fd5c9c85..a9e08b4aac088 100644 --- a/enterprise/coderd/coderdenttest/coderdenttest.go +++ b/enterprise/coderd/coderdenttest/coderdenttest.go @@ -85,17 +85,18 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c } type LicenseOptions struct { - AccountType string - AccountID string - Trial bool - AllFeatures bool - GraceAt time.Time - ExpiresAt time.Time - UserLimit int64 - AuditLog bool - BrowserOnly bool - SCIM bool - WorkspaceQuota bool + AccountType string + AccountID string + Trial bool + AllFeatures bool + GraceAt time.Time + ExpiresAt time.Time + UserLimit int64 + AuditLog bool + BrowserOnly bool + SCIM bool + WorkspaceQuota bool + HighAvailability bool } // AddLicense generates a new license with the options provided and inserts it. diff --git a/enterprise/coderd/license/license_test.go b/enterprise/coderd/license/license_test.go index 85958fbf4f60d..39d6e05fb50d3 100644 --- a/enterprise/coderd/license/license_test.go +++ b/enterprise/coderd/license/license_test.go @@ -20,10 +20,11 @@ import ( func TestEntitlements(t *testing.T) { t.Parallel() all := map[string]bool{ - codersdk.FeatureAuditLog: true, - codersdk.FeatureBrowserOnly: true, - codersdk.FeatureSCIM: true, - codersdk.FeatureWorkspaceQuota: true, + codersdk.FeatureAuditLog: true, + codersdk.FeatureBrowserOnly: true, + codersdk.FeatureSCIM: true, + codersdk.FeatureWorkspaceQuota: true, + codersdk.FeatureHighAvailability: true, } t.Run("Defaults", func(t *testing.T) { diff --git a/enterprise/coderd/licenses_test.go b/enterprise/coderd/licenses_test.go index 59d36cc9157a6..5b4c89212578d 100644 --- a/enterprise/coderd/licenses_test.go +++ b/enterprise/coderd/licenses_test.go @@ -99,21 +99,23 @@ func TestGetLicense(t *testing.T) { assert.Equal(t, int32(1), licenses[0].ID) assert.Equal(t, "testing", licenses[0].Claims["account_id"]) assert.Equal(t, map[string]interface{}{ - codersdk.FeatureUserLimit: json.Number("0"), - codersdk.FeatureAuditLog: json.Number("1"), - codersdk.FeatureSCIM: json.Number("1"), - codersdk.FeatureBrowserOnly: json.Number("1"), - codersdk.FeatureWorkspaceQuota: json.Number("0"), + codersdk.FeatureUserLimit: json.Number("0"), + codersdk.FeatureAuditLog: json.Number("1"), + codersdk.FeatureSCIM: json.Number("1"), + codersdk.FeatureBrowserOnly: json.Number("1"), + codersdk.FeatureWorkspaceQuota: json.Number("0"), + codersdk.FeatureHighAvailability: json.Number("0"), }, licenses[0].Claims["features"]) assert.Equal(t, int32(2), licenses[1].ID) assert.Equal(t, "testing2", licenses[1].Claims["account_id"]) assert.Equal(t, true, licenses[1].Claims["trial"]) assert.Equal(t, map[string]interface{}{ - codersdk.FeatureUserLimit: json.Number("200"), - codersdk.FeatureAuditLog: json.Number("1"), - codersdk.FeatureSCIM: json.Number("1"), - codersdk.FeatureBrowserOnly: json.Number("1"), - codersdk.FeatureWorkspaceQuota: json.Number("0"), + codersdk.FeatureUserLimit: json.Number("200"), + codersdk.FeatureAuditLog: json.Number("1"), + codersdk.FeatureSCIM: json.Number("1"), + codersdk.FeatureBrowserOnly: json.Number("1"), + codersdk.FeatureWorkspaceQuota: json.Number("0"), + codersdk.FeatureHighAvailability: json.Number("0"), }, licenses[1].Claims["features"]) }) } From a0bcd6464f16483c9524a69137de1dcc7d309095 Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Fri, 7 Oct 2022 12:53:18 -0500 Subject: [PATCH 08/79] fixup! impelement high availability feature --- enterprise/coderd/license/license_test.go | 26 ++++++++++++----------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/enterprise/coderd/license/license_test.go b/enterprise/coderd/license/license_test.go index 39d6e05fb50d3..204c6e7c3f5a2 100644 --- a/enterprise/coderd/license/license_test.go +++ b/enterprise/coderd/license/license_test.go @@ -60,11 +60,12 @@ func TestEntitlements(t *testing.T) { db := databasefake.New() db.InsertLicense(context.Background(), database.InsertLicenseParams{ JWT: coderdenttest.GenerateLicense(t, coderdenttest.LicenseOptions{ - UserLimit: 100, - AuditLog: true, - BrowserOnly: true, - SCIM: true, - WorkspaceQuota: true, + UserLimit: 100, + AuditLog: true, + BrowserOnly: true, + SCIM: true, + WorkspaceQuota: true, + HighAvailability: true, }), Exp: time.Now().Add(time.Hour), }) @@ -81,13 +82,14 @@ func TestEntitlements(t *testing.T) { db := databasefake.New() db.InsertLicense(context.Background(), database.InsertLicenseParams{ JWT: coderdenttest.GenerateLicense(t, coderdenttest.LicenseOptions{ - UserLimit: 100, - AuditLog: true, - BrowserOnly: true, - SCIM: true, - WorkspaceQuota: true, - GraceAt: time.Now().Add(-time.Hour), - ExpiresAt: time.Now().Add(time.Hour), + UserLimit: 100, + AuditLog: true, + BrowserOnly: true, + SCIM: true, + WorkspaceQuota: true, + HighAvailability: true, + GraceAt: time.Now().Add(-time.Hour), + ExpiresAt: time.Now().Add(time.Hour), }), Exp: time.Now().Add(time.Hour), }) From 1f33018bd1c586956c748e65c08e2049fcfdee78 Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Fri, 7 Oct 2022 13:02:40 -0500 Subject: [PATCH 09/79] fixup! impelement high availability feature --- enterprise/coderd/coderdenttest/coderdenttest.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/enterprise/coderd/coderdenttest/coderdenttest.go b/enterprise/coderd/coderdenttest/coderdenttest.go index a9e08b4aac088..2c4250325b567 100644 --- a/enterprise/coderd/coderdenttest/coderdenttest.go +++ b/enterprise/coderd/coderdenttest/coderdenttest.go @@ -132,6 +132,10 @@ func GenerateLicense(t *testing.T, options LicenseOptions) string { if options.WorkspaceQuota { workspaceQuota = 1 } + highAvailability := int64(0) + if options.HighAvailability { + highAvailability = 1 + } c := &license.Claims{ RegisteredClaims: jwt.RegisteredClaims{ @@ -147,11 +151,12 @@ func GenerateLicense(t *testing.T, options LicenseOptions) string { Version: license.CurrentVersion, AllFeatures: options.AllFeatures, Features: license.Features{ - UserLimit: options.UserLimit, - AuditLog: auditLog, - BrowserOnly: browserOnly, - SCIM: scim, - WorkspaceQuota: workspaceQuota, + UserLimit: options.UserLimit, + AuditLog: auditLog, + BrowserOnly: browserOnly, + SCIM: scim, + WorkspaceQuota: workspaceQuota, + HighAvailability: highAvailability, }, } tok := jwt.NewWithClaims(jwt.SigningMethodEdDSA, c) From b6a507020417a5704d7d1336336cb5b961fa42eb Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Fri, 7 Oct 2022 13:11:20 -0500 Subject: [PATCH 10/79] fixup! impelement high availability feature --- enterprise/cli/features_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/enterprise/cli/features_test.go b/enterprise/cli/features_test.go index f5e7b1ff3520a..f892182f164fe 100644 --- a/enterprise/cli/features_test.go +++ b/enterprise/cli/features_test.go @@ -57,7 +57,7 @@ func TestFeaturesList(t *testing.T) { var entitlements codersdk.Entitlements err := json.Unmarshal(buf.Bytes(), &entitlements) require.NoError(t, err, "unmarshal JSON output") - assert.Len(t, entitlements.Features, 5) + assert.Len(t, entitlements.Features, 6) assert.Empty(t, entitlements.Warnings) assert.Equal(t, codersdk.EntitlementNotEntitled, entitlements.Features[codersdk.FeatureUserLimit].Entitlement) @@ -69,6 +69,8 @@ func TestFeaturesList(t *testing.T) { entitlements.Features[codersdk.FeatureWorkspaceQuota].Entitlement) assert.Equal(t, codersdk.EntitlementNotEntitled, entitlements.Features[codersdk.FeatureSCIM].Entitlement) + assert.Equal(t, codersdk.EntitlementNotEntitled, + entitlements.Features[codersdk.FeatureHighAvailability].Entitlement) assert.False(t, entitlements.HasLicense) assert.False(t, entitlements.Experimental) }) From 1883430c952607d28b188ca41c51fee7006e2250 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 12 Oct 2022 04:22:37 +0000 Subject: [PATCH 11/79] Add replicas --- coderd/database/databasefake/databasefake.go | 78 ++++ coderd/database/dump.sql | 16 +- .../migrations/000059_replicas.down.sql | 2 + .../migrations/000059_replicas.up.sql | 26 ++ coderd/database/models.go | 14 + coderd/database/querier.go | 5 + coderd/database/queries.sql.go | 183 ++++++++- coderd/database/queries/replicas.sql | 33 ++ enterprise/replica/replica.go | 371 ++++++++++++++++++ enterprise/replica/replica_test.go | 193 +++++++++ enterprise/tailmesh/tailmesh.go | 32 ++ 11 files changed, 949 insertions(+), 4 deletions(-) create mode 100644 coderd/database/migrations/000059_replicas.down.sql create mode 100644 coderd/database/migrations/000059_replicas.up.sql create mode 100644 coderd/database/queries/replicas.sql create mode 100644 enterprise/replica/replica.go create mode 100644 enterprise/replica/replica_test.go create mode 100644 enterprise/tailmesh/tailmesh.go diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index 1a2a919925ec2..ae41d9c23620b 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -107,6 +107,7 @@ type data struct { workspaceApps []database.WorkspaceApp workspaces []database.Workspace licenses []database.License + replicas []database.Replica deploymentID string lastLicenseID int32 @@ -3025,3 +3026,80 @@ func (q *fakeQuerier) DeleteGroupByID(_ context.Context, id uuid.UUID) error { return sql.ErrNoRows } + +func (q *fakeQuerier) DeleteReplicasUpdatedBefore(ctx context.Context, before time.Time) error { + q.mutex.Lock() + defer q.mutex.Unlock() + + for i, replica := range q.replicas { + if replica.UpdatedAt.Before(before) { + q.replicas = append(q.replicas[:i], q.replicas[i+1:]...) + } + } + + return nil +} + +func (q *fakeQuerier) InsertReplica(_ context.Context, arg database.InsertReplicaParams) (database.Replica, error) { + q.mutex.Lock() + defer q.mutex.Unlock() + + replica := database.Replica{ + ID: arg.ID, + CreatedAt: arg.CreatedAt, + StartedAt: arg.StartedAt, + UpdatedAt: arg.UpdatedAt, + Hostname: arg.Hostname, + RegionID: arg.RegionID, + RelayAddress: arg.RelayAddress, + Version: arg.Version, + } + q.replicas = append(q.replicas, replica) + return replica, nil +} + +func (q *fakeQuerier) UpdateReplica(_ context.Context, arg database.UpdateReplicaParams) (database.Replica, error) { + q.mutex.Lock() + defer q.mutex.Unlock() + + for index, replica := range q.replicas { + if replica.ID != arg.ID { + continue + } + replica.Hostname = arg.Hostname + replica.StartedAt = arg.StartedAt + replica.StoppedAt = arg.StoppedAt + replica.UpdatedAt = arg.UpdatedAt + replica.RelayAddress = arg.RelayAddress + replica.RegionID = arg.RegionID + replica.Version = arg.Version + replica.Error = arg.Error + q.replicas[index] = replica + return replica, nil + } + return database.Replica{}, sql.ErrNoRows +} + +func (q *fakeQuerier) GetReplicasUpdatedAfter(_ context.Context, updatedAt time.Time) ([]database.Replica, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + replicas := make([]database.Replica, 0) + for _, replica := range q.replicas { + if replica.UpdatedAt.After(updatedAt) && !replica.StoppedAt.Valid { + replicas = append(replicas, replica) + } + } + return replicas, nil +} + +func (q *fakeQuerier) GetReplicaByID(_ context.Context, id uuid.UUID) (database.Replica, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + + for _, replica := range q.replicas { + if replica.ID == id { + return replica, nil + } + } + return database.Replica{}, sql.ErrNoRows +} diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index eb16074e90525..4b956fb64f10e 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -245,7 +245,8 @@ CREATE TABLE provisioner_daemons ( created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone, name character varying(64) NOT NULL, - provisioners provisioner_type[] NOT NULL + provisioners provisioner_type[] NOT NULL, + replica_id uuid ); CREATE TABLE provisioner_job_logs ( @@ -276,6 +277,19 @@ CREATE TABLE provisioner_jobs ( worker_id uuid ); +CREATE TABLE replicas ( + id uuid NOT NULL, + created_at timestamp with time zone NOT NULL, + started_at timestamp with time zone NOT NULL, + stopped_at timestamp with time zone, + updated_at timestamp with time zone NOT NULL, + hostname text NOT NULL, + region_id integer NOT NULL, + relay_address text NOT NULL, + version text NOT NULL, + error text +); + CREATE TABLE site_configs ( key character varying(256) NOT NULL, value character varying(8192) NOT NULL diff --git a/coderd/database/migrations/000059_replicas.down.sql b/coderd/database/migrations/000059_replicas.down.sql new file mode 100644 index 0000000000000..4cca6615d4213 --- /dev/null +++ b/coderd/database/migrations/000059_replicas.down.sql @@ -0,0 +1,2 @@ +DROP TABLE replicas; +ALTER TABLE provisioner_daemons DROP COLUMN replica_id; diff --git a/coderd/database/migrations/000059_replicas.up.sql b/coderd/database/migrations/000059_replicas.up.sql new file mode 100644 index 0000000000000..a07587f35a234 --- /dev/null +++ b/coderd/database/migrations/000059_replicas.up.sql @@ -0,0 +1,26 @@ +CREATE TABLE IF NOT EXISTS replicas ( + -- A unique identifier for the replica that is stored on disk. + -- For persistent replicas, this will be reused. + -- For ephemeral replicas, this will be a new UUID for each one. + id uuid NOT NULL, + created_at timestamp with time zone NOT NULL, + -- The time the replica was created. + started_at timestamp with time zone NOT NULL, + -- The time the replica was last seen. + stopped_at timestamp with time zone, + -- Updated periodically to ensure the replica is still alive. + updated_at timestamp with time zone NOT NULL, + -- Hostname is the hostname of the replica. + hostname text NOT NULL, + -- Region is the region the replica is in. + -- We only DERP mesh to the same region ID of a running replica. + region_id integer NOT NULL, + -- An address that should be accessible to other replicas. + relay_address text NOT NULL, + -- Version is the Coder version of the replica. + version text NOT NULL, + error text +); + +-- Associates a provisioner daemon with a replica. +ALTER TABLE provisioner_daemons ADD COLUMN replica_id uuid; diff --git a/coderd/database/models.go b/coderd/database/models.go index f669b5e618138..9d73e097bfe0f 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -487,6 +487,7 @@ type ProvisionerDaemon struct { UpdatedAt sql.NullTime `db:"updated_at" json:"updated_at"` Name string `db:"name" json:"name"` Provisioners []ProvisionerType `db:"provisioners" json:"provisioners"` + ReplicaID uuid.NullUUID `db:"replica_id" json:"replica_id"` } type ProvisionerJob struct { @@ -517,6 +518,19 @@ type ProvisionerJobLog struct { Output string `db:"output" json:"output"` } +type Replica struct { + ID uuid.UUID `db:"id" json:"id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + StartedAt time.Time `db:"started_at" json:"started_at"` + StoppedAt sql.NullTime `db:"stopped_at" json:"stopped_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + Hostname string `db:"hostname" json:"hostname"` + RegionID int32 `db:"region_id" json:"region_id"` + RelayAddress string `db:"relay_address" json:"relay_address"` + Version string `db:"version" json:"version"` + Error sql.NullString `db:"error" json:"error"` +} + type SiteConfig struct { Key string `db:"key" json:"key"` Value string `db:"value" json:"value"` diff --git a/coderd/database/querier.go b/coderd/database/querier.go index b58f6abbccfb8..db789e3399939 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -26,6 +26,7 @@ type sqlcQuerier interface { DeleteLicense(ctx context.Context, id int32) (int32, error) DeleteOldAgentStats(ctx context.Context) error DeleteParameterValueByID(ctx context.Context, id uuid.UUID) error + DeleteReplicasUpdatedBefore(ctx context.Context, updatedAt time.Time) error GetAPIKeyByID(ctx context.Context, id string) (APIKey, error) GetAPIKeysByLoginType(ctx context.Context, loginType LoginType) ([]APIKey, error) GetAPIKeysLastUsedAfter(ctx context.Context, lastUsed time.Time) ([]APIKey, error) @@ -66,6 +67,8 @@ type sqlcQuerier interface { GetProvisionerJobsByIDs(ctx context.Context, ids []uuid.UUID) ([]ProvisionerJob, error) GetProvisionerJobsCreatedAfter(ctx context.Context, createdAt time.Time) ([]ProvisionerJob, error) GetProvisionerLogsByIDBetween(ctx context.Context, arg GetProvisionerLogsByIDBetweenParams) ([]ProvisionerJobLog, error) + GetReplicaByID(ctx context.Context, id uuid.UUID) (Replica, error) + GetReplicasUpdatedAfter(ctx context.Context, updatedAt time.Time) ([]Replica, error) GetTemplateByID(ctx context.Context, id uuid.UUID) (Template, error) GetTemplateByOrganizationAndName(ctx context.Context, arg GetTemplateByOrganizationAndNameParams) (Template, error) GetTemplateDAUs(ctx context.Context, templateID uuid.UUID) ([]GetTemplateDAUsRow, error) @@ -134,6 +137,7 @@ type sqlcQuerier interface { InsertProvisionerDaemon(ctx context.Context, arg InsertProvisionerDaemonParams) (ProvisionerDaemon, error) InsertProvisionerJob(ctx context.Context, arg InsertProvisionerJobParams) (ProvisionerJob, error) InsertProvisionerJobLogs(ctx context.Context, arg InsertProvisionerJobLogsParams) ([]ProvisionerJobLog, error) + InsertReplica(ctx context.Context, arg InsertReplicaParams) (Replica, error) InsertTemplate(ctx context.Context, arg InsertTemplateParams) (Template, error) InsertTemplateVersion(ctx context.Context, arg InsertTemplateVersionParams) (TemplateVersion, error) InsertUser(ctx context.Context, arg InsertUserParams) (User, error) @@ -154,6 +158,7 @@ type sqlcQuerier interface { UpdateProvisionerJobByID(ctx context.Context, arg UpdateProvisionerJobByIDParams) error UpdateProvisionerJobWithCancelByID(ctx context.Context, arg UpdateProvisionerJobWithCancelByIDParams) error UpdateProvisionerJobWithCompleteByID(ctx context.Context, arg UpdateProvisionerJobWithCompleteByIDParams) error + UpdateReplica(ctx context.Context, arg UpdateReplicaParams) (Replica, error) UpdateTemplateActiveVersionByID(ctx context.Context, arg UpdateTemplateActiveVersionByIDParams) error UpdateTemplateDeletedByID(ctx context.Context, arg UpdateTemplateDeletedByIDParams) error UpdateTemplateMetaByID(ctx context.Context, arg UpdateTemplateMetaByIDParams) (Template, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index ba90e102b819a..adfe532446a4e 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -1985,7 +1985,7 @@ func (q *sqlQuerier) ParameterValues(ctx context.Context, arg ParameterValuesPar const getProvisionerDaemonByID = `-- name: GetProvisionerDaemonByID :one SELECT - id, created_at, updated_at, name, provisioners + id, created_at, updated_at, name, provisioners, replica_id FROM provisioner_daemons WHERE @@ -2001,13 +2001,14 @@ func (q *sqlQuerier) GetProvisionerDaemonByID(ctx context.Context, id uuid.UUID) &i.UpdatedAt, &i.Name, pq.Array(&i.Provisioners), + &i.ReplicaID, ) return i, err } const getProvisionerDaemons = `-- name: GetProvisionerDaemons :many SELECT - id, created_at, updated_at, name, provisioners + id, created_at, updated_at, name, provisioners, replica_id FROM provisioner_daemons ` @@ -2027,6 +2028,7 @@ func (q *sqlQuerier) GetProvisionerDaemons(ctx context.Context) ([]ProvisionerDa &i.UpdatedAt, &i.Name, pq.Array(&i.Provisioners), + &i.ReplicaID, ); err != nil { return nil, err } @@ -2050,7 +2052,7 @@ INSERT INTO provisioners ) VALUES - ($1, $2, $3, $4) RETURNING id, created_at, updated_at, name, provisioners + ($1, $2, $3, $4) RETURNING id, created_at, updated_at, name, provisioners, replica_id ` type InsertProvisionerDaemonParams struct { @@ -2074,6 +2076,7 @@ func (q *sqlQuerier) InsertProvisionerDaemon(ctx context.Context, arg InsertProv &i.UpdatedAt, &i.Name, pq.Array(&i.Provisioners), + &i.ReplicaID, ) return i, err } @@ -2531,6 +2534,180 @@ func (q *sqlQuerier) UpdateProvisionerJobWithCompleteByID(ctx context.Context, a return err } +const deleteReplicasUpdatedBefore = `-- name: DeleteReplicasUpdatedBefore :exec +DELETE FROM replicas WHERE updated_at < $1 +` + +func (q *sqlQuerier) DeleteReplicasUpdatedBefore(ctx context.Context, updatedAt time.Time) error { + _, err := q.db.ExecContext(ctx, deleteReplicasUpdatedBefore, updatedAt) + return err +} + +const getReplicaByID = `-- name: GetReplicaByID :one +SELECT id, created_at, started_at, stopped_at, updated_at, hostname, region_id, relay_address, version, error FROM replicas WHERE id = $1 +` + +func (q *sqlQuerier) GetReplicaByID(ctx context.Context, id uuid.UUID) (Replica, error) { + row := q.db.QueryRowContext(ctx, getReplicaByID, id) + var i Replica + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.StartedAt, + &i.StoppedAt, + &i.UpdatedAt, + &i.Hostname, + &i.RegionID, + &i.RelayAddress, + &i.Version, + &i.Error, + ) + return i, err +} + +const getReplicasUpdatedAfter = `-- name: GetReplicasUpdatedAfter :many +SELECT id, created_at, started_at, stopped_at, updated_at, hostname, region_id, relay_address, version, error FROM replicas WHERE updated_at > $1 AND stopped_at IS NULL +` + +func (q *sqlQuerier) GetReplicasUpdatedAfter(ctx context.Context, updatedAt time.Time) ([]Replica, error) { + rows, err := q.db.QueryContext(ctx, getReplicasUpdatedAfter, updatedAt) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Replica + for rows.Next() { + var i Replica + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.StartedAt, + &i.StoppedAt, + &i.UpdatedAt, + &i.Hostname, + &i.RegionID, + &i.RelayAddress, + &i.Version, + &i.Error, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const insertReplica = `-- name: InsertReplica :one +INSERT INTO replicas ( + id, + created_at, + started_at, + updated_at, + hostname, + region_id, + relay_address, + version + +) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id, created_at, started_at, stopped_at, updated_at, hostname, region_id, relay_address, version, error +` + +type InsertReplicaParams struct { + ID uuid.UUID `db:"id" json:"id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + StartedAt time.Time `db:"started_at" json:"started_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + Hostname string `db:"hostname" json:"hostname"` + RegionID int32 `db:"region_id" json:"region_id"` + RelayAddress string `db:"relay_address" json:"relay_address"` + Version string `db:"version" json:"version"` +} + +func (q *sqlQuerier) InsertReplica(ctx context.Context, arg InsertReplicaParams) (Replica, error) { + row := q.db.QueryRowContext(ctx, insertReplica, + arg.ID, + arg.CreatedAt, + arg.StartedAt, + arg.UpdatedAt, + arg.Hostname, + arg.RegionID, + arg.RelayAddress, + arg.Version, + ) + var i Replica + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.StartedAt, + &i.StoppedAt, + &i.UpdatedAt, + &i.Hostname, + &i.RegionID, + &i.RelayAddress, + &i.Version, + &i.Error, + ) + return i, err +} + +const updateReplica = `-- name: UpdateReplica :one +UPDATE replicas SET + updated_at = $2, + started_at = $3, + stopped_at = $4, + relay_address = $5, + region_id = $6, + hostname = $7, + version = $8, + error = $9 +WHERE id = $1 RETURNING id, created_at, started_at, stopped_at, updated_at, hostname, region_id, relay_address, version, error +` + +type UpdateReplicaParams struct { + ID uuid.UUID `db:"id" json:"id"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + StartedAt time.Time `db:"started_at" json:"started_at"` + StoppedAt sql.NullTime `db:"stopped_at" json:"stopped_at"` + RelayAddress string `db:"relay_address" json:"relay_address"` + RegionID int32 `db:"region_id" json:"region_id"` + Hostname string `db:"hostname" json:"hostname"` + Version string `db:"version" json:"version"` + Error sql.NullString `db:"error" json:"error"` +} + +func (q *sqlQuerier) UpdateReplica(ctx context.Context, arg UpdateReplicaParams) (Replica, error) { + row := q.db.QueryRowContext(ctx, updateReplica, + arg.ID, + arg.UpdatedAt, + arg.StartedAt, + arg.StoppedAt, + arg.RelayAddress, + arg.RegionID, + arg.Hostname, + arg.Version, + arg.Error, + ) + var i Replica + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.StartedAt, + &i.StoppedAt, + &i.UpdatedAt, + &i.Hostname, + &i.RegionID, + &i.RelayAddress, + &i.Version, + &i.Error, + ) + return i, err +} + const getDeploymentID = `-- name: GetDeploymentID :one SELECT value FROM site_configs WHERE key = 'deployment_id' ` diff --git a/coderd/database/queries/replicas.sql b/coderd/database/queries/replicas.sql new file mode 100644 index 0000000000000..a7aa5b0aa1dee --- /dev/null +++ b/coderd/database/queries/replicas.sql @@ -0,0 +1,33 @@ +-- name: GetReplicasUpdatedAfter :many +SELECT * FROM replicas WHERE updated_at > $1 AND stopped_at IS NULL; + +-- name: GetReplicaByID :one +SELECT * FROM replicas WHERE id = $1; + +-- name: InsertReplica :one +INSERT INTO replicas ( + id, + created_at, + started_at, + updated_at, + hostname, + region_id, + relay_address, + version + +) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *; + +-- name: UpdateReplica :one +UPDATE replicas SET + updated_at = $2, + started_at = $3, + stopped_at = $4, + relay_address = $5, + region_id = $6, + hostname = $7, + version = $8, + error = $9 +WHERE id = $1 RETURNING *; + +-- name: DeleteReplicasUpdatedBefore :exec +DELETE FROM replicas WHERE updated_at < $1; diff --git a/enterprise/replica/replica.go b/enterprise/replica/replica.go new file mode 100644 index 0000000000000..ca0c450651e64 --- /dev/null +++ b/enterprise/replica/replica.go @@ -0,0 +1,371 @@ +package replica + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/http" + "os" + "strings" + "sync" + "time" + + "github.com/google/uuid" + "golang.org/x/xerrors" + + "cdr.dev/slog" + + "github.com/coder/coder/buildinfo" + "github.com/coder/coder/coderd/database" +) + +var ( + PubsubEvent = "replica" +) + +type Options struct { + ID uuid.UUID + UpdateInterval time.Duration + PeerTimeout time.Duration + // Mesh will dial active replicas with the same region ID to ensure + // they are reachable. If not, an error will be updated on the replica. + Mesh bool + RelayAddress string + RegionID int32 +} + +// New registers the replica with the database and periodically updates to ensure +// it's healthy. It contacts all other alive replicas to ensure they are reachable. +func New(ctx context.Context, logger slog.Logger, db database.Store, pubsub database.Pubsub, options Options) (*Server, error) { + if options.ID == uuid.Nil { + panic("An ID must be provided!") + } + if options.PeerTimeout == 0 { + options.PeerTimeout = 3 * time.Second + } + if options.UpdateInterval == 0 { + options.UpdateInterval = 5 * time.Second + } + hostname, err := os.Hostname() + if err != nil { + return nil, xerrors.Errorf("get hostname: %w", err) + } + var replica database.Replica + _, err = db.GetReplicaByID(ctx, options.ID) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + return nil, xerrors.Errorf("get replica: %w", err) + } + replica, err = db.InsertReplica(ctx, database.InsertReplicaParams{ + ID: options.ID, + CreatedAt: database.Now(), + StartedAt: database.Now(), + UpdatedAt: database.Now(), + Hostname: hostname, + RegionID: options.RegionID, + RelayAddress: options.RelayAddress, + Version: buildinfo.Version(), + }) + if err != nil { + return nil, xerrors.Errorf("insert replica: %w", err) + } + } else { + replica, err = db.UpdateReplica(ctx, database.UpdateReplicaParams{ + ID: options.ID, + UpdatedAt: database.Now(), + StartedAt: database.Now(), + StoppedAt: sql.NullTime{}, + RelayAddress: options.RelayAddress, + RegionID: options.RegionID, + Hostname: hostname, + Version: buildinfo.Version(), + Error: sql.NullString{}, + }) + if err != nil { + return nil, xerrors.Errorf("update replica: %w", err) + } + } + err = pubsub.Publish(PubsubEvent, []byte(options.ID.String())) + if err != nil { + return nil, xerrors.Errorf("publish new replica: %w", err) + } + ctx, cancelFunc := context.WithCancel(ctx) + server := &Server{ + options: &options, + db: db, + pubsub: pubsub, + self: replica, + logger: logger, + closed: make(chan struct{}), + closeCancel: cancelFunc, + } + err = server.run(ctx) + if err != nil { + return nil, xerrors.Errorf("run replica: %w", err) + } + err = server.subscribe(ctx) + if err != nil { + return nil, xerrors.Errorf("subscribe: %w", err) + } + server.closeWait.Add(1) + go server.loop(ctx) + return server, nil +} + +type Server struct { + options *Options + db database.Store + pubsub database.Pubsub + logger slog.Logger + + closeWait sync.WaitGroup + closeMutex sync.Mutex + closed chan (struct{}) + closeCancel context.CancelFunc + + self database.Replica + mutex sync.Mutex + peers []database.Replica + callback func() +} + +// loop runs the replica update sequence on an update interval. +func (s *Server) loop(ctx context.Context) { + defer s.closeWait.Done() + ticker := time.NewTicker(s.options.UpdateInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + err := s.run(ctx) + if err != nil && !errors.Is(err, context.Canceled) { + s.logger.Warn(ctx, "run replica update loop", slog.Error(err)) + } + } +} + +// subscribe listens for new replica information! +func (s *Server) subscribe(ctx context.Context) error { + needsUpdate := false + updating := false + updateMutex := sync.Mutex{} + + // This loop will continually update nodes as updates are processed. + // The intent is to always be up to date without spamming the run + // function, so if a new update comes in while one is being processed, + // it will reprocess afterwards. + var update func() + update = func() { + err := s.run(ctx) + if err != nil && !errors.Is(err, context.Canceled) { + s.logger.Error(ctx, "run replica from subscribe", slog.Error(err)) + } + updateMutex.Lock() + if needsUpdate { + needsUpdate = false + updateMutex.Unlock() + update() + return + } + updating = false + updateMutex.Unlock() + } + cancelFunc, err := s.pubsub.Subscribe(PubsubEvent, func(ctx context.Context, message []byte) { + updateMutex.Lock() + defer updateMutex.Unlock() + id, err := uuid.Parse(string(message)) + if err != nil { + return + } + // Don't process updates for ourself! + if id == s.options.ID { + return + } + if updating { + needsUpdate = true + return + } + updating = true + go update() + }) + if err != nil { + return err + } + go func() { + <-ctx.Done() + cancelFunc() + }() + return nil +} + +func (s *Server) run(ctx context.Context) error { + s.closeMutex.Lock() + s.closeWait.Add(1) + s.closeMutex.Unlock() + go func() { + s.closeWait.Done() + }() + // Expect replicas to update once every three times the interval... + // If they don't, assume death! + replicas, err := s.db.GetReplicasUpdatedAfter(ctx, database.Now().Add(-3*s.options.UpdateInterval)) + if err != nil { + return xerrors.Errorf("get replicas: %w", err) + } + + s.mutex.Lock() + s.peers = make([]database.Replica, 0, len(replicas)) + for _, replica := range replicas { + if replica.ID == s.options.ID { + continue + } + s.peers = append(s.peers, replica) + } + s.mutex.Unlock() + + var wg sync.WaitGroup + var mu sync.Mutex + failed := make([]string, 0) + for _, peer := range s.Regional() { + wg.Add(1) + peer := peer + go func() { + defer wg.Done() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, peer.RelayAddress, nil) + if err != nil { + s.logger.Error(ctx, "create http request for relay probe", + slog.F("relay_address", peer.RelayAddress), slog.Error(err)) + return + } + client := http.Client{ + Timeout: s.options.PeerTimeout, + } + res, err := client.Do(req) + if err != nil { + mu.Lock() + failed = append(failed, fmt.Sprintf("relay %s (%s): %s", peer.Hostname, peer.RelayAddress, err)) + mu.Unlock() + return + } + _ = res.Body.Close() + }() + } + wg.Wait() + replicaError := sql.NullString{} + if len(failed) > 0 { + replicaError = sql.NullString{ + Valid: true, + String: fmt.Sprintf("Failed to dial peers: %s", strings.Join(failed, ", ")), + } + } + + replica, err := s.db.UpdateReplica(ctx, database.UpdateReplicaParams{ + ID: s.self.ID, + UpdatedAt: database.Now(), + StartedAt: s.self.StartedAt, + StoppedAt: s.self.StoppedAt, + RelayAddress: s.self.RelayAddress, + RegionID: s.self.RegionID, + Hostname: s.self.Hostname, + Version: s.self.Version, + Error: replicaError, + }) + if err != nil { + return xerrors.Errorf("update replica: %w", err) + } + s.mutex.Lock() + if s.self.Error.String != replica.Error.String { + // Publish an update occurred! + err = s.pubsub.Publish(PubsubEvent, []byte(s.self.ID.String())) + if err != nil { + s.mutex.Unlock() + return xerrors.Errorf("publish replica update: %w", err) + } + } + s.self = replica + if s.callback != nil { + go s.callback() + } + s.mutex.Unlock() + return nil +} + +// Self represents the current replica. +func (s *Server) Self() database.Replica { + s.mutex.Lock() + defer s.mutex.Unlock() + return s.self +} + +// All returns every replica, including itself. +func (s *Server) All() []database.Replica { + s.mutex.Lock() + defer s.mutex.Unlock() + return append(s.peers, s.self) +} + +// Regional returns all replicas in the same region excluding itself. +func (s *Server) Regional() []database.Replica { + s.mutex.Lock() + defer s.mutex.Unlock() + replicas := make([]database.Replica, 0) + for _, replica := range s.peers { + if replica.RegionID != s.self.RegionID { + continue + } + replicas = append(replicas, replica) + } + return replicas +} + +// SetCallback sets a function to execute whenever new peers +// are refreshed or updated. +func (s *Server) SetCallback(callback func()) { + s.mutex.Lock() + defer s.mutex.Unlock() + s.callback = callback + // Instantly call the callback to inform replicas! + go callback() +} + +func (s *Server) Close() error { + s.closeMutex.Lock() + select { + case <-s.closed: + s.closeMutex.Unlock() + return nil + default: + } + close(s.closed) + s.closeCancel() + s.closeWait.Wait() + s.closeMutex.Unlock() + + ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelFunc() + _, err := s.db.UpdateReplica(ctx, database.UpdateReplicaParams{ + ID: s.self.ID, + UpdatedAt: database.Now(), + StartedAt: s.self.StartedAt, + StoppedAt: sql.NullTime{ + Time: database.Now(), + Valid: true, + }, + RelayAddress: s.self.RelayAddress, + RegionID: s.self.RegionID, + Hostname: s.self.Hostname, + Version: s.self.Version, + Error: s.self.Error, + }) + if err != nil { + return xerrors.Errorf("update replica: %w", err) + } + err = s.pubsub.Publish(PubsubEvent, []byte(s.self.ID.String())) + if err != nil { + return xerrors.Errorf("publish replica update: %w", err) + } + return nil +} diff --git a/enterprise/replica/replica_test.go b/enterprise/replica/replica_test.go new file mode 100644 index 0000000000000..74efb3b40470e --- /dev/null +++ b/enterprise/replica/replica_test.go @@ -0,0 +1,193 @@ +package replica_test + +import ( + "context" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/goleak" + + "cdr.dev/slog/sloggers/slogtest" + "github.com/coder/coder/coderd/database" + "github.com/coder/coder/coderd/database/dbtestutil" + "github.com/coder/coder/enterprise/replica" + "github.com/coder/coder/testutil" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} + +func TestReplica(t *testing.T) { + t.Parallel() + t.Run("CreateOnNew", func(t *testing.T) { + // This ensures that a new replica is created on New. + t.Parallel() + db, pubsub := dbtestutil.NewDB(t) + id := uuid.New() + cancel, err := pubsub.Subscribe(replica.PubsubEvent, func(ctx context.Context, message []byte) { + assert.Equal(t, []byte(id.String()), message) + }) + require.NoError(t, err) + defer cancel() + server, err := replica.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replica.Options{ + ID: id, + }) + require.NoError(t, err) + _ = server.Close() + require.NoError(t, err) + }) + t.Run("UpdatesOnNew", func(t *testing.T) { + // This ensures that a replica is updated when it initially connects + // and immediately publishes it's existence! + t.Parallel() + db, pubsub := dbtestutil.NewDB(t) + id := uuid.New() + _, err := db.InsertReplica(context.Background(), database.InsertReplicaParams{ + ID: id, + }) + require.NoError(t, err) + cancel, err := pubsub.Subscribe(replica.PubsubEvent, func(ctx context.Context, message []byte) { + assert.Equal(t, []byte(id.String()), message) + }) + require.NoError(t, err) + defer cancel() + server, err := replica.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replica.Options{ + ID: id, + }) + require.NoError(t, err) + _ = server.Close() + require.NoError(t, err) + }) + t.Run("ConnectsToPeerReplica", func(t *testing.T) { + // Ensures that the replica reports a successful status for + // accessing all of its peers. + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + db, pubsub := dbtestutil.NewDB(t) + peer, err := db.InsertReplica(context.Background(), database.InsertReplicaParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + StartedAt: database.Now(), + UpdatedAt: database.Now(), + Hostname: "something", + RelayAddress: srv.URL, + }) + require.NoError(t, err) + server, err := replica.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replica.Options{ + ID: uuid.New(), + }) + require.NoError(t, err) + require.Len(t, server.Regional(), 1) + require.Equal(t, peer.ID, server.Regional()[0].ID) + require.False(t, server.Self().Error.Valid) + _ = server.Close() + }) + t.Run("ConnectsToFakePeerWithError", func(t *testing.T) { + t.Parallel() + db, pubsub := dbtestutil.NewDB(t) + var count atomic.Int32 + cancel, err := pubsub.Subscribe(replica.PubsubEvent, func(ctx context.Context, message []byte) { + count.Add(1) + }) + require.NoError(t, err) + defer cancel() + peer, err := db.InsertReplica(context.Background(), database.InsertReplicaParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + StartedAt: database.Now(), + UpdatedAt: database.Now(), + Hostname: "something", + // Fake address to hit! + RelayAddress: "http://169.254.169.254", + }) + require.NoError(t, err) + server, err := replica.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replica.Options{ + ID: uuid.New(), + PeerTimeout: 1 * time.Millisecond, + }) + require.NoError(t, err) + require.Len(t, server.Regional(), 1) + require.Equal(t, peer.ID, server.Regional()[0].ID) + require.True(t, server.Self().Error.Valid) + require.Contains(t, server.Self().Error.String, "Failed to dial peers") + // Once for the initial creation of a replica, and another time for the error. + require.Equal(t, int32(2), count.Load()) + _ = server.Close() + }) + t.Run("RefreshOnPublish", func(t *testing.T) { + // Refresh when a new replica appears! + t.Parallel() + db, pubsub := dbtestutil.NewDB(t) + id := uuid.New() + server, err := replica.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replica.Options{ + ID: id, + }) + require.NoError(t, err) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + peer, err := db.InsertReplica(context.Background(), database.InsertReplicaParams{ + ID: uuid.New(), + RelayAddress: srv.URL, + UpdatedAt: database.Now(), + }) + require.NoError(t, err) + // Publish multiple times to ensure it can handle that case. + err = pubsub.Publish(replica.PubsubEvent, []byte(peer.ID.String())) + require.NoError(t, err) + err = pubsub.Publish(replica.PubsubEvent, []byte(peer.ID.String())) + require.NoError(t, err) + require.Eventually(t, func() bool { + return len(server.Regional()) == 1 + }, testutil.WaitShort, testutil.IntervalFast) + _ = server.Close() + }) + t.Run("TwentyConcurrent", func(t *testing.T) { + // Ensures that twenty concurrent replicas can spawn and all + // discover each other in parallel! + t.Parallel() + db, pubsub := dbtestutil.NewDB(t) + logger := slogtest.Make(t, nil) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + var wg sync.WaitGroup + count := 20 + wg.Add(count) + for i := 0; i < count; i++ { + server, err := replica.New(context.Background(), logger, db, pubsub, replica.Options{ + ID: uuid.New(), + RelayAddress: srv.URL, + }) + require.NoError(t, err) + t.Cleanup(func() { + _ = server.Close() + }) + done := false + server.SetCallback(func() { + if len(server.All()) != count { + return + } + if done { + return + } + done = true + wg.Done() + }) + } + wg.Wait() + }) +} diff --git a/enterprise/tailmesh/tailmesh.go b/enterprise/tailmesh/tailmesh.go new file mode 100644 index 0000000000000..46e1c97fffcc9 --- /dev/null +++ b/enterprise/tailmesh/tailmesh.go @@ -0,0 +1,32 @@ +package tailmesh + +import ( + "context" + + "cdr.dev/slog" + "github.com/coder/coder/tailnet" + "tailscale.com/derp" + "tailscale.com/derp/derphttp" +) + +func New(logger slog.Logger, server *derp.Server) *Mesh { + +} + +type Mesh struct { + logger slog.Logger + server *derp.Server + ctx context.Context + + active map[string]context.CancelFunc +} + +func (m *Mesh) SetAddresses(addresses []string) { + for _, address := range addresses { + client, err := derphttp.NewClient(m.server.PrivateKey(), address, tailnet.Logger(m.logger)) + if err != nil { + + } + go client.RunWatchConnectionLoop() + } +} From 7dc968c52313e1dbe485ea5b0b2d9f8edc01c0b3 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 12 Oct 2022 17:53:00 +0000 Subject: [PATCH 12/79] Add DERP meshing to arbitrary addresses --- enterprise/derpmesh/derpmesh.go | 124 +++++++++++++++++++++++ enterprise/derpmesh/derpmesh_test.go | 146 +++++++++++++++++++++++++++ enterprise/tailmesh/tailmesh.go | 32 ------ 3 files changed, 270 insertions(+), 32 deletions(-) create mode 100644 enterprise/derpmesh/derpmesh.go create mode 100644 enterprise/derpmesh/derpmesh_test.go delete mode 100644 enterprise/tailmesh/tailmesh.go diff --git a/enterprise/derpmesh/derpmesh.go b/enterprise/derpmesh/derpmesh.go new file mode 100644 index 0000000000000..610fd749132cc --- /dev/null +++ b/enterprise/derpmesh/derpmesh.go @@ -0,0 +1,124 @@ +package derpmesh + +import ( + "context" + "sync" + + "golang.org/x/xerrors" + "tailscale.com/derp" + "tailscale.com/derp/derphttp" + "tailscale.com/types/key" + + "github.com/coder/coder/tailnet" + + "cdr.dev/slog" +) + +func New(logger slog.Logger, server *derp.Server) *Mesh { + return &Mesh{ + logger: logger, + server: server, + ctx: context.Background(), + closed: make(chan struct{}), + active: make(map[string]context.CancelFunc), + } +} + +type Mesh struct { + logger slog.Logger + server *derp.Server + ctx context.Context + + mutex sync.Mutex + closed chan struct{} + active map[string]context.CancelFunc +} + +// SetAddresses performs a diff of the incoming addresses and adds +// or removes DERP clients from the mesh. +func (m *Mesh) SetAddresses(addresses []string) { + total := make(map[string]struct{}, 0) + for _, address := range addresses { + total[address] = struct{}{} + added, err := m.addAddress(address) + if err != nil { + m.logger.Error(m.ctx, "failed to add address", slog.F("address", address), slog.Error(err)) + continue + } + if added { + m.logger.Debug(m.ctx, "added mesh address", slog.F("address", address)) + } + } + + m.mutex.Lock() + for address := range m.active { + _, found := total[address] + if found { + continue + } + removed := m.removeAddress(address) + if removed { + m.logger.Debug(m.ctx, "removed mesh address", slog.F("address", address)) + } + } + m.mutex.Unlock() +} + +// addAddress begins meshing with a new address. +// It's expected that this is a full HTTP address with a path. +// e.g. http://127.0.0.1:8080/derp +func (m *Mesh) addAddress(address string) (bool, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + _, isActive := m.active[address] + if isActive { + return false, nil + } + client, err := derphttp.NewClient(m.server.PrivateKey(), address, tailnet.Logger(m.logger)) + if err != nil { + return false, xerrors.Errorf("create derp client: %w", err) + } + client.MeshKey = m.server.MeshKey() + ctx, cancelFunc := context.WithCancel(m.ctx) + closed := make(chan struct{}) + closeFunc := func() { + cancelFunc() + _ = client.Close() + <-closed + } + m.active[address] = closeFunc + go func() { + defer close(closed) + client.RunWatchConnectionLoop(ctx, m.server.PublicKey(), tailnet.Logger(m.logger), func(np key.NodePublic) { + m.server.AddPacketForwarder(np, client) + }, func(np key.NodePublic) { + m.server.RemovePacketForwarder(np, client) + }) + }() + return true, nil +} + +// removeAddress stops meshing with a given address. +func (m *Mesh) removeAddress(address string) bool { + cancelFunc, isActive := m.active[address] + if isActive { + cancelFunc() + } + return isActive +} + +// Close ends all active meshes with the DERP server. +func (m *Mesh) Close() error { + m.mutex.Lock() + defer m.mutex.Unlock() + select { + case <-m.closed: + return nil + default: + } + close(m.closed) + for _, cancelFunc := range m.active { + cancelFunc() + } + return nil +} diff --git a/enterprise/derpmesh/derpmesh_test.go b/enterprise/derpmesh/derpmesh_test.go new file mode 100644 index 0000000000000..313c33da99bad --- /dev/null +++ b/enterprise/derpmesh/derpmesh_test.go @@ -0,0 +1,146 @@ +package derpmesh_test + +import ( + "context" + "errors" + "io" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/goleak" + "tailscale.com/derp" + "tailscale.com/derp/derphttp" + "tailscale.com/types/key" + + "cdr.dev/slog" + "cdr.dev/slog/sloggers/slogtest" + "github.com/coder/coder/enterprise/derpmesh" + "github.com/coder/coder/tailnet" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} + +func TestDERPMesh(t *testing.T) { + t.Parallel() + t.Run("ExchangeMessages", func(t *testing.T) { + // This tests messages passing through multiple DERP servers. + t.Parallel() + firstServer, firstServerURL := startDERP(t) + defer firstServer.Close() + secondServer, secondServerURL := startDERP(t) + firstMesh := derpmesh.New(slogtest.Make(t, nil).Named("first").Leveled(slog.LevelDebug), firstServer) + firstMesh.SetAddresses([]string{secondServerURL}) + secondMesh := derpmesh.New(slogtest.Make(t, nil).Named("second").Leveled(slog.LevelDebug), secondServer) + secondMesh.SetAddresses([]string{firstServerURL}) + defer firstMesh.Close() + defer secondMesh.Close() + + first := key.NewNode() + second := key.NewNode() + firstClient, err := derphttp.NewClient(first, secondServerURL, tailnet.Logger(slogtest.Make(t, nil))) + require.NoError(t, err) + secondClient, err := derphttp.NewClient(second, firstServerURL, tailnet.Logger(slogtest.Make(t, nil))) + require.NoError(t, err) + err = secondClient.Connect(context.Background()) + require.NoError(t, err) + + sent := []byte("hello world") + err = firstClient.Send(second.Public(), sent) + require.NoError(t, err) + + got := recvData(t, secondClient) + require.Equal(t, sent, got) + }) + t.Run("RemoveAddress", func(t *testing.T) { + // This tests messages passing through multiple DERP servers. + t.Parallel() + server, serverURL := startDERP(t) + mesh := derpmesh.New(slogtest.Make(t, nil).Named("first").Leveled(slog.LevelDebug), server) + mesh.SetAddresses([]string{"http://fake.com"}) + // This should trigger a removal... + mesh.SetAddresses([]string{}) + defer mesh.Close() + + first := key.NewNode() + second := key.NewNode() + firstClient, err := derphttp.NewClient(first, serverURL, tailnet.Logger(slogtest.Make(t, nil))) + require.NoError(t, err) + secondClient, err := derphttp.NewClient(second, serverURL, tailnet.Logger(slogtest.Make(t, nil))) + require.NoError(t, err) + err = secondClient.Connect(context.Background()) + require.NoError(t, err) + sent := []byte("hello world") + err = firstClient.Send(second.Public(), sent) + require.NoError(t, err) + got := recvData(t, secondClient) + require.Equal(t, sent, got) + }) + t.Run("TwentyMeshes", func(t *testing.T) { + t.Parallel() + meshes := make([]*derpmesh.Mesh, 0, 20) + serverURLs := make([]string, 0, 20) + for i := 0; i < 20; i++ { + server, url := startDERP(t) + mesh := derpmesh.New(slogtest.Make(t, nil).Named("mesh").Leveled(slog.LevelDebug), server) + t.Cleanup(func() { + _ = server.Close() + _ = mesh.Close() + }) + serverURLs = append(serverURLs, url) + meshes = append(meshes, mesh) + } + for _, mesh := range meshes { + mesh.SetAddresses(serverURLs) + } + + first := key.NewNode() + second := key.NewNode() + firstClient, err := derphttp.NewClient(first, serverURLs[9], tailnet.Logger(slogtest.Make(t, nil))) + require.NoError(t, err) + secondClient, err := derphttp.NewClient(second, serverURLs[16], tailnet.Logger(slogtest.Make(t, nil))) + require.NoError(t, err) + err = secondClient.Connect(context.Background()) + require.NoError(t, err) + + sent := []byte("hello world") + err = firstClient.Send(second.Public(), sent) + require.NoError(t, err) + + got := recvData(t, secondClient) + require.Equal(t, sent, got) + }) +} + +func recvData(t *testing.T, client *derphttp.Client) []byte { + for { + msg, err := client.Recv() + if errors.Is(err, io.EOF) { + return nil + } + assert.NoError(t, err) + t.Logf("derp: %T", msg) + switch msg := msg.(type) { + case derp.ReceivedPacket: + return msg.Data + default: + // Drop all others! + } + } +} + +func startDERP(t *testing.T) (*derp.Server, string) { + logf := tailnet.Logger(slogtest.Make(t, nil)) + d := derp.NewServer(key.NewNode(), logf) + d.SetMeshKey("some-key") + server := httptest.NewUnstartedServer(derphttp.Handler(d)) + server.Start() + t.Cleanup(func() { + _ = d.Close() + }) + t.Cleanup(server.Close) + return d, server.URL +} diff --git a/enterprise/tailmesh/tailmesh.go b/enterprise/tailmesh/tailmesh.go deleted file mode 100644 index 46e1c97fffcc9..0000000000000 --- a/enterprise/tailmesh/tailmesh.go +++ /dev/null @@ -1,32 +0,0 @@ -package tailmesh - -import ( - "context" - - "cdr.dev/slog" - "github.com/coder/coder/tailnet" - "tailscale.com/derp" - "tailscale.com/derp/derphttp" -) - -func New(logger slog.Logger, server *derp.Server) *Mesh { - -} - -type Mesh struct { - logger slog.Logger - server *derp.Server - ctx context.Context - - active map[string]context.CancelFunc -} - -func (m *Mesh) SetAddresses(addresses []string) { - for _, address := range addresses { - client, err := derphttp.NewClient(m.server.PrivateKey(), address, tailnet.Logger(m.logger)) - if err != nil { - - } - go client.RunWatchConnectionLoop() - } -} From 1dcf0d01899a2544cbf0e3a1134b0a0c6d0e4bce Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 12 Oct 2022 17:55:22 +0000 Subject: [PATCH 13/79] Move packages to highavailability folder --- enterprise/{ => highavailability}/derpmesh/derpmesh.go | 0 enterprise/{ => highavailability}/derpmesh/derpmesh_test.go | 2 +- enterprise/{ => highavailability}/replica/replica.go | 0 enterprise/{ => highavailability}/replica/replica_test.go | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) rename enterprise/{ => highavailability}/derpmesh/derpmesh.go (100%) rename enterprise/{ => highavailability}/derpmesh/derpmesh_test.go (98%) rename enterprise/{ => highavailability}/replica/replica.go (100%) rename enterprise/{ => highavailability}/replica/replica_test.go (98%) diff --git a/enterprise/derpmesh/derpmesh.go b/enterprise/highavailability/derpmesh/derpmesh.go similarity index 100% rename from enterprise/derpmesh/derpmesh.go rename to enterprise/highavailability/derpmesh/derpmesh.go diff --git a/enterprise/derpmesh/derpmesh_test.go b/enterprise/highavailability/derpmesh/derpmesh_test.go similarity index 98% rename from enterprise/derpmesh/derpmesh_test.go rename to enterprise/highavailability/derpmesh/derpmesh_test.go index 313c33da99bad..6e1154fc3d6a8 100644 --- a/enterprise/derpmesh/derpmesh_test.go +++ b/enterprise/highavailability/derpmesh/derpmesh_test.go @@ -16,7 +16,7 @@ import ( "cdr.dev/slog" "cdr.dev/slog/sloggers/slogtest" - "github.com/coder/coder/enterprise/derpmesh" + "github.com/coder/coder/enterprise/highavailability/derpmesh" "github.com/coder/coder/tailnet" ) diff --git a/enterprise/replica/replica.go b/enterprise/highavailability/replica/replica.go similarity index 100% rename from enterprise/replica/replica.go rename to enterprise/highavailability/replica/replica.go diff --git a/enterprise/replica/replica_test.go b/enterprise/highavailability/replica/replica_test.go similarity index 98% rename from enterprise/replica/replica_test.go rename to enterprise/highavailability/replica/replica_test.go index 74efb3b40470e..a5bda874ea166 100644 --- a/enterprise/replica/replica_test.go +++ b/enterprise/highavailability/replica/replica_test.go @@ -17,7 +17,7 @@ import ( "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/coderd/database" "github.com/coder/coder/coderd/database/dbtestutil" - "github.com/coder/coder/enterprise/replica" + "github.com/coder/coder/enterprise/highavailability/replica" "github.com/coder/coder/testutil" ) From 289e13913a8586f2e160d56406e5e2a3e4d47254 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 12 Oct 2022 18:03:51 +0000 Subject: [PATCH 14/79] Move coordinator to high availability package --- agent/agent_test.go | 2 +- coderd/coderd.go | 2 +- coderd/wsconncache/wsconncache_test.go | 2 +- enterprise/coderd/coderd.go | 8 +++---- .../coordinator.go | 6 +++-- .../coordinator_test.go | 14 +++++------ tailnet/coordinator.go | 23 ++++++++++--------- tailnet/coordinator_test.go | 6 ++--- 8 files changed, 33 insertions(+), 30 deletions(-) rename enterprise/{tailnet => highavailability}/coordinator.go (98%) rename enterprise/{tailnet => highavailability}/coordinator_test.go (92%) diff --git a/agent/agent_test.go b/agent/agent_test.go index 38d70846dfd8b..06a33598b755f 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -560,7 +560,7 @@ func setupAgent(t *testing.T, metadata codersdk.WorkspaceAgentMetadata, ptyTimeo if metadata.DERPMap == nil { metadata.DERPMap = tailnettest.RunDERPAndSTUN(t) } - coordinator := tailnet.NewMemoryCoordinator() + coordinator := tailnet.NewCoordinator() agentID := uuid.New() statsCh := make(chan *codersdk.AgentStats) closer := agent.New(agent.Options{ diff --git a/coderd/coderd.go b/coderd/coderd.go index d6dba70b9f80b..4976b3a58cde2 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -119,7 +119,7 @@ func New(options *Options) *API { options.PrometheusRegistry = prometheus.NewRegistry() } if options.TailnetCoordinator == nil { - options.TailnetCoordinator = tailnet.NewMemoryCoordinator() + options.TailnetCoordinator = tailnet.NewCoordinator() } if options.Auditor == nil { options.Auditor = audit.NewNop() diff --git a/coderd/wsconncache/wsconncache_test.go b/coderd/wsconncache/wsconncache_test.go index 2b5ed06b45784..003d3cddb8b7a 100644 --- a/coderd/wsconncache/wsconncache_test.go +++ b/coderd/wsconncache/wsconncache_test.go @@ -143,7 +143,7 @@ func TestCache(t *testing.T) { func setupAgent(t *testing.T, metadata codersdk.WorkspaceAgentMetadata, ptyTimeout time.Duration) *codersdk.AgentConn { metadata.DERPMap = tailnettest.RunDERPAndSTUN(t) - coordinator := tailnet.NewMemoryCoordinator() + coordinator := tailnet.NewCoordinator() agentID := uuid.New() closer := agent.New(agent.Options{ FetchMetadata: func(ctx context.Context) (codersdk.WorkspaceAgentMetadata, error) { diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index d49a6cd2c8a9d..dfa9c25e4cf77 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -23,8 +23,8 @@ import ( "github.com/coder/coder/enterprise/audit" "github.com/coder/coder/enterprise/audit/backends" "github.com/coder/coder/enterprise/coderd/license" - "github.com/coder/coder/enterprise/tailnet" - agpltailnet "github.com/coder/coder/tailnet" + "github.com/coder/coder/enterprise/highavailability" + "github.com/coder/coder/tailnet" ) // New constructs an Enterprise coderd API instance. @@ -206,9 +206,9 @@ func (api *API) updateEntitlements(ctx context.Context) error { } if changed, enabled := featureChanged(codersdk.FeatureHighAvailability); changed { - coordinator := agpltailnet.NewMemoryCoordinator() + coordinator := tailnet.NewCoordinator() if enabled { - haCoordinator, err := tailnet.NewHACoordinator(api.Logger, api.Pubsub) + haCoordinator, err := highavailability.NewCoordinator(api.Logger, api.Pubsub) if err != nil { api.Logger.Error(ctx, "unable to setup HA tailnet coordinator", slog.Error(err)) // If we try to setup the HA coordinator and it fails, nothing diff --git a/enterprise/tailnet/coordinator.go b/enterprise/highavailability/coordinator.go similarity index 98% rename from enterprise/tailnet/coordinator.go rename to enterprise/highavailability/coordinator.go index 6bf2327507165..7c41e47d44f1d 100644 --- a/enterprise/tailnet/coordinator.go +++ b/enterprise/highavailability/coordinator.go @@ -1,4 +1,4 @@ -package tailnet +package highavailability import ( "bytes" @@ -18,7 +18,9 @@ import ( agpl "github.com/coder/coder/tailnet" ) -func NewHACoordinator(logger slog.Logger, pubsub database.Pubsub) (agpl.Coordinator, error) { +// NewCoordinator creates a new high availability coordinator +// that uses PostgreSQL pubsub to exchange handshakes. +func NewCoordinator(logger slog.Logger, pubsub database.Pubsub) (agpl.Coordinator, error) { coord := &haCoordinator{ id: uuid.New(), log: logger, diff --git a/enterprise/tailnet/coordinator_test.go b/enterprise/highavailability/coordinator_test.go similarity index 92% rename from enterprise/tailnet/coordinator_test.go rename to enterprise/highavailability/coordinator_test.go index 4889cd1c8ba60..1e86c08f1b1ed 100644 --- a/enterprise/tailnet/coordinator_test.go +++ b/enterprise/highavailability/coordinator_test.go @@ -1,4 +1,4 @@ -package tailnet_test +package highavailability_test import ( "net" @@ -11,7 +11,7 @@ import ( "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/coderd/database" - "github.com/coder/coder/enterprise/tailnet" + "github.com/coder/coder/enterprise/highavailability" agpl "github.com/coder/coder/tailnet" "github.com/coder/coder/testutil" ) @@ -20,7 +20,7 @@ func TestCoordinatorSingle(t *testing.T) { t.Parallel() t.Run("ClientWithoutAgent", func(t *testing.T) { t.Parallel() - coordinator, err := tailnet.NewHACoordinator(slogtest.Make(t, nil), database.NewPubsubInMemory()) + coordinator, err := highavailability.NewCoordinator(slogtest.Make(t, nil), database.NewPubsubInMemory()) require.NoError(t, err) defer coordinator.Close() @@ -48,7 +48,7 @@ func TestCoordinatorSingle(t *testing.T) { t.Run("AgentWithoutClients", func(t *testing.T) { t.Parallel() - coordinator, err := tailnet.NewHACoordinator(slogtest.Make(t, nil), database.NewPubsubInMemory()) + coordinator, err := highavailability.NewCoordinator(slogtest.Make(t, nil), database.NewPubsubInMemory()) require.NoError(t, err) defer coordinator.Close() @@ -76,7 +76,7 @@ func TestCoordinatorSingle(t *testing.T) { t.Run("AgentWithClient", func(t *testing.T) { t.Parallel() - coordinator, err := tailnet.NewHACoordinator(slogtest.Make(t, nil), database.NewPubsubInMemory()) + coordinator, err := highavailability.NewCoordinator(slogtest.Make(t, nil), database.NewPubsubInMemory()) require.NoError(t, err) defer coordinator.Close() @@ -169,11 +169,11 @@ func TestCoordinatorHA(t *testing.T) { pubsub := database.NewPubsubInMemory() - coordinator1, err := tailnet.NewHACoordinator(slogtest.Make(t, nil), pubsub) + coordinator1, err := highavailability.NewCoordinator(slogtest.Make(t, nil), pubsub) require.NoError(t, err) defer coordinator1.Close() - coordinator2, err := tailnet.NewHACoordinator(slogtest.Make(t, nil), pubsub) + coordinator2, err := highavailability.NewCoordinator(slogtest.Make(t, nil), pubsub) require.NoError(t, err) defer coordinator2.Close() diff --git a/tailnet/coordinator.go b/tailnet/coordinator.go index 150a323bcfe52..96de8d295162e 100644 --- a/tailnet/coordinator.go +++ b/tailnet/coordinator.go @@ -94,11 +94,11 @@ func ServeCoordinator(conn net.Conn, updateNodes func(node []*Node) error) (func }, errChan } -// NewMemoryCoordinator constructs a new in-memory connection coordinator. This +// NewCoordinator constructs a new in-memory connection coordinator. This // coordinator is incompatible with multiple Coder replicas as all node data is // in-memory. -func NewMemoryCoordinator() Coordinator { - return &memoryCoordinator{ +func NewCoordinator() Coordinator { + return &coordinator{ closed: false, nodes: map[uuid.UUID]*Node{}, agentSockets: map[uuid.UUID]net.Conn{}, @@ -106,13 +106,14 @@ func NewMemoryCoordinator() Coordinator { } } -// MemoryCoordinator exchanges nodes with agents to establish connections. +// coordinator exchanges nodes with agents to establish connections entirely in-memory. +// The Enterprise implementation provides this for high-availability. // ┌──────────────────┐ ┌────────────────────┐ ┌───────────────────┐ ┌──────────────────┐ // │tailnet.Coordinate├──►│tailnet.AcceptClient│◄─►│tailnet.AcceptAgent│◄──┤tailnet.Coordinate│ // └──────────────────┘ └────────────────────┘ └───────────────────┘ └──────────────────┘ // This coordinator is incompatible with multiple Coder // replicas as all node data is in-memory. -type memoryCoordinator struct { +type coordinator struct { mutex sync.Mutex closed bool @@ -126,7 +127,7 @@ type memoryCoordinator struct { } // Node returns an in-memory node by ID. -func (c *memoryCoordinator) Node(id uuid.UUID) *Node { +func (c *coordinator) Node(id uuid.UUID) *Node { c.mutex.Lock() defer c.mutex.Unlock() node := c.nodes[id] @@ -135,7 +136,7 @@ func (c *memoryCoordinator) Node(id uuid.UUID) *Node { // ServeClient accepts a WebSocket connection that wants to connect to an agent // with the specified ID. -func (c *memoryCoordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID) error { +func (c *coordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID) error { c.mutex.Lock() if c.closed { @@ -194,7 +195,7 @@ func (c *memoryCoordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid. } } -func (c *memoryCoordinator) handleNextClientMessage(id, agent uuid.UUID, decoder *json.Decoder) error { +func (c *coordinator) handleNextClientMessage(id, agent uuid.UUID, decoder *json.Decoder) error { var node Node err := decoder.Decode(&node) if err != nil { @@ -234,7 +235,7 @@ func (c *memoryCoordinator) handleNextClientMessage(id, agent uuid.UUID, decoder // ServeAgent accepts a WebSocket connection to an agent that // listens to incoming connections and publishes node updates. -func (c *memoryCoordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { +func (c *coordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { c.mutex.Lock() if c.closed { @@ -293,7 +294,7 @@ func (c *memoryCoordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { } } -func (c *memoryCoordinator) handleNextAgentMessage(id uuid.UUID, decoder *json.Decoder) error { +func (c *coordinator) handleNextAgentMessage(id uuid.UUID, decoder *json.Decoder) error { var node Node err := decoder.Decode(&node) if err != nil { @@ -334,7 +335,7 @@ func (c *memoryCoordinator) handleNextAgentMessage(id uuid.UUID, decoder *json.D // Close closes all of the open connections in the coordinator and stops the // coordinator from accepting new connections. -func (c *memoryCoordinator) Close() error { +func (c *coordinator) Close() error { c.mutex.Lock() defer c.mutex.Unlock() diff --git a/tailnet/coordinator_test.go b/tailnet/coordinator_test.go index e0ed44420ede2..a4a020deadf93 100644 --- a/tailnet/coordinator_test.go +++ b/tailnet/coordinator_test.go @@ -16,7 +16,7 @@ func TestCoordinator(t *testing.T) { t.Parallel() t.Run("ClientWithoutAgent", func(t *testing.T) { t.Parallel() - coordinator := tailnet.NewMemoryCoordinator() + coordinator := tailnet.NewCoordinator() client, server := net.Pipe() sendNode, errChan := tailnet.ServeCoordinator(client, func(node []*tailnet.Node) error { return nil @@ -40,7 +40,7 @@ func TestCoordinator(t *testing.T) { t.Run("AgentWithoutClients", func(t *testing.T) { t.Parallel() - coordinator := tailnet.NewMemoryCoordinator() + coordinator := tailnet.NewCoordinator() client, server := net.Pipe() sendNode, errChan := tailnet.ServeCoordinator(client, func(node []*tailnet.Node) error { return nil @@ -64,7 +64,7 @@ func TestCoordinator(t *testing.T) { t.Run("AgentWithClient", func(t *testing.T) { t.Parallel() - coordinator := tailnet.NewMemoryCoordinator() + coordinator := tailnet.NewCoordinator() agentWS, agentServerWS := net.Pipe() defer agentWS.Close() From 585bc1dfc81996b9967475de7f4249c93b24aa46 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 12 Oct 2022 22:36:05 +0000 Subject: [PATCH 15/79] Add flags for HA --- cli/config/file.go | 5 ++ cli/deployment/flags.go | 15 +++++ cli/root.go | 2 +- cli/server.go | 4 +- coderd/coderd.go | 8 ++- coderd/coderdtest/coderdtest.go | 17 +++-- codersdk/flags.go | 2 + codersdk/replicas.go | 22 +++++++ enterprise/cli/server.go | 25 +++++++- enterprise/coderd/coderd.go | 64 ++++++++++++++++--- .../coderd/coderdenttest/coderdenttest.go | 7 +- enterprise/coderd/replicas.go | 1 + enterprise/coderd/replicas_test.go | 38 +++++++++++ .../highavailability/derpmesh/derpmesh.go | 1 + .../highavailability/replica/replica.go | 7 +- 15 files changed, 190 insertions(+), 28 deletions(-) create mode 100644 codersdk/replicas.go create mode 100644 enterprise/coderd/replicas.go create mode 100644 enterprise/coderd/replicas_test.go diff --git a/cli/config/file.go b/cli/config/file.go index a98237afed22b..388ce0881f304 100644 --- a/cli/config/file.go +++ b/cli/config/file.go @@ -13,6 +13,11 @@ func (r Root) Session() File { return File(filepath.Join(string(r), "session")) } +// ReplicaID is a unique identifier for the Coder server. +func (r Root) ReplicaID() File { + return File(filepath.Join(string(r), "replica_id")) +} + func (r Root) URL() File { return File(filepath.Join(string(r), "url")) } diff --git a/cli/deployment/flags.go b/cli/deployment/flags.go index 3a03bea762b1c..35ae248a0a722 100644 --- a/cli/deployment/flags.go +++ b/cli/deployment/flags.go @@ -85,6 +85,13 @@ func Flags() *codersdk.DeploymentFlags { Description: "Addresses for STUN servers to establish P2P connections. Set empty to disable P2P connections.", Default: []string{"stun.l.google.com:19302"}, }, + DerpServerRelayAddress: &codersdk.StringFlag{ + Name: "DERP Server Relay Address", + Flag: "derp-server-relay-address", + EnvVar: "CODER_DERP_SERVER_RELAY_ADDRESS", + Description: "An HTTP address that is accessible by other replicas to relay DERP traffic. Required for high availability.", + Enterprise: true, + }, DerpConfigURL: &codersdk.StringFlag{ Name: "DERP Config URL", Flag: "derp-config-url", @@ -123,6 +130,14 @@ func Flags() *codersdk.DeploymentFlags { Description: "The bind address to serve pprof.", Default: "127.0.0.1:6060", }, + HighAvailability: &codersdk.BoolFlag{ + Name: "High Availability", + Flag: "high-availability", + EnvVar: "CODER_HIGH_AVAILABILITY", + Description: "Specifies whether high availability is enabled.", + Default: true, + Enterprise: true, + }, CacheDir: &codersdk.StringFlag{ Name: "Cache Directory", Flag: "cache-dir", diff --git a/cli/root.go b/cli/root.go index e7104e64284eb..e29aa534da0a8 100644 --- a/cli/root.go +++ b/cli/root.go @@ -100,7 +100,7 @@ func Core() []*cobra.Command { } func AGPL() []*cobra.Command { - all := append(Core(), Server(deployment.Flags(), func(_ context.Context, o *coderd.Options) (*coderd.API, error) { + all := append(Core(), Server(deployment.Flags(), func(_ context.Context, _ config.Root, o *coderd.Options) (*coderd.API, error) { return coderd.New(o), nil })) return all diff --git a/cli/server.go b/cli/server.go index e3cad09ca27ff..fc5f131da3d7b 100644 --- a/cli/server.go +++ b/cli/server.go @@ -67,7 +67,7 @@ import ( ) // nolint:gocyclo -func Server(dflags *codersdk.DeploymentFlags, newAPI func(context.Context, *coderd.Options) (*coderd.API, error)) *cobra.Command { +func Server(dflags *codersdk.DeploymentFlags, newAPI func(context.Context, config.Root, *coderd.Options) (*coderd.API, error)) *cobra.Command { root := &cobra.Command{ Use: "server", Short: "Start a Coder server", @@ -463,7 +463,7 @@ func Server(dflags *codersdk.DeploymentFlags, newAPI func(context.Context, *code ), dflags.PromAddress.Value, "prometheus")() } - coderAPI, err := newAPI(ctx, options) + coderAPI, err := newAPI(ctx, config, options) if err != nil { return err } diff --git a/coderd/coderd.go b/coderd/coderd.go index 4976b3a58cde2..57b78520d1b50 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -77,6 +77,7 @@ type Options struct { AutoImportTemplates []AutoImportTemplate TailnetCoordinator tailnet.Coordinator + DERPServer *derp.Server DERPMap *tailcfg.DERPMap MetricsCacheRefreshInterval time.Duration @@ -121,6 +122,9 @@ func New(options *Options) *API { if options.TailnetCoordinator == nil { options.TailnetCoordinator = tailnet.NewCoordinator() } + if options.DERPServer == nil { + options.DERPServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger)) + } if options.Auditor == nil { options.Auditor = audit.NewNop() } @@ -160,7 +164,6 @@ func New(options *Options) *API { api.WorkspaceQuotaEnforcer.Store(&options.WorkspaceQuotaEnforcer) api.workspaceAgentCache = wsconncache.New(api.dialWorkspaceAgentTailnet, 0) api.TailnetCoordinator.Store(&options.TailnetCoordinator) - api.derpServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger)) oauthConfigs := &httpmw.OAuth2Configs{ Github: options.GithubOAuth2Config, OIDC: options.OIDCConfig, @@ -228,7 +231,7 @@ func New(options *Options) *API { r.Route("/%40{user}/{workspace_and_agent}/apps/{workspaceapp}", apps) r.Route("/@{user}/{workspace_and_agent}/apps/{workspaceapp}", apps) r.Route("/derp", func(r chi.Router) { - r.Get("/", derphttp.Handler(api.derpServer).ServeHTTP) + r.Get("/", derphttp.Handler(api.DERPServer).ServeHTTP) // This is used when UDP is blocked, and latency must be checked via HTTP(s). r.Get("/latency-check", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) @@ -540,7 +543,6 @@ type API struct { // RootHandler serves "/" RootHandler chi.Router - derpServer *derp.Server metricsCache *metricscache.Cache siteHandler http.Handler websocketWaitMutex sync.Mutex diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index d7ac4eb14be97..23305cbcbab36 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -81,6 +81,12 @@ type Options struct { MetricsCacheRefreshInterval time.Duration AgentStatsRefreshInterval time.Duration DeploymentFlags *codersdk.DeploymentFlags + + // Overriding the database is heavily discouraged. + // It should only be used in cases where multiple Coder + // test instances are running against the same database. + Database database.Store + Pubsub database.Pubsub } // New constructs a codersdk client connected to an in-memory API instance. @@ -135,13 +141,14 @@ func NewOptions(t *testing.T, options *Options) (*httptest.Server, context.Cance close(options.AutobuildStats) }) } - - db, pubsub := dbtestutil.NewDB(t) + if options.Database == nil { + options.Database, options.Pubsub = dbtestutil.NewDB(t) + } ctx, cancelFunc := context.WithCancel(context.Background()) lifecycleExecutor := executor.New( ctx, - db, + options.Database, slogtest.Make(t, nil).Named("autobuild.executor").Leveled(slog.LevelDebug), options.AutobuildTicker, ).WithStatsChannel(options.AutobuildStats) @@ -181,8 +188,8 @@ func NewOptions(t *testing.T, options *Options) (*httptest.Server, context.Cance AppHostname: options.AppHostname, Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug), CacheDir: t.TempDir(), - Database: db, - Pubsub: pubsub, + Database: options.Database, + Pubsub: options.Pubsub, Auditor: options.Auditor, AWSCertificates: options.AWSCertificates, diff --git a/codersdk/flags.go b/codersdk/flags.go index 92f02941a57f8..2dd1323a1fddc 100644 --- a/codersdk/flags.go +++ b/codersdk/flags.go @@ -19,6 +19,7 @@ type DeploymentFlags struct { DerpServerRegionCode *StringFlag `json:"derp_server_region_code" typescript:",notnull"` DerpServerRegionName *StringFlag `json:"derp_server_region_name" typescript:",notnull"` DerpServerSTUNAddresses *StringArrayFlag `json:"derp_server_stun_address" typescript:",notnull"` + DerpServerRelayAddress *StringFlag `json:"derp_server_relay_address" typescript:",notnull"` DerpConfigURL *StringFlag `json:"derp_config_url" typescript:",notnull"` DerpConfigPath *StringFlag `json:"derp_config_path" typescript:",notnull"` PromEnabled *BoolFlag `json:"prom_enabled" typescript:",notnull"` @@ -59,6 +60,7 @@ type DeploymentFlags struct { Verbose *BoolFlag `json:"verbose" typescript:",notnull"` AuditLogging *BoolFlag `json:"audit_logging" typescript:",notnull"` BrowserOnly *BoolFlag `json:"browser_only" typescript:",notnull"` + HighAvailability *BoolFlag `json:"high_availability" typescript:",notnull"` SCIMAuthHeader *StringFlag `json:"scim_auth_header" typescript:",notnull"` UserWorkspaceQuota *IntFlag `json:"user_workspace_quota" typescript:",notnull"` } diff --git a/codersdk/replicas.go b/codersdk/replicas.go new file mode 100644 index 0000000000000..341b460792ddd --- /dev/null +++ b/codersdk/replicas.go @@ -0,0 +1,22 @@ +package codersdk + +import ( + "time" + + "github.com/google/uuid" +) + +type Replica struct { + // ID is the unique identifier for the replica. + ID uuid.UUID `json:"id"` + // Hostname is the hostname of the replica. + Hostname string `json:"hostname"` + // CreatedAt is when the replica was first seen. + CreatedAt time.Time `json:"created_at"` + // Active determines whether the replica is online. + Active bool `json:"active"` + // RelayAddress is the accessible address to relay DERP connections. + RelayAddress string `json:"relay_address"` + // Error is the error. + Error string `json:"error"` +} diff --git a/enterprise/cli/server.go b/enterprise/cli/server.go index 62af6f2888373..e34bdaccfd342 100644 --- a/enterprise/cli/server.go +++ b/enterprise/cli/server.go @@ -3,8 +3,12 @@ package cli import ( "context" + "github.com/google/uuid" "github.com/spf13/cobra" + "cdr.dev/slog" + + "github.com/coder/coder/cli/config" "github.com/coder/coder/cli/deployment" "github.com/coder/coder/enterprise/coderd" @@ -14,14 +18,29 @@ import ( func server() *cobra.Command { dflags := deployment.Flags() - cmd := agpl.Server(dflags, func(ctx context.Context, options *agplcoderd.Options) (*agplcoderd.API, error) { + cmd := agpl.Server(dflags, func(ctx context.Context, cfg config.Root, options *agplcoderd.Options) (*agplcoderd.API, error) { + replicaIDRaw, err := cfg.ReplicaID().Read() + if err != nil { + replicaIDRaw = uuid.NewString() + } + replicaID, err := uuid.Parse(replicaIDRaw) + if err != nil { + options.Logger.Warn(ctx, "failed to parse replica id", slog.Error(err), slog.F("replica_id", replicaIDRaw)) + replicaID = uuid.New() + } o := &coderd.Options{ AuditLogging: dflags.AuditLogging.Value, BrowserOnly: dflags.BrowserOnly.Value, SCIMAPIKey: []byte(dflags.SCIMAuthHeader.Value), UserWorkspaceQuota: dflags.UserWorkspaceQuota.Value, - RBACEnabled: true, - Options: options, + RBAC: true, + HighAvailability: dflags.HighAvailability.Value, + + ReplicaID: replicaID, + DERPServerRelayAddress: dflags.DerpServerRelayAddress.Value, + DERPServerRegionID: dflags.DerpServerRegionID.Value, + + Options: options, } api, err := coderd.New(ctx, o) if err != nil { diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index dfa9c25e4cf77..f18776ade2c61 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -11,6 +11,7 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/go-chi/chi/v5" + "github.com/google/uuid" "cdr.dev/slog" "github.com/coder/coder/coderd" @@ -24,6 +25,8 @@ import ( "github.com/coder/coder/enterprise/audit/backends" "github.com/coder/coder/enterprise/coderd/license" "github.com/coder/coder/enterprise/highavailability" + "github.com/coder/coder/enterprise/highavailability/derpmesh" + "github.com/coder/coder/enterprise/highavailability/replica" "github.com/coder/coder/tailnet" ) @@ -43,6 +46,7 @@ func New(ctx context.Context, options *Options) (*API, error) { Options: options, cancelEntitlementsLoop: cancelFunc, } + oauthConfigs := &httpmw.OAuth2Configs{ Github: options.GithubOAuth2Config, OIDC: options.OIDCConfig, @@ -113,7 +117,27 @@ func New(ctx context.Context, options *Options) (*API, error) { }) } - err := api.updateEntitlements(ctx) + // If high availability is disabled and multiple replicas appear, show an error. + // If high availability is enabled and the built-in DERP is but the DERP relay isn't set, show an error. + // We need to block meshing if high availability is disabled, because the meshing code would just work. + // SetAddresses([]string{}) + + api.AGPL.RootHandler.Route("/replicas", func(r chi.Router) { + + }) + + var err error + api.replica, err = replica.New(ctx, options.Logger, options.Database, options.Pubsub, replica.Options{ + ID: options.ReplicaID, + RelayAddress: options.DERPServerRelayAddress, + RegionID: int32(options.DERPServerRegionID), + }) + if err != nil { + return nil, xerrors.Errorf("initialize replica: %w", err) + } + api.derpMesh = derpmesh.New(options.Logger, api.DERPServer) + + err = api.updateEntitlements(ctx) if err != nil { return nil, xerrors.Errorf("update entitlements: %w", err) } @@ -125,12 +149,18 @@ func New(ctx context.Context, options *Options) (*API, error) { type Options struct { *coderd.Options - RBACEnabled bool + RBAC bool AuditLogging bool // Whether to block non-browser connections. BrowserOnly bool SCIMAPIKey []byte UserWorkspaceQuota int + HighAvailability bool + + // Used for high availability. + DERPServerRelayAddress string + DERPServerRegionID int + ReplicaID uuid.UUID EntitlementsUpdateInterval time.Duration Keys map[string]ed25519.PublicKey @@ -140,6 +170,11 @@ type API struct { AGPL *coderd.API *Options + // Detects multiple Coder replicas running at the same time. + replica *replica.Server + // Meshes DERP connections from multiple replicas. + derpMesh *derpmesh.Mesh + cancelEntitlementsLoop func() entitlementsMu sync.RWMutex entitlements codersdk.Entitlements @@ -147,6 +182,8 @@ type API struct { func (api *API) Close() error { api.cancelEntitlementsLoop() + _ = api.replica.Close() + _ = api.derpMesh.Close() return api.AGPL.Close() } @@ -155,11 +192,12 @@ func (api *API) updateEntitlements(ctx context.Context) error { defer api.entitlementsMu.Unlock() entitlements, err := license.Entitlements(ctx, api.Database, api.Logger, api.Keys, map[string]bool{ - codersdk.FeatureAuditLog: api.AuditLogging, - codersdk.FeatureBrowserOnly: api.BrowserOnly, - codersdk.FeatureSCIM: len(api.SCIMAPIKey) != 0, - codersdk.FeatureWorkspaceQuota: api.UserWorkspaceQuota != 0, - codersdk.FeatureTemplateRBAC: api.RBACEnabled, + codersdk.FeatureAuditLog: api.AuditLogging, + codersdk.FeatureBrowserOnly: api.BrowserOnly, + codersdk.FeatureSCIM: len(api.SCIMAPIKey) != 0, + codersdk.FeatureWorkspaceQuota: api.UserWorkspaceQuota != 0, + codersdk.FeatureHighAvailability: api.HighAvailability, + codersdk.FeatureTemplateRBAC: api.RBAC, }) if err != nil { return err @@ -210,13 +248,23 @@ func (api *API) updateEntitlements(ctx context.Context) error { if enabled { haCoordinator, err := highavailability.NewCoordinator(api.Logger, api.Pubsub) if err != nil { - api.Logger.Error(ctx, "unable to setup HA tailnet coordinator", slog.Error(err)) + api.Logger.Error(ctx, "unable to set up high availability coordinator", slog.Error(err)) // If we try to setup the HA coordinator and it fails, nothing // is actually changing. changed = false } else { coordinator = haCoordinator } + + api.replica.SetCallback(func() { + addresses := make([]string, 0) + for _, replica := range api.replica.Regional() { + addresses = append(addresses, replica.RelayAddress) + } + api.derpMesh.SetAddresses(addresses) + }) + } else { + api.derpMesh.SetAddresses([]string{}) } // Recheck changed in case the HA coordinator failed to set up. diff --git a/enterprise/coderd/coderdenttest/coderdenttest.go b/enterprise/coderd/coderdenttest/coderdenttest.go index bc6b0375df638..c5ec2391d97bf 100644 --- a/enterprise/coderd/coderdenttest/coderdenttest.go +++ b/enterprise/coderd/coderdenttest/coderdenttest.go @@ -9,6 +9,7 @@ import ( "time" "github.com/golang-jwt/jwt/v4" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -62,10 +63,14 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c } srv, cancelFunc, oop := coderdtest.NewOptions(t, options.Options) coderAPI, err := coderd.New(context.Background(), &coderd.Options{ - RBACEnabled: true, + RBAC: true, AuditLogging: options.AuditLogging, BrowserOnly: options.BrowserOnly, SCIMAPIKey: options.SCIMAPIKey, + DERPServerRelayAddress: oop.AccessURL.String(), + DERPServerRegionID: 1, + HighAvailability: true, + ReplicaID: uuid.New(), UserWorkspaceQuota: options.UserWorkspaceQuota, Options: oop, EntitlementsUpdateInterval: options.EntitlementsUpdateInterval, diff --git a/enterprise/coderd/replicas.go b/enterprise/coderd/replicas.go new file mode 100644 index 0000000000000..ddb2b8b672186 --- /dev/null +++ b/enterprise/coderd/replicas.go @@ -0,0 +1 @@ +package coderd diff --git a/enterprise/coderd/replicas_test.go b/enterprise/coderd/replicas_test.go new file mode 100644 index 0000000000000..1a5a3ed5f4eee --- /dev/null +++ b/enterprise/coderd/replicas_test.go @@ -0,0 +1,38 @@ +package coderd_test + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/coder/coder/coderd/coderdtest" + "github.com/coder/coder/coderd/database/dbtestutil" + "github.com/coder/coder/codersdk" + "github.com/coder/coder/enterprise/coderd/coderdenttest" +) + +func TestReplicas(t *testing.T) { + t.Parallel() + db, pubsub := dbtestutil.NewDB(t) + firstClient := coderdenttest.New(t, &coderdenttest.Options{ + Options: &coderdtest.Options{ + Database: db, + Pubsub: pubsub, + }, + }) + _ = coderdtest.CreateFirstUser(t, firstClient) + + secondClient := coderdenttest.New(t, &coderdenttest.Options{ + Options: &coderdtest.Options{ + Database: db, + Pubsub: pubsub, + }, + }) + secondClient.SessionToken = firstClient.SessionToken + + user, err := secondClient.User(context.Background(), codersdk.Me) + require.NoError(t, err) + fmt.Printf("%+v\n", user) +} diff --git a/enterprise/highavailability/derpmesh/derpmesh.go b/enterprise/highavailability/derpmesh/derpmesh.go index 610fd749132cc..94341079cd43f 100644 --- a/enterprise/highavailability/derpmesh/derpmesh.go +++ b/enterprise/highavailability/derpmesh/derpmesh.go @@ -14,6 +14,7 @@ import ( "cdr.dev/slog" ) +// New constructs a new mesh for DERP servers. func New(logger slog.Logger, server *derp.Server) *Mesh { return &Mesh{ logger: logger, diff --git a/enterprise/highavailability/replica/replica.go b/enterprise/highavailability/replica/replica.go index ca0c450651e64..6855b32852e3e 100644 --- a/enterprise/highavailability/replica/replica.go +++ b/enterprise/highavailability/replica/replica.go @@ -28,11 +28,8 @@ type Options struct { ID uuid.UUID UpdateInterval time.Duration PeerTimeout time.Duration - // Mesh will dial active replicas with the same region ID to ensure - // they are reachable. If not, an error will be updated on the replica. - Mesh bool - RelayAddress string - RegionID int32 + RelayAddress string + RegionID int32 } // New registers the replica with the database and periodically updates to ensure From fdb3557f7fe4599c77ccd06689712cc6856a6319 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 13 Oct 2022 00:35:43 +0000 Subject: [PATCH 16/79] Rename to replicasync --- .vscode/settings.json | 1 + enterprise/coderd/coderd.go | 12 ++++----- .../replica.go => replicasync/replicasync.go} | 25 +++++++++--------- .../replicasync_test.go} | 26 +++++++++---------- 4 files changed, 33 insertions(+), 31 deletions(-) rename enterprise/highavailability/{replica/replica.go => replicasync/replicasync.go} (93%) rename enterprise/highavailability/{replica/replica_test.go => replicasync/replicasync_test.go} (80%) diff --git a/.vscode/settings.json b/.vscode/settings.json index e9a32e850c980..f556563596bc0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -85,6 +85,7 @@ "ptytest", "quickstart", "reconfig", + "replicasync", "retrier", "rpty", "sdkproto", diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index f18776ade2c61..19812ea1f8b42 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -26,7 +26,7 @@ import ( "github.com/coder/coder/enterprise/coderd/license" "github.com/coder/coder/enterprise/highavailability" "github.com/coder/coder/enterprise/highavailability/derpmesh" - "github.com/coder/coder/enterprise/highavailability/replica" + "github.com/coder/coder/enterprise/highavailability/replicasync" "github.com/coder/coder/tailnet" ) @@ -127,7 +127,7 @@ func New(ctx context.Context, options *Options) (*API, error) { }) var err error - api.replica, err = replica.New(ctx, options.Logger, options.Database, options.Pubsub, replica.Options{ + api.replicaManager, err = replicasync.New(ctx, options.Logger, options.Database, options.Pubsub, replicasync.Options{ ID: options.ReplicaID, RelayAddress: options.DERPServerRelayAddress, RegionID: int32(options.DERPServerRegionID), @@ -171,7 +171,7 @@ type API struct { *Options // Detects multiple Coder replicas running at the same time. - replica *replica.Server + replicaManager *replicasync.Manager // Meshes DERP connections from multiple replicas. derpMesh *derpmesh.Mesh @@ -182,7 +182,7 @@ type API struct { func (api *API) Close() error { api.cancelEntitlementsLoop() - _ = api.replica.Close() + _ = api.replicaManager.Close() _ = api.derpMesh.Close() return api.AGPL.Close() } @@ -256,9 +256,9 @@ func (api *API) updateEntitlements(ctx context.Context) error { coordinator = haCoordinator } - api.replica.SetCallback(func() { + api.replicaManager.SetCallback(func() { addresses := make([]string, 0) - for _, replica := range api.replica.Regional() { + for _, replica := range api.replicaManager.Regional() { addresses = append(addresses, replica.RelayAddress) } api.derpMesh.SetAddresses(addresses) diff --git a/enterprise/highavailability/replica/replica.go b/enterprise/highavailability/replicasync/replicasync.go similarity index 93% rename from enterprise/highavailability/replica/replica.go rename to enterprise/highavailability/replicasync/replicasync.go index 6855b32852e3e..c632f8df2462b 100644 --- a/enterprise/highavailability/replica/replica.go +++ b/enterprise/highavailability/replicasync/replicasync.go @@ -1,4 +1,4 @@ -package replica +package replicasync import ( "context" @@ -34,7 +34,7 @@ type Options struct { // New registers the replica with the database and periodically updates to ensure // it's healthy. It contacts all other alive replicas to ensure they are reachable. -func New(ctx context.Context, logger slog.Logger, db database.Store, pubsub database.Pubsub, options Options) (*Server, error) { +func New(ctx context.Context, logger slog.Logger, db database.Store, pubsub database.Pubsub, options Options) (*Manager, error) { if options.ID == uuid.Nil { panic("An ID must be provided!") } @@ -88,7 +88,7 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, pubsub data return nil, xerrors.Errorf("publish new replica: %w", err) } ctx, cancelFunc := context.WithCancel(ctx) - server := &Server{ + server := &Manager{ options: &options, db: db, pubsub: pubsub, @@ -110,7 +110,8 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, pubsub data return server, nil } -type Server struct { +// Manager keeps the replica up to date and in sync with other replicas. +type Manager struct { options *Options db database.Store pubsub database.Pubsub @@ -128,7 +129,7 @@ type Server struct { } // loop runs the replica update sequence on an update interval. -func (s *Server) loop(ctx context.Context) { +func (s *Manager) loop(ctx context.Context) { defer s.closeWait.Done() ticker := time.NewTicker(s.options.UpdateInterval) defer ticker.Stop() @@ -146,7 +147,7 @@ func (s *Server) loop(ctx context.Context) { } // subscribe listens for new replica information! -func (s *Server) subscribe(ctx context.Context) error { +func (s *Manager) subscribe(ctx context.Context) error { needsUpdate := false updating := false updateMutex := sync.Mutex{} @@ -199,7 +200,7 @@ func (s *Server) subscribe(ctx context.Context) error { return nil } -func (s *Server) run(ctx context.Context) error { +func (s *Manager) run(ctx context.Context) error { s.closeMutex.Lock() s.closeWait.Add(1) s.closeMutex.Unlock() @@ -291,21 +292,21 @@ func (s *Server) run(ctx context.Context) error { } // Self represents the current replica. -func (s *Server) Self() database.Replica { +func (s *Manager) Self() database.Replica { s.mutex.Lock() defer s.mutex.Unlock() return s.self } // All returns every replica, including itself. -func (s *Server) All() []database.Replica { +func (s *Manager) All() []database.Replica { s.mutex.Lock() defer s.mutex.Unlock() return append(s.peers, s.self) } // Regional returns all replicas in the same region excluding itself. -func (s *Server) Regional() []database.Replica { +func (s *Manager) Regional() []database.Replica { s.mutex.Lock() defer s.mutex.Unlock() replicas := make([]database.Replica, 0) @@ -320,7 +321,7 @@ func (s *Server) Regional() []database.Replica { // SetCallback sets a function to execute whenever new peers // are refreshed or updated. -func (s *Server) SetCallback(callback func()) { +func (s *Manager) SetCallback(callback func()) { s.mutex.Lock() defer s.mutex.Unlock() s.callback = callback @@ -328,7 +329,7 @@ func (s *Server) SetCallback(callback func()) { go callback() } -func (s *Server) Close() error { +func (s *Manager) Close() error { s.closeMutex.Lock() select { case <-s.closed: diff --git a/enterprise/highavailability/replica/replica_test.go b/enterprise/highavailability/replicasync/replicasync_test.go similarity index 80% rename from enterprise/highavailability/replica/replica_test.go rename to enterprise/highavailability/replicasync/replicasync_test.go index a5bda874ea166..f4d800650f939 100644 --- a/enterprise/highavailability/replica/replica_test.go +++ b/enterprise/highavailability/replicasync/replicasync_test.go @@ -1,4 +1,4 @@ -package replica_test +package replicasync_test import ( "context" @@ -17,7 +17,7 @@ import ( "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/coderd/database" "github.com/coder/coder/coderd/database/dbtestutil" - "github.com/coder/coder/enterprise/highavailability/replica" + "github.com/coder/coder/enterprise/highavailability/replicasync" "github.com/coder/coder/testutil" ) @@ -32,12 +32,12 @@ func TestReplica(t *testing.T) { t.Parallel() db, pubsub := dbtestutil.NewDB(t) id := uuid.New() - cancel, err := pubsub.Subscribe(replica.PubsubEvent, func(ctx context.Context, message []byte) { + cancel, err := pubsub.Subscribe(replicasync.PubsubEvent, func(ctx context.Context, message []byte) { assert.Equal(t, []byte(id.String()), message) }) require.NoError(t, err) defer cancel() - server, err := replica.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replica.Options{ + server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replicasync.Options{ ID: id, }) require.NoError(t, err) @@ -54,12 +54,12 @@ func TestReplica(t *testing.T) { ID: id, }) require.NoError(t, err) - cancel, err := pubsub.Subscribe(replica.PubsubEvent, func(ctx context.Context, message []byte) { + cancel, err := pubsub.Subscribe(replicasync.PubsubEvent, func(ctx context.Context, message []byte) { assert.Equal(t, []byte(id.String()), message) }) require.NoError(t, err) defer cancel() - server, err := replica.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replica.Options{ + server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replicasync.Options{ ID: id, }) require.NoError(t, err) @@ -84,7 +84,7 @@ func TestReplica(t *testing.T) { RelayAddress: srv.URL, }) require.NoError(t, err) - server, err := replica.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replica.Options{ + server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replicasync.Options{ ID: uuid.New(), }) require.NoError(t, err) @@ -97,7 +97,7 @@ func TestReplica(t *testing.T) { t.Parallel() db, pubsub := dbtestutil.NewDB(t) var count atomic.Int32 - cancel, err := pubsub.Subscribe(replica.PubsubEvent, func(ctx context.Context, message []byte) { + cancel, err := pubsub.Subscribe(replicasync.PubsubEvent, func(ctx context.Context, message []byte) { count.Add(1) }) require.NoError(t, err) @@ -112,7 +112,7 @@ func TestReplica(t *testing.T) { RelayAddress: "http://169.254.169.254", }) require.NoError(t, err) - server, err := replica.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replica.Options{ + server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replicasync.Options{ ID: uuid.New(), PeerTimeout: 1 * time.Millisecond, }) @@ -130,7 +130,7 @@ func TestReplica(t *testing.T) { t.Parallel() db, pubsub := dbtestutil.NewDB(t) id := uuid.New() - server, err := replica.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replica.Options{ + server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replicasync.Options{ ID: id, }) require.NoError(t, err) @@ -145,9 +145,9 @@ func TestReplica(t *testing.T) { }) require.NoError(t, err) // Publish multiple times to ensure it can handle that case. - err = pubsub.Publish(replica.PubsubEvent, []byte(peer.ID.String())) + err = pubsub.Publish(replicasync.PubsubEvent, []byte(peer.ID.String())) require.NoError(t, err) - err = pubsub.Publish(replica.PubsubEvent, []byte(peer.ID.String())) + err = pubsub.Publish(replicasync.PubsubEvent, []byte(peer.ID.String())) require.NoError(t, err) require.Eventually(t, func() bool { return len(server.Regional()) == 1 @@ -168,7 +168,7 @@ func TestReplica(t *testing.T) { count := 20 wg.Add(count) for i := 0; i < count; i++ { - server, err := replica.New(context.Background(), logger, db, pubsub, replica.Options{ + server, err := replicasync.New(context.Background(), logger, db, pubsub, replicasync.Options{ ID: uuid.New(), RelayAddress: srv.URL, }) From 9124b0045cacf0b518a7299f9a8eb48842d2ecc7 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 13 Oct 2022 03:44:47 +0000 Subject: [PATCH 17/79] Denest packages for replicas --- enterprise/coderd/coderd.go | 22 ++++++------------- .../derpmesh/derpmesh.go | 0 .../derpmesh/derpmesh_test.go | 2 +- .../replicasync/replicasync.go | 0 .../replicasync/replicasync_test.go | 2 +- .../coordinator.go | 2 +- .../coordinator_test.go | 14 ++++++------ 7 files changed, 17 insertions(+), 25 deletions(-) rename enterprise/{highavailability => }/derpmesh/derpmesh.go (100%) rename enterprise/{highavailability => }/derpmesh/derpmesh_test.go (98%) rename enterprise/{highavailability => }/replicasync/replicasync.go (100%) rename enterprise/{highavailability => }/replicasync/replicasync_test.go (98%) rename enterprise/{highavailability => tailnet}/coordinator.go (99%) rename enterprise/{highavailability => tailnet}/coordinator_test.go (92%) diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 19812ea1f8b42..342b992c8076f 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -24,10 +24,10 @@ import ( "github.com/coder/coder/enterprise/audit" "github.com/coder/coder/enterprise/audit/backends" "github.com/coder/coder/enterprise/coderd/license" - "github.com/coder/coder/enterprise/highavailability" - "github.com/coder/coder/enterprise/highavailability/derpmesh" - "github.com/coder/coder/enterprise/highavailability/replicasync" - "github.com/coder/coder/tailnet" + "github.com/coder/coder/enterprise/derpmesh" + "github.com/coder/coder/enterprise/replicasync" + "github.com/coder/coder/enterprise/tailnet" + agpltailnet "github.com/coder/coder/tailnet" ) // New constructs an Enterprise coderd API instance. @@ -117,15 +117,6 @@ func New(ctx context.Context, options *Options) (*API, error) { }) } - // If high availability is disabled and multiple replicas appear, show an error. - // If high availability is enabled and the built-in DERP is but the DERP relay isn't set, show an error. - // We need to block meshing if high availability is disabled, because the meshing code would just work. - // SetAddresses([]string{}) - - api.AGPL.RootHandler.Route("/replicas", func(r chi.Router) { - - }) - var err error api.replicaManager, err = replicasync.New(ctx, options.Logger, options.Database, options.Pubsub, replicasync.Options{ ID: options.ReplicaID, @@ -244,9 +235,9 @@ func (api *API) updateEntitlements(ctx context.Context) error { } if changed, enabled := featureChanged(codersdk.FeatureHighAvailability); changed { - coordinator := tailnet.NewCoordinator() + coordinator := agpltailnet.NewCoordinator() if enabled { - haCoordinator, err := highavailability.NewCoordinator(api.Logger, api.Pubsub) + haCoordinator, err := tailnet.NewCoordinator(api.Logger, api.Pubsub) if err != nil { api.Logger.Error(ctx, "unable to set up high availability coordinator", slog.Error(err)) // If we try to setup the HA coordinator and it fails, nothing @@ -265,6 +256,7 @@ func (api *API) updateEntitlements(ctx context.Context) error { }) } else { api.derpMesh.SetAddresses([]string{}) + api.replicaManager.SetCallback(func() {}) } // Recheck changed in case the HA coordinator failed to set up. diff --git a/enterprise/highavailability/derpmesh/derpmesh.go b/enterprise/derpmesh/derpmesh.go similarity index 100% rename from enterprise/highavailability/derpmesh/derpmesh.go rename to enterprise/derpmesh/derpmesh.go diff --git a/enterprise/highavailability/derpmesh/derpmesh_test.go b/enterprise/derpmesh/derpmesh_test.go similarity index 98% rename from enterprise/highavailability/derpmesh/derpmesh_test.go rename to enterprise/derpmesh/derpmesh_test.go index 6e1154fc3d6a8..313c33da99bad 100644 --- a/enterprise/highavailability/derpmesh/derpmesh_test.go +++ b/enterprise/derpmesh/derpmesh_test.go @@ -16,7 +16,7 @@ import ( "cdr.dev/slog" "cdr.dev/slog/sloggers/slogtest" - "github.com/coder/coder/enterprise/highavailability/derpmesh" + "github.com/coder/coder/enterprise/derpmesh" "github.com/coder/coder/tailnet" ) diff --git a/enterprise/highavailability/replicasync/replicasync.go b/enterprise/replicasync/replicasync.go similarity index 100% rename from enterprise/highavailability/replicasync/replicasync.go rename to enterprise/replicasync/replicasync.go diff --git a/enterprise/highavailability/replicasync/replicasync_test.go b/enterprise/replicasync/replicasync_test.go similarity index 98% rename from enterprise/highavailability/replicasync/replicasync_test.go rename to enterprise/replicasync/replicasync_test.go index f4d800650f939..5ce774ea5f29a 100644 --- a/enterprise/highavailability/replicasync/replicasync_test.go +++ b/enterprise/replicasync/replicasync_test.go @@ -17,7 +17,7 @@ import ( "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/coderd/database" "github.com/coder/coder/coderd/database/dbtestutil" - "github.com/coder/coder/enterprise/highavailability/replicasync" + "github.com/coder/coder/enterprise/replicasync" "github.com/coder/coder/testutil" ) diff --git a/enterprise/highavailability/coordinator.go b/enterprise/tailnet/coordinator.go similarity index 99% rename from enterprise/highavailability/coordinator.go rename to enterprise/tailnet/coordinator.go index 7c41e47d44f1d..0643f7a259719 100644 --- a/enterprise/highavailability/coordinator.go +++ b/enterprise/tailnet/coordinator.go @@ -1,4 +1,4 @@ -package highavailability +package tailnet import ( "bytes" diff --git a/enterprise/highavailability/coordinator_test.go b/enterprise/tailnet/coordinator_test.go similarity index 92% rename from enterprise/highavailability/coordinator_test.go rename to enterprise/tailnet/coordinator_test.go index 1e86c08f1b1ed..83fac250b2916 100644 --- a/enterprise/highavailability/coordinator_test.go +++ b/enterprise/tailnet/coordinator_test.go @@ -1,4 +1,4 @@ -package highavailability_test +package tailnet_test import ( "net" @@ -11,7 +11,7 @@ import ( "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/coderd/database" - "github.com/coder/coder/enterprise/highavailability" + "github.com/coder/coder/enterprise/tailnet" agpl "github.com/coder/coder/tailnet" "github.com/coder/coder/testutil" ) @@ -20,7 +20,7 @@ func TestCoordinatorSingle(t *testing.T) { t.Parallel() t.Run("ClientWithoutAgent", func(t *testing.T) { t.Parallel() - coordinator, err := highavailability.NewCoordinator(slogtest.Make(t, nil), database.NewPubsubInMemory()) + coordinator, err := tailnet.NewCoordinator(slogtest.Make(t, nil), database.NewPubsubInMemory()) require.NoError(t, err) defer coordinator.Close() @@ -48,7 +48,7 @@ func TestCoordinatorSingle(t *testing.T) { t.Run("AgentWithoutClients", func(t *testing.T) { t.Parallel() - coordinator, err := highavailability.NewCoordinator(slogtest.Make(t, nil), database.NewPubsubInMemory()) + coordinator, err := tailnet.NewCoordinator(slogtest.Make(t, nil), database.NewPubsubInMemory()) require.NoError(t, err) defer coordinator.Close() @@ -76,7 +76,7 @@ func TestCoordinatorSingle(t *testing.T) { t.Run("AgentWithClient", func(t *testing.T) { t.Parallel() - coordinator, err := highavailability.NewCoordinator(slogtest.Make(t, nil), database.NewPubsubInMemory()) + coordinator, err := tailnet.NewCoordinator(slogtest.Make(t, nil), database.NewPubsubInMemory()) require.NoError(t, err) defer coordinator.Close() @@ -169,11 +169,11 @@ func TestCoordinatorHA(t *testing.T) { pubsub := database.NewPubsubInMemory() - coordinator1, err := highavailability.NewCoordinator(slogtest.Make(t, nil), pubsub) + coordinator1, err := tailnet.NewCoordinator(slogtest.Make(t, nil), pubsub) require.NoError(t, err) defer coordinator1.Close() - coordinator2, err := highavailability.NewCoordinator(slogtest.Make(t, nil), pubsub) + coordinator2, err := tailnet.NewCoordinator(slogtest.Make(t, nil), pubsub) require.NoError(t, err) defer coordinator2.Close() From d5555f6938978c38906aaf4ae6518c290ff0b983 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 13 Oct 2022 17:53:54 +0000 Subject: [PATCH 18/79] Add test for multiple replicas --- coderd/coderd.go | 1 + .../coderd/coderdenttest/coderdenttest.go | 11 ++++---- enterprise/coderd/license/license.go | 11 +++++--- enterprise/coderd/license/license_test.go | 2 +- enterprise/coderd/replicas_test.go | 26 ++++++++++++++----- 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index 57b78520d1b50..1b4e674de11cb 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -124,6 +124,7 @@ func New(options *Options) *API { } if options.DERPServer == nil { options.DERPServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger)) + options.DERPServer.SetMeshKey("todo-kyle-change-this") } if options.Auditor == nil { options.Auditor = audit.NewNop() diff --git a/enterprise/coderd/coderdenttest/coderdenttest.go b/enterprise/coderd/coderdenttest/coderdenttest.go index c5ec2391d97bf..57440ac37082e 100644 --- a/enterprise/coderd/coderdenttest/coderdenttest.go +++ b/enterprise/coderd/coderdenttest/coderdenttest.go @@ -63,11 +63,12 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c } srv, cancelFunc, oop := coderdtest.NewOptions(t, options.Options) coderAPI, err := coderd.New(context.Background(), &coderd.Options{ - RBAC: true, - AuditLogging: options.AuditLogging, - BrowserOnly: options.BrowserOnly, - SCIMAPIKey: options.SCIMAPIKey, - DERPServerRelayAddress: oop.AccessURL.String(), + RBAC: true, + AuditLogging: options.AuditLogging, + BrowserOnly: options.BrowserOnly, + SCIMAPIKey: options.SCIMAPIKey, + // TODO: Kyle change this before merge! + DERPServerRelayAddress: oop.AccessURL.String() + "/derp", DERPServerRegionID: 1, HighAvailability: true, ReplicaID: uuid.New(), diff --git a/enterprise/coderd/license/license.go b/enterprise/coderd/license/license.go index d7643683a6d2f..43f8b53094c7c 100644 --- a/enterprise/coderd/license/license.go +++ b/enterprise/coderd/license/license.go @@ -153,9 +153,6 @@ func Entitlements( case codersdk.EntitlementNotEntitled: entitlements.Warnings = append(entitlements.Warnings, fmt.Sprintf("%s is enabled but your license is not entitled to this feature.", niceName)) - // Disable the feature and add a warning... - feature.Enabled = false - entitlements.Features[featureName] = feature case codersdk.EntitlementGracePeriod: entitlements.Warnings = append(entitlements.Warnings, fmt.Sprintf("%s is enabled but your license for this feature is expired.", niceName)) @@ -164,6 +161,14 @@ func Entitlements( } } + for _, featureName := range codersdk.FeatureNames { + feature := entitlements.Features[featureName] + if feature.Entitlement == codersdk.EntitlementNotEntitled { + feature.Enabled = false + entitlements.Features[featureName] = feature + } + } + return entitlements, nil } diff --git a/enterprise/coderd/license/license_test.go b/enterprise/coderd/license/license_test.go index e1fbdc6d3d9fa..f1318e26bae47 100644 --- a/enterprise/coderd/license/license_test.go +++ b/enterprise/coderd/license/license_test.go @@ -31,7 +31,7 @@ func TestEntitlements(t *testing.T) { t.Run("Defaults", func(t *testing.T) { t.Parallel() db := databasefake.New() - entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, map[string]bool{}) + entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, all) require.NoError(t, err) require.False(t, entitlements.HasLicense) require.False(t, entitlements.Trial) diff --git a/enterprise/coderd/replicas_test.go b/enterprise/coderd/replicas_test.go index 1a5a3ed5f4eee..52836a720f623 100644 --- a/enterprise/coderd/replicas_test.go +++ b/enterprise/coderd/replicas_test.go @@ -2,14 +2,16 @@ package coderd_test import ( "context" - "fmt" "testing" + "time" "github.com/stretchr/testify/require" + "cdr.dev/slog" + "cdr.dev/slog/sloggers/slogtest" + "github.com/coder/coder/coderd/coderdtest" "github.com/coder/coder/coderd/database/dbtestutil" - "github.com/coder/coder/codersdk" "github.com/coder/coder/enterprise/coderd/coderdenttest" ) @@ -18,11 +20,15 @@ func TestReplicas(t *testing.T) { db, pubsub := dbtestutil.NewDB(t) firstClient := coderdenttest.New(t, &coderdenttest.Options{ Options: &coderdtest.Options{ - Database: db, - Pubsub: pubsub, + IncludeProvisionerDaemon: true, + Database: db, + Pubsub: pubsub, }, }) - _ = coderdtest.CreateFirstUser(t, firstClient) + firstUser := coderdtest.CreateFirstUser(t, firstClient) + coderdenttest.AddLicense(t, firstClient, coderdenttest.LicenseOptions{ + HighAvailability: true, + }) secondClient := coderdenttest.New(t, &coderdenttest.Options{ Options: &coderdtest.Options{ @@ -32,7 +38,13 @@ func TestReplicas(t *testing.T) { }) secondClient.SessionToken = firstClient.SessionToken - user, err := secondClient.User(context.Background(), codersdk.Me) + agentID := setupWorkspaceAgent(t, firstClient, firstUser) + conn, err := secondClient.DialWorkspaceAgentTailnet(context.Background(), slogtest.Make(t, nil).Leveled(slog.LevelDebug), agentID) require.NoError(t, err) - fmt.Printf("%+v\n", user) + require.Eventually(t, func() bool { + _, err = conn.Ping() + return err == nil + }, 10*time.Second, 250*time.Millisecond) + + _ = conn.Close() } From 8dfc261c7bb6bfa440fa62389b647414fdc57ddb Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 13 Oct 2022 19:26:36 +0000 Subject: [PATCH 19/79] Fix coordination test --- coderd/coderd.go | 5 ++++- enterprise/coderd/workspaceagents_test.go | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index 1b4e674de11cb..6b4b335161c32 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -558,7 +558,10 @@ func (api *API) Close() error { api.websocketWaitMutex.Unlock() api.metricsCache.Close() - + coordinator := api.TailnetCoordinator.Load() + if coordinator != nil { + _ = (*coordinator).Close() + } return api.workspaceAgentCache.Close() } diff --git a/enterprise/coderd/workspaceagents_test.go b/enterprise/coderd/workspaceagents_test.go index 3bb40b75b00f8..24e24e3f5f540 100644 --- a/enterprise/coderd/workspaceagents_test.go +++ b/enterprise/coderd/workspaceagents_test.go @@ -89,9 +89,9 @@ func setupWorkspaceAgent(t *testing.T, client *codersdk.Client, user codersdk.Cr CoordinatorDialer: agentClient.ListenWorkspaceAgentTailnet, Logger: slogtest.Make(t, nil).Named("agent"), }) - defer func() { + t.Cleanup(func() { _ = agentCloser.Close() - }() + }) resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID) return resources[0].Agents[0].ID } From ff5968bd9c5386dea16ed68ed9684870cf6bb0bd Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 13 Oct 2022 19:46:40 +0000 Subject: [PATCH 20/79] Add HA to the helm chart --- helm/templates/coder.yaml | 12 ++++++++---- helm/values.yaml | 8 ++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/helm/templates/coder.yaml b/helm/templates/coder.yaml index 45f3f6e29a32e..1165251fc885b 100644 --- a/helm/templates/coder.yaml +++ b/helm/templates/coder.yaml @@ -14,10 +14,7 @@ metadata: {{- include "coder.labels" . | nindent 4 }} annotations: {{ toYaml .Values.coder.annotations | nindent 4}} spec: - # NOTE: this is currently not used as coder v2 does not support high - # availability yet. - # replicas: {{ .Values.coder.replicaCount }} - replicas: 1 + replicas: {{ .Values.coder.replicaCount }} selector: matchLabels: {{- include "coder.selectorLabels" . | nindent 6 }} @@ -38,6 +35,13 @@ spec: env: - name: CODER_ADDRESS value: "0.0.0.0:{{ include "coder.port" . }}" + # Used for inter-pod communication with high-availability. + - name: KUBE_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: CODER_DERP_SERVER_RELAY_ADDRESS + value: "{{ include "coder.portName" . }}://$(KUBE_POD_IP):{{ include "coder.port" . }}" {{- include "coder.tlsEnv" . | nindent 12 }} {{- with .Values.coder.env -}} {{ toYaml . | nindent 12 }} diff --git a/helm/values.yaml b/helm/values.yaml index cfba214ee6028..3beebdd3fc3b9 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -1,9 +1,9 @@ # coder -- Primary configuration for `coder server`. coder: - # NOTE: this is currently not used as coder v2 does not support high - # availability yet. - # # coder.replicaCount -- The number of Kubernetes deployment replicas. - # replicaCount: 1 + # coder.replicaCount -- The number of Kubernetes deployment replicas. + # This should only be increased if High Availability is enabled. + # This is an Enterprise feature. Contact sales@coder.com. + replicaCount: 1 # coder.image -- The image to use for Coder. image: From 557b390f62d2eee07e0131238c1b25a4e130078d Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 13 Oct 2022 19:49:28 +0000 Subject: [PATCH 21/79] Rename function pointer --- codersdk/replicas.go | 22 ---- enterprise/replicasync/replicasync.go | 148 +++++++++++++------------- 2 files changed, 74 insertions(+), 96 deletions(-) delete mode 100644 codersdk/replicas.go diff --git a/codersdk/replicas.go b/codersdk/replicas.go deleted file mode 100644 index 341b460792ddd..0000000000000 --- a/codersdk/replicas.go +++ /dev/null @@ -1,22 +0,0 @@ -package codersdk - -import ( - "time" - - "github.com/google/uuid" -) - -type Replica struct { - // ID is the unique identifier for the replica. - ID uuid.UUID `json:"id"` - // Hostname is the hostname of the replica. - Hostname string `json:"hostname"` - // CreatedAt is when the replica was first seen. - CreatedAt time.Time `json:"created_at"` - // Active determines whether the replica is online. - Active bool `json:"active"` - // RelayAddress is the accessible address to relay DERP connections. - RelayAddress string `json:"relay_address"` - // Error is the error. - Error string `json:"error"` -} diff --git a/enterprise/replicasync/replicasync.go b/enterprise/replicasync/replicasync.go index c632f8df2462b..4d6038a694940 100644 --- a/enterprise/replicasync/replicasync.go +++ b/enterprise/replicasync/replicasync.go @@ -129,9 +129,9 @@ type Manager struct { } // loop runs the replica update sequence on an update interval. -func (s *Manager) loop(ctx context.Context) { - defer s.closeWait.Done() - ticker := time.NewTicker(s.options.UpdateInterval) +func (m *Manager) loop(ctx context.Context) { + defer m.closeWait.Done() + ticker := time.NewTicker(m.options.UpdateInterval) defer ticker.Stop() for { select { @@ -139,15 +139,15 @@ func (s *Manager) loop(ctx context.Context) { return case <-ticker.C: } - err := s.run(ctx) + err := m.run(ctx) if err != nil && !errors.Is(err, context.Canceled) { - s.logger.Warn(ctx, "run replica update loop", slog.Error(err)) + m.logger.Warn(ctx, "run replica update loop", slog.Error(err)) } } } // subscribe listens for new replica information! -func (s *Manager) subscribe(ctx context.Context) error { +func (m *Manager) subscribe(ctx context.Context) error { needsUpdate := false updating := false updateMutex := sync.Mutex{} @@ -158,9 +158,9 @@ func (s *Manager) subscribe(ctx context.Context) error { // it will reprocess afterwards. var update func() update = func() { - err := s.run(ctx) + err := m.run(ctx) if err != nil && !errors.Is(err, context.Canceled) { - s.logger.Error(ctx, "run replica from subscribe", slog.Error(err)) + m.logger.Error(ctx, "run replica from subscribe", slog.Error(err)) } updateMutex.Lock() if needsUpdate { @@ -172,7 +172,7 @@ func (s *Manager) subscribe(ctx context.Context) error { updating = false updateMutex.Unlock() } - cancelFunc, err := s.pubsub.Subscribe(PubsubEvent, func(ctx context.Context, message []byte) { + cancelFunc, err := m.pubsub.Subscribe(PubsubEvent, func(ctx context.Context, message []byte) { updateMutex.Lock() defer updateMutex.Unlock() id, err := uuid.Parse(string(message)) @@ -180,7 +180,7 @@ func (s *Manager) subscribe(ctx context.Context) error { return } // Don't process updates for ourself! - if id == s.options.ID { + if id == m.options.ID { return } if updating { @@ -200,46 +200,46 @@ func (s *Manager) subscribe(ctx context.Context) error { return nil } -func (s *Manager) run(ctx context.Context) error { - s.closeMutex.Lock() - s.closeWait.Add(1) - s.closeMutex.Unlock() +func (m *Manager) run(ctx context.Context) error { + m.closeMutex.Lock() + m.closeWait.Add(1) + m.closeMutex.Unlock() go func() { - s.closeWait.Done() + m.closeWait.Done() }() // Expect replicas to update once every three times the interval... // If they don't, assume death! - replicas, err := s.db.GetReplicasUpdatedAfter(ctx, database.Now().Add(-3*s.options.UpdateInterval)) + replicas, err := m.db.GetReplicasUpdatedAfter(ctx, database.Now().Add(-3*m.options.UpdateInterval)) if err != nil { return xerrors.Errorf("get replicas: %w", err) } - s.mutex.Lock() - s.peers = make([]database.Replica, 0, len(replicas)) + m.mutex.Lock() + m.peers = make([]database.Replica, 0, len(replicas)) for _, replica := range replicas { - if replica.ID == s.options.ID { + if replica.ID == m.options.ID { continue } - s.peers = append(s.peers, replica) + m.peers = append(m.peers, replica) } - s.mutex.Unlock() + m.mutex.Unlock() var wg sync.WaitGroup var mu sync.Mutex failed := make([]string, 0) - for _, peer := range s.Regional() { + for _, peer := range m.Regional() { wg.Add(1) peer := peer go func() { defer wg.Done() req, err := http.NewRequestWithContext(ctx, http.MethodGet, peer.RelayAddress, nil) if err != nil { - s.logger.Error(ctx, "create http request for relay probe", + m.logger.Error(ctx, "create http request for relay probe", slog.F("relay_address", peer.RelayAddress), slog.Error(err)) return } client := http.Client{ - Timeout: s.options.PeerTimeout, + Timeout: m.options.PeerTimeout, } res, err := client.Do(req) if err != nil { @@ -260,58 +260,58 @@ func (s *Manager) run(ctx context.Context) error { } } - replica, err := s.db.UpdateReplica(ctx, database.UpdateReplicaParams{ - ID: s.self.ID, + replica, err := m.db.UpdateReplica(ctx, database.UpdateReplicaParams{ + ID: m.self.ID, UpdatedAt: database.Now(), - StartedAt: s.self.StartedAt, - StoppedAt: s.self.StoppedAt, - RelayAddress: s.self.RelayAddress, - RegionID: s.self.RegionID, - Hostname: s.self.Hostname, - Version: s.self.Version, + StartedAt: m.self.StartedAt, + StoppedAt: m.self.StoppedAt, + RelayAddress: m.self.RelayAddress, + RegionID: m.self.RegionID, + Hostname: m.self.Hostname, + Version: m.self.Version, Error: replicaError, }) if err != nil { return xerrors.Errorf("update replica: %w", err) } - s.mutex.Lock() - if s.self.Error.String != replica.Error.String { + m.mutex.Lock() + if m.self.Error.String != replica.Error.String { // Publish an update occurred! - err = s.pubsub.Publish(PubsubEvent, []byte(s.self.ID.String())) + err = m.pubsub.Publish(PubsubEvent, []byte(m.self.ID.String())) if err != nil { - s.mutex.Unlock() + m.mutex.Unlock() return xerrors.Errorf("publish replica update: %w", err) } } - s.self = replica - if s.callback != nil { - go s.callback() + m.self = replica + if m.callback != nil { + go m.callback() } - s.mutex.Unlock() + m.mutex.Unlock() return nil } // Self represents the current replica. -func (s *Manager) Self() database.Replica { - s.mutex.Lock() - defer s.mutex.Unlock() - return s.self +func (m *Manager) Self() database.Replica { + m.mutex.Lock() + defer m.mutex.Unlock() + return m.self } // All returns every replica, including itself. -func (s *Manager) All() []database.Replica { - s.mutex.Lock() - defer s.mutex.Unlock() - return append(s.peers, s.self) +func (m *Manager) All() []database.Replica { + m.mutex.Lock() + defer m.mutex.Unlock() + return append(m.peers, m.self) } // Regional returns all replicas in the same region excluding itself. -func (s *Manager) Regional() []database.Replica { - s.mutex.Lock() - defer s.mutex.Unlock() +func (m *Manager) Regional() []database.Replica { + m.mutex.Lock() + defer m.mutex.Unlock() replicas := make([]database.Replica, 0) - for _, replica := range s.peers { - if replica.RegionID != s.self.RegionID { + for _, replica := range m.peers { + if replica.RegionID != m.self.RegionID { continue } replicas = append(replicas, replica) @@ -321,47 +321,47 @@ func (s *Manager) Regional() []database.Replica { // SetCallback sets a function to execute whenever new peers // are refreshed or updated. -func (s *Manager) SetCallback(callback func()) { - s.mutex.Lock() - defer s.mutex.Unlock() - s.callback = callback +func (m *Manager) SetCallback(callback func()) { + m.mutex.Lock() + defer m.mutex.Unlock() + m.callback = callback // Instantly call the callback to inform replicas! go callback() } -func (s *Manager) Close() error { - s.closeMutex.Lock() +func (m *Manager) Close() error { + m.closeMutex.Lock() select { - case <-s.closed: - s.closeMutex.Unlock() + case <-m.closed: + m.closeMutex.Unlock() return nil default: } - close(s.closed) - s.closeCancel() - s.closeWait.Wait() - s.closeMutex.Unlock() + close(m.closed) + m.closeCancel() + m.closeWait.Wait() + m.closeMutex.Unlock() ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Second) defer cancelFunc() - _, err := s.db.UpdateReplica(ctx, database.UpdateReplicaParams{ - ID: s.self.ID, + _, err := m.db.UpdateReplica(ctx, database.UpdateReplicaParams{ + ID: m.self.ID, UpdatedAt: database.Now(), - StartedAt: s.self.StartedAt, + StartedAt: m.self.StartedAt, StoppedAt: sql.NullTime{ Time: database.Now(), Valid: true, }, - RelayAddress: s.self.RelayAddress, - RegionID: s.self.RegionID, - Hostname: s.self.Hostname, - Version: s.self.Version, - Error: s.self.Error, + RelayAddress: m.self.RelayAddress, + RegionID: m.self.RegionID, + Hostname: m.self.Hostname, + Version: m.self.Version, + Error: m.self.Error, }) if err != nil { return xerrors.Errorf("update replica: %w", err) } - err = s.pubsub.Publish(PubsubEvent, []byte(s.self.ID.String())) + err = m.pubsub.Publish(PubsubEvent, []byte(m.self.ID.String())) if err != nil { return xerrors.Errorf("publish replica update: %w", err) } From 186a5e2623d3aa57c7816a0fd6d9ff917e531298 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 13 Oct 2022 20:27:32 +0000 Subject: [PATCH 22/79] Add warnings for HA --- .vscode/settings.json | 2 + cli/deployment/flags.go | 8 --- codersdk/deployment.go | 26 +++++++ codersdk/flags.go | 1 - enterprise/cli/server.go | 30 +++++--- enterprise/coderd/coderd.go | 5 +- .../coderd/coderdenttest/coderdenttest.go | 12 ++-- enterprise/coderd/license/license.go | 23 ++++++ enterprise/coderd/license/license_test.go | 72 ++++++++++++++++--- enterprise/derpmesh/derpmesh.go | 13 ++++ 10 files changed, 156 insertions(+), 36 deletions(-) create mode 100644 codersdk/deployment.go diff --git a/.vscode/settings.json b/.vscode/settings.json index f556563596bc0..2e6ff3d23704c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -19,6 +19,7 @@ "derphttp", "derpmap", "devel", + "dflags", "drpc", "drpcconn", "drpcmux", @@ -88,6 +89,7 @@ "replicasync", "retrier", "rpty", + "SCIM", "sdkproto", "sdktrace", "Signup", diff --git a/cli/deployment/flags.go b/cli/deployment/flags.go index 35ae248a0a722..8c6608b552586 100644 --- a/cli/deployment/flags.go +++ b/cli/deployment/flags.go @@ -130,14 +130,6 @@ func Flags() *codersdk.DeploymentFlags { Description: "The bind address to serve pprof.", Default: "127.0.0.1:6060", }, - HighAvailability: &codersdk.BoolFlag{ - Name: "High Availability", - Flag: "high-availability", - EnvVar: "CODER_HIGH_AVAILABILITY", - Description: "Specifies whether high availability is enabled.", - Default: true, - Enterprise: true, - }, CacheDir: &codersdk.StringFlag{ Name: "Cache Directory", Flag: "cache-dir", diff --git a/codersdk/deployment.go b/codersdk/deployment.go new file mode 100644 index 0000000000000..a1227b09e3f63 --- /dev/null +++ b/codersdk/deployment.go @@ -0,0 +1,26 @@ +package codersdk + +import ( + "time" + + "github.com/google/uuid" +) + +type DeploymentInfo struct { + Replicas []Replica `json:"replicas"` +} + +type Replica struct { + // ID is the unique identifier for the replica. + ID uuid.UUID `json:"id"` + // Hostname is the hostname of the replica. + Hostname string `json:"hostname"` + // CreatedAt is when the replica was first seen. + CreatedAt time.Time `json:"created_at"` + // Active determines whether the replica is online. + Active bool `json:"active"` + // RelayAddress is the accessible address to relay DERP connections. + RelayAddress string `json:"relay_address"` + // Error is the error. + Error string `json:"error"` +} diff --git a/codersdk/flags.go b/codersdk/flags.go index 2dd1323a1fddc..09ca65b1ea813 100644 --- a/codersdk/flags.go +++ b/codersdk/flags.go @@ -60,7 +60,6 @@ type DeploymentFlags struct { Verbose *BoolFlag `json:"verbose" typescript:",notnull"` AuditLogging *BoolFlag `json:"audit_logging" typescript:",notnull"` BrowserOnly *BoolFlag `json:"browser_only" typescript:",notnull"` - HighAvailability *BoolFlag `json:"high_availability" typescript:",notnull"` SCIMAuthHeader *StringFlag `json:"scim_auth_header" typescript:",notnull"` UserWorkspaceQuota *IntFlag `json:"user_workspace_quota" typescript:",notnull"` } diff --git a/enterprise/cli/server.go b/enterprise/cli/server.go index e34bdaccfd342..cc44985e0a4d4 100644 --- a/enterprise/cli/server.go +++ b/enterprise/cli/server.go @@ -2,9 +2,11 @@ package cli import ( "context" + "net/url" "github.com/google/uuid" "github.com/spf13/cobra" + "golang.org/x/xerrors" "cdr.dev/slog" @@ -20,22 +22,35 @@ func server() *cobra.Command { dflags := deployment.Flags() cmd := agpl.Server(dflags, func(ctx context.Context, cfg config.Root, options *agplcoderd.Options) (*agplcoderd.API, error) { replicaIDRaw, err := cfg.ReplicaID().Read() + generatedReplicaID := false if err != nil { replicaIDRaw = uuid.NewString() + generatedReplicaID = true } replicaID, err := uuid.Parse(replicaIDRaw) if err != nil { options.Logger.Warn(ctx, "failed to parse replica id", slog.Error(err), slog.F("replica_id", replicaIDRaw)) replicaID = uuid.New() + generatedReplicaID = true + } + if generatedReplicaID { + // Make sure we save it to be reused later! + _ = cfg.ReplicaID().Write(replicaID.String()) + } + + if dflags.DerpServerRelayAddress.Value != "" { + _, err := url.Parse(dflags.DerpServerRelayAddress.Value) + if err != nil { + return nil, xerrors.Errorf("derp-server-relay-address must be a valid HTTP URL: %w", err) + } } - o := &coderd.Options{ - AuditLogging: dflags.AuditLogging.Value, - BrowserOnly: dflags.BrowserOnly.Value, - SCIMAPIKey: []byte(dflags.SCIMAuthHeader.Value), - UserWorkspaceQuota: dflags.UserWorkspaceQuota.Value, - RBAC: true, - HighAvailability: dflags.HighAvailability.Value, + o := &coderd.Options{ + AuditLogging: dflags.AuditLogging.Value, + BrowserOnly: dflags.BrowserOnly.Value, + SCIMAPIKey: []byte(dflags.SCIMAuthHeader.Value), + UserWorkspaceQuota: dflags.UserWorkspaceQuota.Value, + RBAC: true, ReplicaID: replicaID, DERPServerRelayAddress: dflags.DerpServerRelayAddress.Value, DERPServerRegionID: dflags.DerpServerRegionID.Value, @@ -50,6 +65,5 @@ func server() *cobra.Command { }) deployment.AttachFlags(cmd.Flags(), dflags, true) - return cmd } diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 342b992c8076f..b06e843b658a5 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -146,7 +146,6 @@ type Options struct { BrowserOnly bool SCIMAPIKey []byte UserWorkspaceQuota int - HighAvailability bool // Used for high availability. DERPServerRelayAddress string @@ -182,12 +181,12 @@ func (api *API) updateEntitlements(ctx context.Context) error { api.entitlementsMu.Lock() defer api.entitlementsMu.Unlock() - entitlements, err := license.Entitlements(ctx, api.Database, api.Logger, api.Keys, map[string]bool{ + entitlements, err := license.Entitlements(ctx, api.Database, api.Logger, len(api.replicaManager.All()), api.Keys, map[string]bool{ codersdk.FeatureAuditLog: api.AuditLogging, codersdk.FeatureBrowserOnly: api.BrowserOnly, codersdk.FeatureSCIM: len(api.SCIMAPIKey) != 0, codersdk.FeatureWorkspaceQuota: api.UserWorkspaceQuota != 0, - codersdk.FeatureHighAvailability: api.HighAvailability, + codersdk.FeatureHighAvailability: api.DERPServerRelayAddress != "", codersdk.FeatureTemplateRBAC: api.RBAC, }) if err != nil { diff --git a/enterprise/coderd/coderdenttest/coderdenttest.go b/enterprise/coderd/coderdenttest/coderdenttest.go index 57440ac37082e..24f0bffd5017f 100644 --- a/enterprise/coderd/coderdenttest/coderdenttest.go +++ b/enterprise/coderd/coderdenttest/coderdenttest.go @@ -63,14 +63,12 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c } srv, cancelFunc, oop := coderdtest.NewOptions(t, options.Options) coderAPI, err := coderd.New(context.Background(), &coderd.Options{ - RBAC: true, - AuditLogging: options.AuditLogging, - BrowserOnly: options.BrowserOnly, - SCIMAPIKey: options.SCIMAPIKey, - // TODO: Kyle change this before merge! - DERPServerRelayAddress: oop.AccessURL.String() + "/derp", + RBAC: true, + AuditLogging: options.AuditLogging, + BrowserOnly: options.BrowserOnly, + SCIMAPIKey: options.SCIMAPIKey, + DERPServerRelayAddress: oop.AccessURL.String(), DERPServerRegionID: 1, - HighAvailability: true, ReplicaID: uuid.New(), UserWorkspaceQuota: options.UserWorkspaceQuota, Options: oop, diff --git a/enterprise/coderd/license/license.go b/enterprise/coderd/license/license.go index 43f8b53094c7c..633b1a5056cab 100644 --- a/enterprise/coderd/license/license.go +++ b/enterprise/coderd/license/license.go @@ -21,6 +21,7 @@ func Entitlements( ctx context.Context, db database.Store, logger slog.Logger, + replicaCount int, keys map[string]ed25519.PublicKey, enablements map[string]bool, ) (codersdk.Entitlements, error) { @@ -144,6 +145,10 @@ func Entitlements( if featureName == codersdk.FeatureUserLimit { continue } + // High availability has it's own warnings based on replica count! + if featureName == codersdk.FeatureHighAvailability { + continue + } feature := entitlements.Features[featureName] if !feature.Enabled { continue @@ -161,6 +166,24 @@ func Entitlements( } } + if replicaCount > 1 { + feature := entitlements.Features[codersdk.FeatureHighAvailability] + + switch feature.Entitlement { + case codersdk.EntitlementNotEntitled: + if entitlements.HasLicense { + entitlements.Warnings = append(entitlements.Warnings, + "You have multiple replicas but your license is not entitled to high availability.") + } else { + entitlements.Warnings = append(entitlements.Warnings, + "You have multiple replicas but high availability is an Enterprise feature. Contact sales to get a license.") + } + case codersdk.EntitlementGracePeriod: + entitlements.Warnings = append(entitlements.Warnings, + "You have multiple replicas but your license for high availability is expired.") + } + } + for _, featureName := range codersdk.FeatureNames { feature := entitlements.Features[featureName] if feature.Entitlement == codersdk.EntitlementNotEntitled { diff --git a/enterprise/coderd/license/license_test.go b/enterprise/coderd/license/license_test.go index f1318e26bae47..5b50bdb97cfe2 100644 --- a/enterprise/coderd/license/license_test.go +++ b/enterprise/coderd/license/license_test.go @@ -31,7 +31,7 @@ func TestEntitlements(t *testing.T) { t.Run("Defaults", func(t *testing.T) { t.Parallel() db := databasefake.New() - entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, all) + entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, all) require.NoError(t, err) require.False(t, entitlements.HasLicense) require.False(t, entitlements.Trial) @@ -47,7 +47,7 @@ func TestEntitlements(t *testing.T) { JWT: coderdenttest.GenerateLicense(t, coderdenttest.LicenseOptions{}), Exp: time.Now().Add(time.Hour), }) - entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, map[string]bool{}) + entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, map[string]bool{}) require.NoError(t, err) require.True(t, entitlements.HasLicense) require.False(t, entitlements.Trial) @@ -71,7 +71,7 @@ func TestEntitlements(t *testing.T) { }), Exp: time.Now().Add(time.Hour), }) - entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, map[string]bool{}) + entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, map[string]bool{}) require.NoError(t, err) require.True(t, entitlements.HasLicense) require.False(t, entitlements.Trial) @@ -96,7 +96,7 @@ func TestEntitlements(t *testing.T) { }), Exp: time.Now().Add(time.Hour), }) - entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, all) + entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, all) require.NoError(t, err) require.True(t, entitlements.HasLicense) require.False(t, entitlements.Trial) @@ -104,6 +104,9 @@ func TestEntitlements(t *testing.T) { if featureName == codersdk.FeatureUserLimit { continue } + if featureName == codersdk.FeatureHighAvailability { + continue + } niceName := strings.Title(strings.ReplaceAll(featureName, "_", " ")) require.Equal(t, codersdk.EntitlementGracePeriod, entitlements.Features[featureName].Entitlement) require.Contains(t, entitlements.Warnings, fmt.Sprintf("%s is enabled but your license for this feature is expired.", niceName)) @@ -116,7 +119,7 @@ func TestEntitlements(t *testing.T) { JWT: coderdenttest.GenerateLicense(t, coderdenttest.LicenseOptions{}), Exp: time.Now().Add(time.Hour), }) - entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, all) + entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, all) require.NoError(t, err) require.True(t, entitlements.HasLicense) require.False(t, entitlements.Trial) @@ -124,6 +127,9 @@ func TestEntitlements(t *testing.T) { if featureName == codersdk.FeatureUserLimit { continue } + if featureName == codersdk.FeatureHighAvailability { + continue + } niceName := strings.Title(strings.ReplaceAll(featureName, "_", " ")) // Ensures features that are not entitled are properly disabled. require.False(t, entitlements.Features[featureName].Enabled) @@ -142,7 +148,7 @@ func TestEntitlements(t *testing.T) { }), Exp: time.Now().Add(time.Hour), }) - entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, map[string]bool{}) + entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, map[string]bool{}) require.NoError(t, err) require.True(t, entitlements.HasLicense) require.Contains(t, entitlements.Warnings, "Your deployment has 2 active users but is only licensed for 1.") @@ -164,7 +170,7 @@ func TestEntitlements(t *testing.T) { }), Exp: time.Now().Add(time.Hour), }) - entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, map[string]bool{}) + entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, map[string]bool{}) require.NoError(t, err) require.True(t, entitlements.HasLicense) require.Empty(t, entitlements.Warnings) @@ -187,7 +193,7 @@ func TestEntitlements(t *testing.T) { }), }) - entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, map[string]bool{}) + entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, map[string]bool{}) require.NoError(t, err) require.True(t, entitlements.HasLicense) require.False(t, entitlements.Trial) @@ -202,7 +208,7 @@ func TestEntitlements(t *testing.T) { AllFeatures: true, }), }) - entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, all) + entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, all) require.NoError(t, err) require.True(t, entitlements.HasLicense) require.False(t, entitlements.Trial) @@ -214,4 +220,52 @@ func TestEntitlements(t *testing.T) { require.Equal(t, codersdk.EntitlementEntitled, entitlements.Features[featureName].Entitlement) } }) + + t.Run("MultipleReplicasNoLicense", func(t *testing.T) { + t.Parallel() + db := databasefake.New() + entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 2, coderdenttest.Keys, all) + require.NoError(t, err) + require.False(t, entitlements.HasLicense) + require.Len(t, entitlements.Warnings, 1) + require.Equal(t, "You have multiple replicas but high availability is an Enterprise feature. Contact sales to get a license.", entitlements.Warnings[0]) + }) + + t.Run("MultipleReplicasNotEntitled", func(t *testing.T) { + t.Parallel() + db := databasefake.New() + db.InsertLicense(context.Background(), database.InsertLicenseParams{ + Exp: time.Now().Add(time.Hour), + JWT: coderdenttest.GenerateLicense(t, coderdenttest.LicenseOptions{ + AuditLog: true, + }), + }) + entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 2, coderdenttest.Keys, map[string]bool{ + codersdk.FeatureHighAvailability: true, + }) + require.NoError(t, err) + require.True(t, entitlements.HasLicense) + require.Len(t, entitlements.Warnings, 1) + require.Equal(t, "You have multiple replicas but your license is not entitled to high availability.", entitlements.Warnings[0]) + }) + + t.Run("MultipleReplicasGrace", func(t *testing.T) { + t.Parallel() + db := databasefake.New() + db.InsertLicense(context.Background(), database.InsertLicenseParams{ + JWT: coderdenttest.GenerateLicense(t, coderdenttest.LicenseOptions{ + HighAvailability: true, + GraceAt: time.Now().Add(-time.Hour), + ExpiresAt: time.Now().Add(time.Hour), + }), + Exp: time.Now().Add(time.Hour), + }) + entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 2, coderdenttest.Keys, map[string]bool{ + codersdk.FeatureHighAvailability: true, + }) + require.NoError(t, err) + require.True(t, entitlements.HasLicense) + require.Len(t, entitlements.Warnings, 1) + require.Equal(t, "You have multiple replicas but your license for high availability is expired.", entitlements.Warnings[0]) + }) } diff --git a/enterprise/derpmesh/derpmesh.go b/enterprise/derpmesh/derpmesh.go index 94341079cd43f..3ce22c1bd9a11 100644 --- a/enterprise/derpmesh/derpmesh.go +++ b/enterprise/derpmesh/derpmesh.go @@ -2,6 +2,7 @@ package derpmesh import ( "context" + "net/url" "sync" "golang.org/x/xerrors" @@ -40,6 +41,18 @@ type Mesh struct { func (m *Mesh) SetAddresses(addresses []string) { total := make(map[string]struct{}, 0) for _, address := range addresses { + addressURL, err := url.Parse(address) + if err != nil { + m.logger.Error(m.ctx, "invalid address", slog.F("address", err), slog.Error(err)) + continue + } + derpURL, err := addressURL.Parse("/derp") + if err != nil { + m.logger.Error(m.ctx, "parse derp", slog.F("address", err), slog.Error(err)) + continue + } + address = derpURL.String() + total[address] = struct{}{} added, err := m.addAddress(address) if err != nil { From de5b13b380795544cb38cd697c07cd0c63a39e51 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 13 Oct 2022 21:35:41 +0000 Subject: [PATCH 23/79] Add the ability to block endpoints --- .../coderd/coderdenttest/coderdenttest.go | 2 +- enterprise/coderd/replicas_test.go | 80 ++++++++++++------- tailnet/conn.go | 20 +++-- 3 files changed, 67 insertions(+), 35 deletions(-) diff --git a/enterprise/coderd/coderdenttest/coderdenttest.go b/enterprise/coderd/coderdenttest/coderdenttest.go index 24f0bffd5017f..ea172c43116e4 100644 --- a/enterprise/coderd/coderdenttest/coderdenttest.go +++ b/enterprise/coderd/coderdenttest/coderdenttest.go @@ -68,7 +68,7 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c BrowserOnly: options.BrowserOnly, SCIMAPIKey: options.SCIMAPIKey, DERPServerRelayAddress: oop.AccessURL.String(), - DERPServerRegionID: 1, + DERPServerRegionID: oop.DERPMap.RegionIDs()[0], ReplicaID: uuid.New(), UserWorkspaceQuota: options.UserWorkspaceQuota, Options: oop, diff --git a/enterprise/coderd/replicas_test.go b/enterprise/coderd/replicas_test.go index 52836a720f623..01c6be90199f0 100644 --- a/enterprise/coderd/replicas_test.go +++ b/enterprise/coderd/replicas_test.go @@ -3,7 +3,6 @@ package coderd_test import ( "context" "testing" - "time" "github.com/stretchr/testify/require" @@ -13,38 +12,63 @@ import ( "github.com/coder/coder/coderd/coderdtest" "github.com/coder/coder/coderd/database/dbtestutil" "github.com/coder/coder/enterprise/coderd/coderdenttest" + "github.com/coder/coder/testutil" ) func TestReplicas(t *testing.T) { t.Parallel() - db, pubsub := dbtestutil.NewDB(t) - firstClient := coderdenttest.New(t, &coderdenttest.Options{ - Options: &coderdtest.Options{ - IncludeProvisionerDaemon: true, - Database: db, - Pubsub: pubsub, - }, + t.Run("WarningsWithoutLicense", func(t *testing.T) { + t.Parallel() + db, pubsub := dbtestutil.NewDB(t) + firstClient := coderdenttest.New(t, &coderdenttest.Options{ + Options: &coderdtest.Options{ + IncludeProvisionerDaemon: true, + Database: db, + Pubsub: pubsub, + }, + }) + _ = coderdtest.CreateFirstUser(t, firstClient) + secondClient := coderdenttest.New(t, &coderdenttest.Options{ + Options: &coderdtest.Options{ + Database: db, + Pubsub: pubsub, + }, + }) + secondClient.SessionToken = firstClient.SessionToken + ents, err := secondClient.Entitlements(context.Background()) + require.NoError(t, err) + require.Len(t, ents.Warnings, 1) }) - firstUser := coderdtest.CreateFirstUser(t, firstClient) - coderdenttest.AddLicense(t, firstClient, coderdenttest.LicenseOptions{ - HighAvailability: true, - }) - - secondClient := coderdenttest.New(t, &coderdenttest.Options{ - Options: &coderdtest.Options{ - Database: db, - Pubsub: pubsub, - }, - }) - secondClient.SessionToken = firstClient.SessionToken + t.Run("ConnectAcrossMultiple", func(t *testing.T) { + t.Parallel() + db, pubsub := dbtestutil.NewDB(t) + firstClient := coderdenttest.New(t, &coderdenttest.Options{ + Options: &coderdtest.Options{ + IncludeProvisionerDaemon: true, + Database: db, + Pubsub: pubsub, + }, + }) + firstUser := coderdtest.CreateFirstUser(t, firstClient) + coderdenttest.AddLicense(t, firstClient, coderdenttest.LicenseOptions{ + HighAvailability: true, + }) - agentID := setupWorkspaceAgent(t, firstClient, firstUser) - conn, err := secondClient.DialWorkspaceAgentTailnet(context.Background(), slogtest.Make(t, nil).Leveled(slog.LevelDebug), agentID) - require.NoError(t, err) - require.Eventually(t, func() bool { - _, err = conn.Ping() - return err == nil - }, 10*time.Second, 250*time.Millisecond) + secondClient := coderdenttest.New(t, &coderdenttest.Options{ + Options: &coderdtest.Options{ + Database: db, + Pubsub: pubsub, + }, + }) + secondClient.SessionToken = firstClient.SessionToken - _ = conn.Close() + agentID := setupWorkspaceAgent(t, firstClient, firstUser) + conn, err := secondClient.DialWorkspaceAgentTailnet(context.Background(), slogtest.Make(t, nil).Leveled(slog.LevelDebug), agentID) + require.NoError(t, err) + require.Eventually(t, func() bool { + _, err = conn.Ping() + return err == nil + }, testutil.WaitShort, testutil.IntervalFast) + _ = conn.Close() + }) } diff --git a/tailnet/conn.go b/tailnet/conn.go index 1b454d6346b97..19a0cd50f49e6 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -48,7 +48,10 @@ type Options struct { Addresses []netip.Prefix DERPMap *tailcfg.DERPMap - Logger slog.Logger + // BlockEndpoints specifies whether P2P endpoints are blocked. + // If so, only DERPs can establish connections. + BlockEndpoints bool + Logger slog.Logger } // NewConn constructs a new Wireguard server that will accept connections from the addresses provided. @@ -175,6 +178,7 @@ func NewConn(options *Options) (*Conn, error) { wireguardEngine.SetFilter(filter.New(netMap.PacketFilter, localIPs, logIPs, nil, Logger(options.Logger.Named("packet-filter")))) dialContext, dialCancel := context.WithCancel(context.Background()) server := &Conn{ + blockEndpoints: options.BlockEndpoints, dialContext: dialContext, dialCancel: dialCancel, closed: make(chan struct{}), @@ -240,11 +244,12 @@ func IP() netip.Addr { // Conn is an actively listening Wireguard connection. type Conn struct { - dialContext context.Context - dialCancel context.CancelFunc - mutex sync.Mutex - closed chan struct{} - logger slog.Logger + dialContext context.Context + dialCancel context.CancelFunc + mutex sync.Mutex + closed chan struct{} + logger slog.Logger + blockEndpoints bool dialer *tsdial.Dialer tunDevice *tstun.Wrapper @@ -429,6 +434,9 @@ func (c *Conn) sendNode() { PreferredDERP: c.lastPreferredDERP, DERPLatency: c.lastDERPLatency, } + if c.blockEndpoints { + node.Endpoints = nil + } nodeCallback := c.nodeCallback if nodeCallback == nil { return From 9a50ac496ef1f0329f15add7ab4e58652f03b184 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Fri, 14 Oct 2022 15:37:38 +0000 Subject: [PATCH 24/79] Add flag to disable P2P connections --- cli/agent_test.go | 8 +++----- cli/configssh_test.go | 3 +-- cli/portforward.go | 3 +-- cli/speedtest.go | 4 +++- cli/ssh.go | 4 +--- coderd/activitybump_test.go | 4 +++- coderd/coderd.go | 2 +- coderd/templates_test.go | 4 +++- coderd/workspaceagents_test.go | 6 ++++-- codersdk/workspaceagents.go | 23 +++++++++++++++-------- enterprise/coderd/coderd.go | 4 +++- enterprise/coderd/replicas_test.go | 9 ++++++--- enterprise/coderd/workspaceagents_test.go | 5 ++--- enterprise/derpmesh/derpmesh.go | 9 +++++++-- tailnet/conn.go | 3 +++ 15 files changed, 56 insertions(+), 35 deletions(-) diff --git a/cli/agent_test.go b/cli/agent_test.go index dd0cb1d789349..8a90bb4cada3b 100644 --- a/cli/agent_test.go +++ b/cli/agent_test.go @@ -7,8 +7,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "cdr.dev/slog" - "github.com/coder/coder/cli/clitest" "github.com/coder/coder/coderd/coderdtest" "github.com/coder/coder/provisioner/echo" @@ -67,7 +65,7 @@ func TestWorkspaceAgent(t *testing.T) { if assert.NotEmpty(t, workspace.LatestBuild.Resources) && assert.NotEmpty(t, resources[0].Agents) { assert.NotEmpty(t, resources[0].Agents[0].Version) } - dialer, err := client.DialWorkspaceAgentTailnet(ctx, slog.Logger{}, resources[0].Agents[0].ID) + dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil) require.NoError(t, err) defer dialer.Close() require.Eventually(t, func() bool { @@ -128,7 +126,7 @@ func TestWorkspaceAgent(t *testing.T) { if assert.NotEmpty(t, resources) && assert.NotEmpty(t, resources[0].Agents) { assert.NotEmpty(t, resources[0].Agents[0].Version) } - dialer, err := client.DialWorkspaceAgentTailnet(ctx, slog.Logger{}, resources[0].Agents[0].ID) + dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil) require.NoError(t, err) defer dialer.Close() require.Eventually(t, func() bool { @@ -189,7 +187,7 @@ func TestWorkspaceAgent(t *testing.T) { if assert.NotEmpty(t, resources) && assert.NotEmpty(t, resources[0].Agents) { assert.NotEmpty(t, resources[0].Agents[0].Version) } - dialer, err := client.DialWorkspaceAgentTailnet(ctx, slog.Logger{}, resources[0].Agents[0].ID) + dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil) require.NoError(t, err) defer dialer.Close() require.Eventually(t, func() bool { diff --git a/cli/configssh_test.go b/cli/configssh_test.go index 3e1512a0c3471..4553cbe431221 100644 --- a/cli/configssh_test.go +++ b/cli/configssh_test.go @@ -19,7 +19,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "cdr.dev/slog" "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/agent" @@ -115,7 +114,7 @@ func TestConfigSSH(t *testing.T) { _ = agentCloser.Close() }() resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID) - agentConn, err := client.DialWorkspaceAgentTailnet(context.Background(), slog.Logger{}, resources[0].Agents[0].ID) + agentConn, err := client.DialWorkspaceAgent(context.Background(), resources[0].Agents[0].ID, nil) require.NoError(t, err) defer agentConn.Close() diff --git a/cli/portforward.go b/cli/portforward.go index 476809d601558..9cd3bc317c3b4 100644 --- a/cli/portforward.go +++ b/cli/portforward.go @@ -16,7 +16,6 @@ import ( "github.com/spf13/cobra" "golang.org/x/xerrors" - "cdr.dev/slog" "github.com/coder/coder/agent" "github.com/coder/coder/cli/cliflag" "github.com/coder/coder/cli/cliui" @@ -96,7 +95,7 @@ func portForward() *cobra.Command { return xerrors.Errorf("await agent: %w", err) } - conn, err := client.DialWorkspaceAgentTailnet(ctx, slog.Logger{}, workspaceAgent.ID) + conn, err := client.DialWorkspaceAgent(ctx, workspaceAgent.ID, nil) if err != nil { return err } diff --git a/cli/speedtest.go b/cli/speedtest.go index 357048f63ea34..cbb226b341342 100644 --- a/cli/speedtest.go +++ b/cli/speedtest.go @@ -55,7 +55,9 @@ func speedtest() *cobra.Command { if cliflag.IsSetBool(cmd, varVerbose) { logger = logger.Leveled(slog.LevelDebug) } - conn, err := client.DialWorkspaceAgentTailnet(ctx, logger, workspaceAgent.ID) + conn, err := client.DialWorkspaceAgent(ctx, workspaceAgent.ID, &codersdk.DialWorkspaceAgentOptions{ + Logger: logger, + }) if err != nil { return err } diff --git a/cli/ssh.go b/cli/ssh.go index ef8538764e3ac..b4d4f6420da78 100644 --- a/cli/ssh.go +++ b/cli/ssh.go @@ -20,8 +20,6 @@ import ( "golang.org/x/term" "golang.org/x/xerrors" - "cdr.dev/slog" - "github.com/coder/coder/cli/cliflag" "github.com/coder/coder/cli/cliui" "github.com/coder/coder/coderd/autobuild/notify" @@ -86,7 +84,7 @@ func ssh() *cobra.Command { return xerrors.Errorf("await agent: %w", err) } - conn, err := client.DialWorkspaceAgentTailnet(ctx, slog.Logger{}, workspaceAgent.ID) + conn, err := client.DialWorkspaceAgent(ctx, workspaceAgent.ID, nil) if err != nil { return err } diff --git a/coderd/activitybump_test.go b/coderd/activitybump_test.go index b12c8bc170a29..746bef0c9994e 100644 --- a/coderd/activitybump_test.go +++ b/coderd/activitybump_test.go @@ -74,7 +74,9 @@ func TestWorkspaceActivityBump(t *testing.T) { client, workspace, assertBumped := setupActivityTest(t) resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID) - conn, err := client.DialWorkspaceAgentTailnet(ctx, slogtest.Make(t, nil), resources[0].Agents[0].ID) + conn, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, &codersdk.DialWorkspaceAgentOptions{ + Logger: slogtest.Make(t, nil), + }) require.NoError(t, err) defer conn.Close() diff --git a/coderd/coderd.go b/coderd/coderd.go index 6b4b335161c32..da1fc0572ccc6 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -123,7 +123,7 @@ func New(options *Options) *API { options.TailnetCoordinator = tailnet.NewCoordinator() } if options.DERPServer == nil { - options.DERPServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger)) + options.DERPServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger.Named("derp").Leveled(slog.LevelDebug))) options.DERPServer.SetMeshKey("todo-kyle-change-this") } if options.Auditor == nil { diff --git a/coderd/templates_test.go b/coderd/templates_test.go index bf547c4d0eb9a..861ad6f459035 100644 --- a/coderd/templates_test.go +++ b/coderd/templates_test.go @@ -626,7 +626,9 @@ func TestTemplateDAUs(t *testing.T) { require.NoError(t, err) assert.Zero(t, workspaces[0].LastUsedAt) - conn, err := client.DialWorkspaceAgentTailnet(ctx, slogtest.Make(t, nil).Named("tailnet"), resources[0].Agents[0].ID) + conn, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, &codersdk.DialWorkspaceAgentOptions{ + Logger: slogtest.Make(t, nil).Named("tailnet"), + }) require.NoError(t, err) defer func() { _ = conn.Close() diff --git a/coderd/workspaceagents_test.go b/coderd/workspaceagents_test.go index 6bd569dde9f71..c5f3d9f16c0d8 100644 --- a/coderd/workspaceagents_test.go +++ b/coderd/workspaceagents_test.go @@ -123,7 +123,7 @@ func TestWorkspaceAgentListen(t *testing.T) { defer cancel() resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID) - conn, err := client.DialWorkspaceAgentTailnet(ctx, slog.Logger{}, resources[0].Agents[0].ID) + conn, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil) require.NoError(t, err) defer func() { _ = conn.Close() @@ -253,7 +253,9 @@ func TestWorkspaceAgentTailnet(t *testing.T) { ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() - conn, err := client.DialWorkspaceAgentTailnet(ctx, slogtest.Make(t, nil).Named("client").Leveled(slog.LevelDebug), resources[0].Agents[0].ID) + conn, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, &codersdk.DialWorkspaceAgentOptions{ + Logger: slogtest.Make(t, nil).Named("client").Leveled(slog.LevelDebug), + }) require.NoError(t, err) defer conn.Close() sshClient, err := conn.SSHClient() diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index 81f82b08d3efa..97d225c3eebb3 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -331,7 +331,13 @@ func (c *Client) ListenWorkspaceAgentTailnet(ctx context.Context) (net.Conn, err return websocket.NetConn(ctx, conn, websocket.MessageBinary), nil } -func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logger, agentID uuid.UUID) (*AgentConn, error) { +type DialWorkspaceAgentOptions struct { + Logger slog.Logger + // BlockEndpoints forced a direct connection through DERP. + BlockEndpoints bool +} + +func (c *Client) DialWorkspaceAgent(ctx context.Context, agentID uuid.UUID, options *DialWorkspaceAgentOptions) (*AgentConn, error) { res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/workspaceagents/%s/connection", agentID), nil) if err != nil { return nil, err @@ -348,9 +354,10 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg ip := tailnet.IP() conn, err := tailnet.NewConn(&tailnet.Options{ - Addresses: []netip.Prefix{netip.PrefixFrom(ip, 128)}, - DERPMap: connInfo.DERPMap, - Logger: logger, + Addresses: []netip.Prefix{netip.PrefixFrom(ip, 128)}, + DERPMap: connInfo.DERPMap, + Logger: options.Logger, + BlockEndpoints: options.BlockEndpoints, }) if err != nil { return nil, xerrors.Errorf("create tailnet: %w", err) @@ -378,7 +385,7 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg defer close(closed) isFirst := true for retrier := retry.New(50*time.Millisecond, 10*time.Second); retrier.Wait(ctx); { - logger.Debug(ctx, "connecting") + options.Logger.Debug(ctx, "connecting") // nolint:bodyclose ws, res, err := websocket.Dial(ctx, coordinateURL.String(), &websocket.DialOptions{ HTTPClient: httpClient, @@ -397,21 +404,21 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg if errors.Is(err, context.Canceled) { return } - logger.Debug(ctx, "failed to dial", slog.Error(err)) + options.Logger.Debug(ctx, "failed to dial", slog.Error(err)) continue } sendNode, errChan := tailnet.ServeCoordinator(websocket.NetConn(ctx, ws, websocket.MessageBinary), func(node []*tailnet.Node) error { return conn.UpdateNodes(node) }) conn.SetNodeCallback(sendNode) - logger.Debug(ctx, "serving coordinator") + options.Logger.Debug(ctx, "serving coordinator") err = <-errChan if errors.Is(err, context.Canceled) { _ = ws.Close(websocket.StatusGoingAway, "") return } if err != nil { - logger.Debug(ctx, "error serving coordinator", slog.Error(err)) + options.Logger.Debug(ctx, "error serving coordinator", slog.Error(err)) _ = ws.Close(websocket.StatusGoingAway, "") continue } diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index b06e843b658a5..252da9ac6f01a 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -3,6 +3,7 @@ package coderd import ( "context" "crypto/ed25519" + "fmt" "net/http" "sync" "time" @@ -126,7 +127,7 @@ func New(ctx context.Context, options *Options) (*API, error) { if err != nil { return nil, xerrors.Errorf("initialize replica: %w", err) } - api.derpMesh = derpmesh.New(options.Logger, api.DERPServer) + api.derpMesh = derpmesh.New(options.Logger.Named("derpmesh"), api.DERPServer) err = api.updateEntitlements(ctx) if err != nil { @@ -246,6 +247,7 @@ func (api *API) updateEntitlements(ctx context.Context) error { coordinator = haCoordinator } + fmt.Printf("HA enabled\n") api.replicaManager.SetCallback(func() { addresses := make([]string, 0) for _, replica := range api.replicaManager.Regional() { diff --git a/enterprise/coderd/replicas_test.go b/enterprise/coderd/replicas_test.go index 01c6be90199f0..1d60b24e6e81a 100644 --- a/enterprise/coderd/replicas_test.go +++ b/enterprise/coderd/replicas_test.go @@ -11,6 +11,7 @@ import ( "github.com/coder/coder/coderd/coderdtest" "github.com/coder/coder/coderd/database/dbtestutil" + "github.com/coder/coder/codersdk" "github.com/coder/coder/enterprise/coderd/coderdenttest" "github.com/coder/coder/testutil" ) @@ -61,14 +62,16 @@ func TestReplicas(t *testing.T) { }, }) secondClient.SessionToken = firstClient.SessionToken - agentID := setupWorkspaceAgent(t, firstClient, firstUser) - conn, err := secondClient.DialWorkspaceAgentTailnet(context.Background(), slogtest.Make(t, nil).Leveled(slog.LevelDebug), agentID) + conn, err := secondClient.DialWorkspaceAgent(context.Background(), agentID, &codersdk.DialWorkspaceAgentOptions{ + BlockEndpoints: true, + Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug), + }) require.NoError(t, err) require.Eventually(t, func() bool { _, err = conn.Ping() return err == nil - }, testutil.WaitShort, testutil.IntervalFast) + }, testutil.WaitLong, testutil.IntervalFast) _ = conn.Close() }) } diff --git a/enterprise/coderd/workspaceagents_test.go b/enterprise/coderd/workspaceagents_test.go index 24e24e3f5f540..a5250b3b81b44 100644 --- a/enterprise/coderd/workspaceagents_test.go +++ b/enterprise/coderd/workspaceagents_test.go @@ -8,7 +8,6 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/require" - "cdr.dev/slog" "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/agent" "github.com/coder/coder/coderd/coderdtest" @@ -33,7 +32,7 @@ func TestBlockNonBrowser(t *testing.T) { BrowserOnly: true, }) id := setupWorkspaceAgent(t, client, user) - _, err := client.DialWorkspaceAgentTailnet(context.Background(), slog.Logger{}, id) + _, err := client.DialWorkspaceAgent(context.Background(), id, nil) var apiErr *codersdk.Error require.ErrorAs(t, err, &apiErr) require.Equal(t, http.StatusConflict, apiErr.StatusCode()) @@ -50,7 +49,7 @@ func TestBlockNonBrowser(t *testing.T) { BrowserOnly: false, }) id := setupWorkspaceAgent(t, client, user) - conn, err := client.DialWorkspaceAgentTailnet(context.Background(), slog.Logger{}, id) + conn, err := client.DialWorkspaceAgent(context.Background(), id, nil) require.NoError(t, err) _ = conn.Close() }) diff --git a/enterprise/derpmesh/derpmesh.go b/enterprise/derpmesh/derpmesh.go index 3ce22c1bd9a11..dbdf7bc1b1f3a 100644 --- a/enterprise/derpmesh/derpmesh.go +++ b/enterprise/derpmesh/derpmesh.go @@ -2,6 +2,7 @@ package derpmesh import ( "context" + "net" "net/url" "sync" @@ -88,11 +89,15 @@ func (m *Mesh) addAddress(address string) (bool, error) { if isActive { return false, nil } - client, err := derphttp.NewClient(m.server.PrivateKey(), address, tailnet.Logger(m.logger)) + client, err := derphttp.NewClient(m.server.PrivateKey(), address, tailnet.Logger(m.logger.Named("client"))) if err != nil { return false, xerrors.Errorf("create derp client: %w", err) } client.MeshKey = m.server.MeshKey() + client.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) { + var dialer net.Dialer + return dialer.DialContext(ctx, network, addr) + }) ctx, cancelFunc := context.WithCancel(m.ctx) closed := make(chan struct{}) closeFunc := func() { @@ -103,7 +108,7 @@ func (m *Mesh) addAddress(address string) (bool, error) { m.active[address] = closeFunc go func() { defer close(closed) - client.RunWatchConnectionLoop(ctx, m.server.PublicKey(), tailnet.Logger(m.logger), func(np key.NodePublic) { + client.RunWatchConnectionLoop(ctx, m.server.PublicKey(), tailnet.Logger(m.logger.Named("loop")), func(np key.NodePublic) { m.server.AddPacketForwarder(np, client) }, func(np key.NodePublic) { m.server.RemovePacketForwarder(np, client) diff --git a/tailnet/conn.go b/tailnet/conn.go index 19a0cd50f49e6..e41ed60a527f3 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -344,6 +344,9 @@ func (c *Conn) UpdateNodes(nodes []*Node) error { // reason. TODO: @kylecarbs debug this! KeepAlive: ok && peerStatus.Active, } + if c.blockEndpoints { + peerNode.Endpoints = nil + } c.peerMap[node.ID] = peerNode } c.netMap.Peers = make([]*tailcfg.Node, 0, len(c.peerMap)) From 6fa941f958ab91be3b30ae2679f58bfdd33ec9b2 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Fri, 14 Oct 2022 16:16:08 +0000 Subject: [PATCH 25/79] Wow, I made the tests pass --- agent/agent_test.go | 2 +- cli/agent_test.go | 6 +++--- cli/portforward.go | 2 +- cli/speedtest.go | 2 +- coderd/workspaceagents_test.go | 2 +- codersdk/agentconn.go | 4 +++- enterprise/coderd/coderd.go | 2 -- enterprise/coderd/replicas_test.go | 5 ++++- 8 files changed, 14 insertions(+), 11 deletions(-) diff --git a/agent/agent_test.go b/agent/agent_test.go index 06a33598b755f..e1269d6003922 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -465,7 +465,7 @@ func TestAgent(t *testing.T) { conn, _ := setupAgent(t, codersdk.WorkspaceAgentMetadata{}, 0) require.Eventually(t, func() bool { - _, err := conn.Ping() + _, err := conn.Ping(context.Background()) return err == nil }, testutil.WaitMedium, testutil.IntervalFast) conn1, err := conn.DialContext(context.Background(), l.Addr().Network(), l.Addr().String()) diff --git a/cli/agent_test.go b/cli/agent_test.go index 8a90bb4cada3b..f487ebfc005ed 100644 --- a/cli/agent_test.go +++ b/cli/agent_test.go @@ -69,7 +69,7 @@ func TestWorkspaceAgent(t *testing.T) { require.NoError(t, err) defer dialer.Close() require.Eventually(t, func() bool { - _, err := dialer.Ping() + _, err := dialer.Ping(ctx) return err == nil }, testutil.WaitMedium, testutil.IntervalFast) cancelFunc() @@ -130,7 +130,7 @@ func TestWorkspaceAgent(t *testing.T) { require.NoError(t, err) defer dialer.Close() require.Eventually(t, func() bool { - _, err := dialer.Ping() + _, err := dialer.Ping(ctx) return err == nil }, testutil.WaitMedium, testutil.IntervalFast) cancelFunc() @@ -191,7 +191,7 @@ func TestWorkspaceAgent(t *testing.T) { require.NoError(t, err) defer dialer.Close() require.Eventually(t, func() bool { - _, err := dialer.Ping() + _, err := dialer.Ping(ctx) return err == nil }, testutil.WaitMedium, testutil.IntervalFast) cancelFunc() diff --git a/cli/portforward.go b/cli/portforward.go index 9cd3bc317c3b4..5a6f4391dd897 100644 --- a/cli/portforward.go +++ b/cli/portforward.go @@ -155,7 +155,7 @@ func portForward() *cobra.Command { case <-ticker.C: } - _, err = conn.Ping() + _, err = conn.Ping(ctx) if err != nil { continue } diff --git a/cli/speedtest.go b/cli/speedtest.go index cbb226b341342..f6c06641ec26f 100644 --- a/cli/speedtest.go +++ b/cli/speedtest.go @@ -70,7 +70,7 @@ func speedtest() *cobra.Command { return ctx.Err() case <-ticker.C: } - dur, err := conn.Ping() + dur, err := conn.Ping(ctx) if err != nil { continue } diff --git a/coderd/workspaceagents_test.go b/coderd/workspaceagents_test.go index c5f3d9f16c0d8..e8dd772095736 100644 --- a/coderd/workspaceagents_test.go +++ b/coderd/workspaceagents_test.go @@ -129,7 +129,7 @@ func TestWorkspaceAgentListen(t *testing.T) { _ = conn.Close() }() require.Eventually(t, func() bool { - _, err := conn.Ping() + _, err := conn.Ping(ctx) return err == nil }, testutil.WaitLong, testutil.IntervalFast) }) diff --git a/codersdk/agentconn.go b/codersdk/agentconn.go index b11c440ce3a65..e75edf1ca6bb0 100644 --- a/codersdk/agentconn.go +++ b/codersdk/agentconn.go @@ -132,7 +132,7 @@ type AgentConn struct { CloseFunc func() } -func (c *AgentConn) Ping() (time.Duration, error) { +func (c *AgentConn) Ping(ctx context.Context) (time.Duration, error) { errCh := make(chan error, 1) durCh := make(chan time.Duration, 1) c.Conn.Ping(TailnetIP, tailcfg.PingDisco, func(pr *ipnstate.PingResult) { @@ -145,6 +145,8 @@ func (c *AgentConn) Ping() (time.Duration, error) { select { case err := <-errCh: return 0, err + case <-ctx.Done(): + return 0, ctx.Err() case dur := <-durCh: return dur, nil } diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 252da9ac6f01a..21bc6f497ee1f 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -3,7 +3,6 @@ package coderd import ( "context" "crypto/ed25519" - "fmt" "net/http" "sync" "time" @@ -247,7 +246,6 @@ func (api *API) updateEntitlements(ctx context.Context) error { coordinator = haCoordinator } - fmt.Printf("HA enabled\n") api.replicaManager.SetCallback(func() { addresses := make([]string, 0) for _, replica := range api.replicaManager.Regional() { diff --git a/enterprise/coderd/replicas_test.go b/enterprise/coderd/replicas_test.go index 1d60b24e6e81a..0da4a05dbbb60 100644 --- a/enterprise/coderd/replicas_test.go +++ b/enterprise/coderd/replicas_test.go @@ -3,6 +3,7 @@ package coderd_test import ( "context" "testing" + "time" "github.com/stretchr/testify/require" @@ -69,7 +70,9 @@ func TestReplicas(t *testing.T) { }) require.NoError(t, err) require.Eventually(t, func() bool { - _, err = conn.Ping() + ctx, cancelFunc := context.WithTimeout(context.Background(), 3*time.Second) + defer cancelFunc() + _, err = conn.Ping(ctx) return err == nil }, testutil.WaitLong, testutil.IntervalFast) _ = conn.Close() From abff96b103bcc4d6a72697154d95477bd9b69aed Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Fri, 14 Oct 2022 16:54:21 +0000 Subject: [PATCH 26/79] Add replicas endpoint --- coderd/rbac/object.go | 4 ++ codersdk/deployment.go | 26 ------------ codersdk/replicas.go | 42 +++++++++++++++++++ codersdk/workspaceagents.go | 4 ++ enterprise/cli/features_test.go | 2 +- enterprise/coderd/coderd.go | 4 ++ enterprise/coderd/coderd_test.go | 2 +- .../coderdenttest/coderdenttest_test.go | 4 ++ enterprise/coderd/replicas.go | 35 ++++++++++++++++ enterprise/coderd/replicas_test.go | 5 +++ enterprise/replicasync/replicasync.go | 8 ++++ enterprise/replicasync/replicasync_test.go | 36 ++++++++++------ site/src/api/typesGenerated.ts | 11 +++++ 13 files changed, 143 insertions(+), 40 deletions(-) delete mode 100644 codersdk/deployment.go create mode 100644 codersdk/replicas.go diff --git a/coderd/rbac/object.go b/coderd/rbac/object.go index 5492e4397d5f7..1a8861c984ce9 100644 --- a/coderd/rbac/object.go +++ b/coderd/rbac/object.go @@ -146,6 +146,10 @@ var ( ResourceDeploymentFlags = Object{ Type: "deployment_flags", } + + ResourceReplicas = Object{ + Type: "replicas", + } ) // Object is used to create objects for authz checks when you have none in diff --git a/codersdk/deployment.go b/codersdk/deployment.go deleted file mode 100644 index a1227b09e3f63..0000000000000 --- a/codersdk/deployment.go +++ /dev/null @@ -1,26 +0,0 @@ -package codersdk - -import ( - "time" - - "github.com/google/uuid" -) - -type DeploymentInfo struct { - Replicas []Replica `json:"replicas"` -} - -type Replica struct { - // ID is the unique identifier for the replica. - ID uuid.UUID `json:"id"` - // Hostname is the hostname of the replica. - Hostname string `json:"hostname"` - // CreatedAt is when the replica was first seen. - CreatedAt time.Time `json:"created_at"` - // Active determines whether the replica is online. - Active bool `json:"active"` - // RelayAddress is the accessible address to relay DERP connections. - RelayAddress string `json:"relay_address"` - // Error is the error. - Error string `json:"error"` -} diff --git a/codersdk/replicas.go b/codersdk/replicas.go new file mode 100644 index 0000000000000..8e698fd3e6345 --- /dev/null +++ b/codersdk/replicas.go @@ -0,0 +1,42 @@ +package codersdk + +import ( + "context" + "encoding/json" + "net/http" + "time" + + "github.com/google/uuid" + "golang.org/x/xerrors" +) + +type Replica struct { + // ID is the unique identifier for the replica. + ID uuid.UUID `json:"id"` + // Hostname is the hostname of the replica. + Hostname string `json:"hostname"` + // CreatedAt is when the replica was first seen. + CreatedAt time.Time `json:"created_at"` + // RelayAddress is the accessible address to relay DERP connections. + RelayAddress string `json:"relay_address"` + // RegionID is the region of the replica. + RegionID int32 `json:"region_id"` + // Error is the error. + Error string `json:"error"` +} + +// Replicas fetches the list of replicas. +func (c *Client) Replicas(ctx context.Context) ([]Replica, error) { + res, err := c.Request(ctx, http.MethodGet, "/api/v2/replicas", nil) + if err != nil { + return nil, xerrors.Errorf("execute request: %w", err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return nil, readBodyAsError(res) + } + + var replicas []Replica + return replicas, json.NewDecoder(res.Body).Decode(&replicas) +} diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index 97d225c3eebb3..c86b399e189ab 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -331,6 +331,7 @@ func (c *Client) ListenWorkspaceAgentTailnet(ctx context.Context) (net.Conn, err return websocket.NetConn(ctx, conn, websocket.MessageBinary), nil } +// @typescript-ignore DialWorkspaceAgentOptions type DialWorkspaceAgentOptions struct { Logger slog.Logger // BlockEndpoints forced a direct connection through DERP. @@ -338,6 +339,9 @@ type DialWorkspaceAgentOptions struct { } func (c *Client) DialWorkspaceAgent(ctx context.Context, agentID uuid.UUID, options *DialWorkspaceAgentOptions) (*AgentConn, error) { + if options == nil { + options = &DialWorkspaceAgentOptions{} + } res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/workspaceagents/%s/connection", agentID), nil) if err != nil { return nil, err diff --git a/enterprise/cli/features_test.go b/enterprise/cli/features_test.go index 1a59e095c3594..78b94a6509526 100644 --- a/enterprise/cli/features_test.go +++ b/enterprise/cli/features_test.go @@ -57,7 +57,7 @@ func TestFeaturesList(t *testing.T) { var entitlements codersdk.Entitlements err := json.Unmarshal(buf.Bytes(), &entitlements) require.NoError(t, err, "unmarshal JSON output") - assert.Len(t, entitlements.Features, 6) + assert.Len(t, entitlements.Features, 7) assert.Empty(t, entitlements.Warnings) assert.Equal(t, codersdk.EntitlementNotEntitled, entitlements.Features[codersdk.FeatureUserLimit].Entitlement) diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 21bc6f497ee1f..1634f82d45366 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -59,6 +59,10 @@ func New(ctx context.Context, options *Options) (*API, error) { api.AGPL.APIHandler.Group(func(r chi.Router) { r.Get("/entitlements", api.serveEntitlements) + r.Route("/replicas", func(r chi.Router) { + r.Use(apiKeyMiddleware) + r.Get("/", api.replicas) + }) r.Route("/licenses", func(r chi.Router) { r.Use(apiKeyMiddleware) r.Post("/", api.postLicense) diff --git a/enterprise/coderd/coderd_test.go b/enterprise/coderd/coderd_test.go index 40421450415a5..7b51845ff3986 100644 --- a/enterprise/coderd/coderd_test.go +++ b/enterprise/coderd/coderd_test.go @@ -85,7 +85,7 @@ func TestEntitlements(t *testing.T) { assert.False(t, res.HasLicense) al = res.Features[codersdk.FeatureAuditLog] assert.Equal(t, codersdk.EntitlementNotEntitled, al.Entitlement) - assert.True(t, al.Enabled) + assert.False(t, al.Enabled) }) t.Run("Pubsub", func(t *testing.T) { t.Parallel() diff --git a/enterprise/coderd/coderdenttest/coderdenttest_test.go b/enterprise/coderd/coderdenttest/coderdenttest_test.go index ef7657ee5301c..0c4e4b3568bf3 100644 --- a/enterprise/coderd/coderdenttest/coderdenttest_test.go +++ b/enterprise/coderd/coderdenttest/coderdenttest_test.go @@ -58,6 +58,10 @@ func TestAuthorizeAllEndpoints(t *testing.T) { AssertAction: rbac.ActionRead, AssertObject: rbac.ResourceLicense, } + assertRoute["GET:/api/v2/replicas"] = coderdtest.RouteCheck{ + AssertAction: rbac.ActionRead, + AssertObject: rbac.ResourceReplicas, + } assertRoute["DELETE:/api/v2/licenses/{id}"] = coderdtest.RouteCheck{ AssertAction: rbac.ActionDelete, AssertObject: rbac.ResourceLicense, diff --git a/enterprise/coderd/replicas.go b/enterprise/coderd/replicas.go index ddb2b8b672186..f8cb64fe553dc 100644 --- a/enterprise/coderd/replicas.go +++ b/enterprise/coderd/replicas.go @@ -1 +1,36 @@ package coderd + +import ( + "net/http" + + "github.com/coder/coder/coderd/database" + "github.com/coder/coder/coderd/httpapi" + "github.com/coder/coder/coderd/rbac" + "github.com/coder/coder/codersdk" +) + +// replicas returns the number of replicas that are active in Coder. +func (api *API) replicas(rw http.ResponseWriter, r *http.Request) { + if !api.AGPL.Authorize(r, rbac.ActionRead, rbac.ResourceReplicas) { + httpapi.ResourceNotFound(rw) + return + } + + replicas := api.replicaManager.All() + res := make([]codersdk.Replica, 0, len(replicas)) + for _, replica := range replicas { + res = append(res, convertReplica(replica)) + } + httpapi.Write(r.Context(), rw, http.StatusOK, res) +} + +func convertReplica(replica database.Replica) codersdk.Replica { + return codersdk.Replica{ + ID: replica.ID, + Hostname: replica.Hostname, + CreatedAt: replica.CreatedAt, + RelayAddress: replica.RelayAddress, + RegionID: replica.RegionID, + Error: replica.Error.String, + } +} diff --git a/enterprise/coderd/replicas_test.go b/enterprise/coderd/replicas_test.go index 0da4a05dbbb60..e51f9cc330dc8 100644 --- a/enterprise/coderd/replicas_test.go +++ b/enterprise/coderd/replicas_test.go @@ -63,6 +63,10 @@ func TestReplicas(t *testing.T) { }, }) secondClient.SessionToken = firstClient.SessionToken + replicas, err := secondClient.Replicas(context.Background()) + require.NoError(t, err) + require.Len(t, replicas, 2) + agentID := setupWorkspaceAgent(t, firstClient, firstUser) conn, err := secondClient.DialWorkspaceAgent(context.Background(), agentID, &codersdk.DialWorkspaceAgentOptions{ BlockEndpoints: true, @@ -76,5 +80,6 @@ func TestReplicas(t *testing.T) { return err == nil }, testutil.WaitLong, testutil.IntervalFast) _ = conn.Close() + }) } diff --git a/enterprise/replicasync/replicasync.go b/enterprise/replicasync/replicasync.go index 4d6038a694940..8b8327038e088 100644 --- a/enterprise/replicasync/replicasync.go +++ b/enterprise/replicasync/replicasync.go @@ -101,6 +101,14 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, pubsub data if err != nil { return nil, xerrors.Errorf("run replica: %w", err) } + peers := server.Regional() + if len(peers) > 0 { + self := server.Self() + if self.RelayAddress == "" { + return nil, xerrors.Errorf("a relay address must be specified when running multiple replicas in the same region") + } + } + err = server.subscribe(ctx) if err != nil { return nil, xerrors.Errorf("subscribe: %w", err) diff --git a/enterprise/replicasync/replicasync_test.go b/enterprise/replicasync/replicasync_test.go index 5ce774ea5f29a..ccacbeb310c23 100644 --- a/enterprise/replicasync/replicasync_test.go +++ b/enterprise/replicasync/replicasync_test.go @@ -5,7 +5,6 @@ import ( "net/http" "net/http/httptest" "sync" - "sync/atomic" "testing" "time" @@ -66,6 +65,25 @@ func TestReplica(t *testing.T) { _ = server.Close() require.NoError(t, err) }) + t.Run("ErrorsWithoutRelayAddress", func(t *testing.T) { + // Ensures that the replica reports a successful status for + // accessing all of its peers. + t.Parallel() + db, pubsub := dbtestutil.NewDB(t) + _, err := db.InsertReplica(context.Background(), database.InsertReplicaParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + StartedAt: database.Now(), + UpdatedAt: database.Now(), + Hostname: "something", + }) + require.NoError(t, err) + _, err = replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replicasync.Options{ + ID: uuid.New(), + }) + require.Error(t, err) + require.Equal(t, "a relay address must be specified when running multiple replicas in the same region", err.Error()) + }) t.Run("ConnectsToPeerReplica", func(t *testing.T) { // Ensures that the replica reports a successful status for // accessing all of its peers. @@ -85,7 +103,8 @@ func TestReplica(t *testing.T) { }) require.NoError(t, err) server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replicasync.Options{ - ID: uuid.New(), + ID: uuid.New(), + RelayAddress: "http://169.254.169.254", }) require.NoError(t, err) require.Len(t, server.Regional(), 1) @@ -96,12 +115,6 @@ func TestReplica(t *testing.T) { t.Run("ConnectsToFakePeerWithError", func(t *testing.T) { t.Parallel() db, pubsub := dbtestutil.NewDB(t) - var count atomic.Int32 - cancel, err := pubsub.Subscribe(replicasync.PubsubEvent, func(ctx context.Context, message []byte) { - count.Add(1) - }) - require.NoError(t, err) - defer cancel() peer, err := db.InsertReplica(context.Background(), database.InsertReplicaParams{ ID: uuid.New(), CreatedAt: database.Now(), @@ -113,16 +126,15 @@ func TestReplica(t *testing.T) { }) require.NoError(t, err) server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replicasync.Options{ - ID: uuid.New(), - PeerTimeout: 1 * time.Millisecond, + ID: uuid.New(), + PeerTimeout: 1 * time.Millisecond, + RelayAddress: "http://169.254.169.254", }) require.NoError(t, err) require.Len(t, server.Regional(), 1) require.Equal(t, peer.ID, server.Regional()[0].ID) require.True(t, server.Self().Error.Valid) require.Contains(t, server.Self().Error.String, "Failed to dial peers") - // Once for the initial creation of a replica, and another time for the error. - require.Equal(t, int32(2), count.Load()) _ = server.Close() }) t.Run("RefreshOnPublish", func(t *testing.T) { diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 72abae519b469..2289d2100be92 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -268,6 +268,7 @@ export interface DeploymentFlags { readonly derp_server_region_code: StringFlag readonly derp_server_region_name: StringFlag readonly derp_server_stun_address: StringArrayFlag + readonly derp_server_relay_address: StringFlag readonly derp_config_url: StringFlag readonly derp_config_path: StringFlag readonly prom_enabled: BoolFlag @@ -522,6 +523,16 @@ export interface PutExtendWorkspaceRequest { readonly deadline: string } +// From codersdk/replicas.go +export interface Replica { + readonly id: string + readonly hostname: string + readonly created_at: string + readonly relay_address: string + readonly region_id: number + readonly error: string +} + // From codersdk/error.go export interface Response { readonly message: string From d6ce2167a243349472929136e0d1fab032c97ee0 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Fri, 14 Oct 2022 17:23:59 +0000 Subject: [PATCH 27/79] Ensure close kills replica --- cli/root.go | 6 ++-- cli/server.go | 9 ++++-- coderd/coderd.go | 2 +- enterprise/cli/server.go | 31 +++---------------- enterprise/coderd/coderd.go | 11 +++++-- .../coderd/coderdenttest/coderdenttest.go | 2 -- enterprise/coderd/license/license.go | 2 +- enterprise/coderd/license/license_test.go | 2 +- enterprise/coderd/replicas_test.go | 7 ++++- 9 files changed, 32 insertions(+), 40 deletions(-) diff --git a/cli/root.go b/cli/root.go index e29aa534da0a8..91d4551916cc0 100644 --- a/cli/root.go +++ b/cli/root.go @@ -4,6 +4,7 @@ import ( "context" "flag" "fmt" + "io" "net/http" "net/url" "os" @@ -100,8 +101,9 @@ func Core() []*cobra.Command { } func AGPL() []*cobra.Command { - all := append(Core(), Server(deployment.Flags(), func(_ context.Context, _ config.Root, o *coderd.Options) (*coderd.API, error) { - return coderd.New(o), nil + all := append(Core(), Server(deployment.Flags(), func(_ context.Context, o *coderd.Options) (*coderd.API, io.Closer, error) { + api := coderd.New(o) + return api, api, nil })) return all } diff --git a/cli/server.go b/cli/server.go index fc5f131da3d7b..3a94716be064d 100644 --- a/cli/server.go +++ b/cli/server.go @@ -67,7 +67,7 @@ import ( ) // nolint:gocyclo -func Server(dflags *codersdk.DeploymentFlags, newAPI func(context.Context, config.Root, *coderd.Options) (*coderd.API, error)) *cobra.Command { +func Server(dflags *codersdk.DeploymentFlags, newAPI func(context.Context, *coderd.Options) (*coderd.API, io.Closer, error)) *cobra.Command { root := &cobra.Command{ Use: "server", Short: "Start a Coder server", @@ -463,11 +463,14 @@ func Server(dflags *codersdk.DeploymentFlags, newAPI func(context.Context, confi ), dflags.PromAddress.Value, "prometheus")() } - coderAPI, err := newAPI(ctx, config, options) + // We use a separate closer so the Enterprise API + // can have it's own close functions. This is cleaner + // than abstracting the Coder API itself. + coderAPI, closer, err := newAPI(ctx, options) if err != nil { return err } - defer coderAPI.Close() + defer closer.Close() client := codersdk.New(localURL) if dflags.TLSEnable.Value { diff --git a/coderd/coderd.go b/coderd/coderd.go index da1fc0572ccc6..bb16553e47c66 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -123,7 +123,7 @@ func New(options *Options) *API { options.TailnetCoordinator = tailnet.NewCoordinator() } if options.DERPServer == nil { - options.DERPServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger.Named("derp").Leveled(slog.LevelDebug))) + options.DERPServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger.Named("derp"))) options.DERPServer.SetMeshKey("todo-kyle-change-this") } if options.Auditor == nil { diff --git a/enterprise/cli/server.go b/enterprise/cli/server.go index cc44985e0a4d4..f3e99c1613ab8 100644 --- a/enterprise/cli/server.go +++ b/enterprise/cli/server.go @@ -2,15 +2,12 @@ package cli import ( "context" + "io" "net/url" - "github.com/google/uuid" "github.com/spf13/cobra" "golang.org/x/xerrors" - "cdr.dev/slog" - - "github.com/coder/coder/cli/config" "github.com/coder/coder/cli/deployment" "github.com/coder/coder/enterprise/coderd" @@ -20,28 +17,11 @@ import ( func server() *cobra.Command { dflags := deployment.Flags() - cmd := agpl.Server(dflags, func(ctx context.Context, cfg config.Root, options *agplcoderd.Options) (*agplcoderd.API, error) { - replicaIDRaw, err := cfg.ReplicaID().Read() - generatedReplicaID := false - if err != nil { - replicaIDRaw = uuid.NewString() - generatedReplicaID = true - } - replicaID, err := uuid.Parse(replicaIDRaw) - if err != nil { - options.Logger.Warn(ctx, "failed to parse replica id", slog.Error(err), slog.F("replica_id", replicaIDRaw)) - replicaID = uuid.New() - generatedReplicaID = true - } - if generatedReplicaID { - // Make sure we save it to be reused later! - _ = cfg.ReplicaID().Write(replicaID.String()) - } - + cmd := agpl.Server(dflags, func(ctx context.Context, options *agplcoderd.Options) (*agplcoderd.API, io.Closer, error) { if dflags.DerpServerRelayAddress.Value != "" { _, err := url.Parse(dflags.DerpServerRelayAddress.Value) if err != nil { - return nil, xerrors.Errorf("derp-server-relay-address must be a valid HTTP URL: %w", err) + return nil, nil, xerrors.Errorf("derp-server-relay-address must be a valid HTTP URL: %w", err) } } @@ -51,7 +31,6 @@ func server() *cobra.Command { SCIMAPIKey: []byte(dflags.SCIMAuthHeader.Value), UserWorkspaceQuota: dflags.UserWorkspaceQuota.Value, RBAC: true, - ReplicaID: replicaID, DERPServerRelayAddress: dflags.DerpServerRelayAddress.Value, DERPServerRegionID: dflags.DerpServerRegionID.Value, @@ -59,9 +38,9 @@ func server() *cobra.Command { } api, err := coderd.New(ctx, o) if err != nil { - return nil, err + return nil, nil, err } - return api.AGPL, nil + return api.AGPL, api, nil }) deployment.AttachFlags(cmd.Flags(), dflags, true) diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 1634f82d45366..a25b432a16a7c 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -123,7 +123,8 @@ func New(ctx context.Context, options *Options) (*API, error) { var err error api.replicaManager, err = replicasync.New(ctx, options.Logger, options.Database, options.Pubsub, replicasync.Options{ - ID: options.ReplicaID, + // Create a new replica ID for each Coder instance! + ID: uuid.New(), RelayAddress: options.DERPServerRelayAddress, RegionID: int32(options.DERPServerRegionID), }) @@ -154,7 +155,6 @@ type Options struct { // Used for high availability. DERPServerRelayAddress string DERPServerRegionID int - ReplicaID uuid.UUID EntitlementsUpdateInterval time.Duration Keys map[string]ed25519.PublicKey @@ -256,10 +256,15 @@ func (api *API) updateEntitlements(ctx context.Context) error { addresses = append(addresses, replica.RelayAddress) } api.derpMesh.SetAddresses(addresses) + _ = api.updateEntitlements(ctx) }) } else { api.derpMesh.SetAddresses([]string{}) - api.replicaManager.SetCallback(func() {}) + api.replicaManager.SetCallback(func() { + // If the amount of replicas change, so should our entitlements. + // This is to display a warning in the UI if the user is unlicensed. + _ = api.updateEntitlements(ctx) + }) } // Recheck changed in case the HA coordinator failed to set up. diff --git a/enterprise/coderd/coderdenttest/coderdenttest.go b/enterprise/coderd/coderdenttest/coderdenttest.go index ea172c43116e4..fd1080a3ff30f 100644 --- a/enterprise/coderd/coderdenttest/coderdenttest.go +++ b/enterprise/coderd/coderdenttest/coderdenttest.go @@ -9,7 +9,6 @@ import ( "time" "github.com/golang-jwt/jwt/v4" - "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -69,7 +68,6 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c SCIMAPIKey: options.SCIMAPIKey, DERPServerRelayAddress: oop.AccessURL.String(), DERPServerRegionID: oop.DERPMap.RegionIDs()[0], - ReplicaID: uuid.New(), UserWorkspaceQuota: options.UserWorkspaceQuota, Options: oop, EntitlementsUpdateInterval: options.EntitlementsUpdateInterval, diff --git a/enterprise/coderd/license/license.go b/enterprise/coderd/license/license.go index 633b1a5056cab..f168f7472c80c 100644 --- a/enterprise/coderd/license/license.go +++ b/enterprise/coderd/license/license.go @@ -176,7 +176,7 @@ func Entitlements( "You have multiple replicas but your license is not entitled to high availability.") } else { entitlements.Warnings = append(entitlements.Warnings, - "You have multiple replicas but high availability is an Enterprise feature. Contact sales to get a license.") + "You have multiple replicas but high availability is an Enterprise feature.") } case codersdk.EntitlementGracePeriod: entitlements.Warnings = append(entitlements.Warnings, diff --git a/enterprise/coderd/license/license_test.go b/enterprise/coderd/license/license_test.go index 5b50bdb97cfe2..4d0f09913037d 100644 --- a/enterprise/coderd/license/license_test.go +++ b/enterprise/coderd/license/license_test.go @@ -228,7 +228,7 @@ func TestEntitlements(t *testing.T) { require.NoError(t, err) require.False(t, entitlements.HasLicense) require.Len(t, entitlements.Warnings, 1) - require.Equal(t, "You have multiple replicas but high availability is an Enterprise feature. Contact sales to get a license.", entitlements.Warnings[0]) + require.Equal(t, "You have multiple replicas but high availability is an Enterprise feature.", entitlements.Warnings[0]) }) t.Run("MultipleReplicasNotEntitled", func(t *testing.T) { diff --git a/enterprise/coderd/replicas_test.go b/enterprise/coderd/replicas_test.go index e51f9cc330dc8..3d41e83deb964 100644 --- a/enterprise/coderd/replicas_test.go +++ b/enterprise/coderd/replicas_test.go @@ -30,7 +30,7 @@ func TestReplicas(t *testing.T) { }, }) _ = coderdtest.CreateFirstUser(t, firstClient) - secondClient := coderdenttest.New(t, &coderdenttest.Options{ + secondClient, _, secondAPI := coderdenttest.NewWithAPI(t, &coderdenttest.Options{ Options: &coderdtest.Options{ Database: db, Pubsub: pubsub, @@ -40,6 +40,11 @@ func TestReplicas(t *testing.T) { ents, err := secondClient.Entitlements(context.Background()) require.NoError(t, err) require.Len(t, ents.Warnings, 1) + _ = secondAPI.Close() + + ents, err = firstClient.Entitlements(context.Background()) + require.NoError(t, err) + require.Len(t, ents.Warnings, 0) }) t.Run("ConnectAcrossMultiple", func(t *testing.T) { t.Parallel() From d7cc0ff9bb7255b150a84b4bb683ac90f87d4089 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Fri, 14 Oct 2022 22:36:51 +0000 Subject: [PATCH 28/79] Update sql --- cli/server.go | 10 +- ...icas.down.sql => 000061_replicas.down.sql} | 0 ...replicas.up.sql => 000061_replicas.up.sql} | 0 codersdk/features.go | 1 + enterprise/coderd/coderd.go | 2 +- enterprise/coderd/license/license.go | 11 ++- enterprise/coderd/license/license_test.go | 10 +- enterprise/derpmesh/derpmesh.go | 22 +++-- enterprise/derpmesh/derpmesh_test.go | 99 +++++++++++++++++-- go.mod | 2 +- go.sum | 4 +- 11 files changed, 124 insertions(+), 37 deletions(-) rename coderd/database/migrations/{000059_replicas.down.sql => 000061_replicas.down.sql} (100%) rename coderd/database/migrations/{000059_replicas.up.sql => 000061_replicas.up.sql} (100%) diff --git a/cli/server.go b/cli/server.go index 3a94716be064d..1ab1a6228f356 100644 --- a/cli/server.go +++ b/cli/server.go @@ -165,9 +165,10 @@ func Server(dflags *codersdk.DeploymentFlags, newAPI func(context.Context, *code } defer listener.Close() + var tlsConfig *tls.Config if dflags.TLSEnable.Value { - listener, err = configureServerTLS( - listener, dflags.TLSMinVersion.Value, + tlsConfig, err = configureTLS( + dflags.TLSMinVersion.Value, dflags.TLSClientAuth.Value, dflags.TLSCertFiles.Value, dflags.TLSKeyFiles.Value, @@ -176,6 +177,7 @@ func Server(dflags *codersdk.DeploymentFlags, newAPI func(context.Context, *code if err != nil { return xerrors.Errorf("configure tls: %w", err) } + listener = tls.NewListener(listener, tlsConfig) } tcpAddr, valid := listener.Addr().(*net.TCPAddr) @@ -888,7 +890,7 @@ func loadCertificates(tlsCertFiles, tlsKeyFiles []string) ([]tls.Certificate, er return certs, nil } -func configureServerTLS(listener net.Listener, tlsMinVersion, tlsClientAuth string, tlsCertFiles, tlsKeyFiles []string, tlsClientCAFile string) (net.Listener, error) { +func configureTLS(tlsMinVersion, tlsClientAuth string, tlsCertFiles, tlsKeyFiles []string, tlsClientCAFile string) (*tls.Config, error) { tlsConfig := &tls.Config{ MinVersion: tls.VersionTLS12, } @@ -958,7 +960,7 @@ func configureServerTLS(listener net.Listener, tlsMinVersion, tlsClientAuth stri tlsConfig.ClientCAs = caPool } - return tls.NewListener(listener, tlsConfig), nil + return tlsConfig, nil } func configureGithubOAuth2(accessURL *url.URL, clientID, clientSecret string, allowSignups bool, allowOrgs []string, rawTeams []string, enterpriseBaseURL string) (*coderd.GithubOAuth2Config, error) { diff --git a/coderd/database/migrations/000059_replicas.down.sql b/coderd/database/migrations/000061_replicas.down.sql similarity index 100% rename from coderd/database/migrations/000059_replicas.down.sql rename to coderd/database/migrations/000061_replicas.down.sql diff --git a/coderd/database/migrations/000059_replicas.up.sql b/coderd/database/migrations/000061_replicas.up.sql similarity index 100% rename from coderd/database/migrations/000059_replicas.up.sql rename to coderd/database/migrations/000061_replicas.up.sql diff --git a/codersdk/features.go b/codersdk/features.go index 799307e8fe898..862411de62872 100644 --- a/codersdk/features.go +++ b/codersdk/features.go @@ -44,6 +44,7 @@ type Feature struct { type Entitlements struct { Features map[string]Feature `json:"features"` Warnings []string `json:"warnings"` + Errors []string `json:"errors"` HasLicense bool `json:"has_license"` Experimental bool `json:"experimental"` Trial bool `json:"trial"` diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 294ff0eef1c71..612e710395722 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -137,7 +137,7 @@ func New(ctx context.Context, options *Options) (*API, error) { if err != nil { return nil, xerrors.Errorf("initialize replica: %w", err) } - api.derpMesh = derpmesh.New(options.Logger.Named("derpmesh"), api.DERPServer) + api.derpMesh = derpmesh.New(options.Logger.Named("derpmesh"), api.DERPServer, nil) err = api.updateEntitlements(ctx) if err != nil { diff --git a/enterprise/coderd/license/license.go b/enterprise/coderd/license/license.go index f168f7472c80c..c5bb689db65a9 100644 --- a/enterprise/coderd/license/license.go +++ b/enterprise/coderd/license/license.go @@ -30,6 +30,7 @@ func Entitlements( entitlements := codersdk.Entitlements{ Features: map[string]codersdk.Feature{}, Warnings: []string{}, + Errors: []string{}, } for _, featureName := range codersdk.FeatureNames { entitlements.Features[featureName] = codersdk.Feature{ @@ -172,15 +173,15 @@ func Entitlements( switch feature.Entitlement { case codersdk.EntitlementNotEntitled: if entitlements.HasLicense { - entitlements.Warnings = append(entitlements.Warnings, - "You have multiple replicas but your license is not entitled to high availability.") + entitlements.Errors = append(entitlements.Warnings, + "You have multiple replicas but your license is not entitled to high availability. You will be unable to connect to workspaces.") } else { - entitlements.Warnings = append(entitlements.Warnings, - "You have multiple replicas but high availability is an Enterprise feature.") + entitlements.Errors = append(entitlements.Warnings, + "You have multiple replicas but high availability is an Enterprise feature. You will be unable to connect to workspaces.") } case codersdk.EntitlementGracePeriod: entitlements.Warnings = append(entitlements.Warnings, - "You have multiple replicas but your license for high availability is expired.") + "You have multiple replicas but your license for high availability is expired. Reduce to one replica or workspace connections will stop working.") } } diff --git a/enterprise/coderd/license/license_test.go b/enterprise/coderd/license/license_test.go index 4d0f09913037d..6def291e3e24c 100644 --- a/enterprise/coderd/license/license_test.go +++ b/enterprise/coderd/license/license_test.go @@ -227,8 +227,8 @@ func TestEntitlements(t *testing.T) { entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 2, coderdenttest.Keys, all) require.NoError(t, err) require.False(t, entitlements.HasLicense) - require.Len(t, entitlements.Warnings, 1) - require.Equal(t, "You have multiple replicas but high availability is an Enterprise feature.", entitlements.Warnings[0]) + require.Len(t, entitlements.Errors, 1) + require.Equal(t, "You have multiple replicas but high availability is an Enterprise feature. You will be unable to connect to workspaces.", entitlements.Errors[0]) }) t.Run("MultipleReplicasNotEntitled", func(t *testing.T) { @@ -245,8 +245,8 @@ func TestEntitlements(t *testing.T) { }) require.NoError(t, err) require.True(t, entitlements.HasLicense) - require.Len(t, entitlements.Warnings, 1) - require.Equal(t, "You have multiple replicas but your license is not entitled to high availability.", entitlements.Warnings[0]) + require.Len(t, entitlements.Errors, 1) + require.Equal(t, "You have multiple replicas but your license is not entitled to high availability. You will be unable to connect to workspaces.", entitlements.Errors[0]) }) t.Run("MultipleReplicasGrace", func(t *testing.T) { @@ -266,6 +266,6 @@ func TestEntitlements(t *testing.T) { require.NoError(t, err) require.True(t, entitlements.HasLicense) require.Len(t, entitlements.Warnings, 1) - require.Equal(t, "You have multiple replicas but your license for high availability is expired.", entitlements.Warnings[0]) + require.Equal(t, "You have multiple replicas but your license for high availability is expired. Reduce to one replica or workspace connections will stop working.", entitlements.Warnings[0]) }) } diff --git a/enterprise/derpmesh/derpmesh.go b/enterprise/derpmesh/derpmesh.go index dbdf7bc1b1f3a..8f51343017593 100644 --- a/enterprise/derpmesh/derpmesh.go +++ b/enterprise/derpmesh/derpmesh.go @@ -2,6 +2,7 @@ package derpmesh import ( "context" + "crypto/tls" "net" "net/url" "sync" @@ -17,20 +18,22 @@ import ( ) // New constructs a new mesh for DERP servers. -func New(logger slog.Logger, server *derp.Server) *Mesh { +func New(logger slog.Logger, server *derp.Server, tlsConfig *tls.Config) *Mesh { return &Mesh{ - logger: logger, - server: server, - ctx: context.Background(), - closed: make(chan struct{}), - active: make(map[string]context.CancelFunc), + logger: logger, + server: server, + tlsConfig: tlsConfig, + ctx: context.Background(), + closed: make(chan struct{}), + active: make(map[string]context.CancelFunc), } } type Mesh struct { - logger slog.Logger - server *derp.Server - ctx context.Context + logger slog.Logger + server *derp.Server + ctx context.Context + tlsConfig *tls.Config mutex sync.Mutex closed chan struct{} @@ -93,6 +96,7 @@ func (m *Mesh) addAddress(address string) (bool, error) { if err != nil { return false, xerrors.Errorf("create derp client: %w", err) } + client.TLSConfig = m.tlsConfig client.MeshKey = m.server.MeshKey() client.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) { var dialer net.Dialer diff --git a/enterprise/derpmesh/derpmesh_test.go b/enterprise/derpmesh/derpmesh_test.go index 313c33da99bad..139e42566ffb1 100644 --- a/enterprise/derpmesh/derpmesh_test.go +++ b/enterprise/derpmesh/derpmesh_test.go @@ -1,11 +1,22 @@ package derpmesh_test import ( + "bytes" "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "errors" "io" + "math/big" + "net" "net/http/httptest" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -29,12 +40,41 @@ func TestDERPMesh(t *testing.T) { t.Run("ExchangeMessages", func(t *testing.T) { // This tests messages passing through multiple DERP servers. t.Parallel() - firstServer, firstServerURL := startDERP(t) + firstServer, firstServerURL, firstTLSName := startDERP(t) defer firstServer.Close() - secondServer, secondServerURL := startDERP(t) - firstMesh := derpmesh.New(slogtest.Make(t, nil).Named("first").Leveled(slog.LevelDebug), firstServer) + secondServer, secondServerURL, secondTLSName := startDERP(t) + firstMesh := derpmesh.New(slogtest.Make(t, nil).Named("first").Leveled(slog.LevelDebug), firstServer, firstTLSName) firstMesh.SetAddresses([]string{secondServerURL}) - secondMesh := derpmesh.New(slogtest.Make(t, nil).Named("second").Leveled(slog.LevelDebug), secondServer) + secondMesh := derpmesh.New(slogtest.Make(t, nil).Named("second").Leveled(slog.LevelDebug), secondServer, secondTLSName) + secondMesh.SetAddresses([]string{firstServerURL}) + defer firstMesh.Close() + defer secondMesh.Close() + + first := key.NewNode() + second := key.NewNode() + firstClient, err := derphttp.NewClient(first, secondServerURL, tailnet.Logger(slogtest.Make(t, nil))) + require.NoError(t, err) + secondClient, err := derphttp.NewClient(second, firstServerURL, tailnet.Logger(slogtest.Make(t, nil))) + require.NoError(t, err) + err = secondClient.Connect(context.Background()) + require.NoError(t, err) + + sent := []byte("hello world") + err = firstClient.Send(second.Public(), sent) + require.NoError(t, err) + + got := recvData(t, secondClient) + require.Equal(t, sent, got) + }) + t.Run("ExchangeMessages", func(t *testing.T) { + // This tests messages passing through multiple DERP servers. + t.Parallel() + firstServer, firstServerURL, firstTLSName := startDERP(t) + defer firstServer.Close() + secondServer, secondServerURL, secondTLSName := startDERP(t) + firstMesh := derpmesh.New(slogtest.Make(t, nil).Named("first").Leveled(slog.LevelDebug), firstServer, firstTLSName) + firstMesh.SetAddresses([]string{secondServerURL}) + secondMesh := derpmesh.New(slogtest.Make(t, nil).Named("second").Leveled(slog.LevelDebug), secondServer, secondTLSName) secondMesh.SetAddresses([]string{firstServerURL}) defer firstMesh.Close() defer secondMesh.Close() @@ -58,8 +98,8 @@ func TestDERPMesh(t *testing.T) { t.Run("RemoveAddress", func(t *testing.T) { // This tests messages passing through multiple DERP servers. t.Parallel() - server, serverURL := startDERP(t) - mesh := derpmesh.New(slogtest.Make(t, nil).Named("first").Leveled(slog.LevelDebug), server) + server, serverURL, tlsName := startDERP(t) + mesh := derpmesh.New(slogtest.Make(t, nil).Named("first").Leveled(slog.LevelDebug), server, tlsName) mesh.SetAddresses([]string{"http://fake.com"}) // This should trigger a removal... mesh.SetAddresses([]string{}) @@ -84,8 +124,8 @@ func TestDERPMesh(t *testing.T) { meshes := make([]*derpmesh.Mesh, 0, 20) serverURLs := make([]string, 0, 20) for i := 0; i < 20; i++ { - server, url := startDERP(t) - mesh := derpmesh.New(slogtest.Make(t, nil).Named("mesh").Leveled(slog.LevelDebug), server) + server, url, tlsName := startDERP(t) + mesh := derpmesh.New(slogtest.Make(t, nil).Named("mesh").Leveled(slog.LevelDebug), server, tlsName) t.Cleanup(func() { _ = server.Close() _ = mesh.Close() @@ -132,15 +172,54 @@ func recvData(t *testing.T, client *derphttp.Client) []byte { } } -func startDERP(t *testing.T) (*derp.Server, string) { +func startDERP(t *testing.T) (*derp.Server, string, *tls.Config) { logf := tailnet.Logger(slogtest.Make(t, nil)) d := derp.NewServer(key.NewNode(), logf) d.SetMeshKey("some-key") server := httptest.NewUnstartedServer(derphttp.Handler(d)) + commonName := "something.org" + server.TLS = &tls.Config{ + Certificates: []tls.Certificate{generateTLSCertificate(t, commonName)}, + } server.Start() t.Cleanup(func() { _ = d.Close() }) t.Cleanup(server.Close) - return d, server.URL + return d, server.URL, server.TLS +} + +func generateTLSCertificate(t testing.TB, commonName string) tls.Certificate { + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + Organization: []string{"Acme Co"}, + CommonName: commonName, + }, + DNSNames: []string{commonName}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(time.Hour * 24 * 180), + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + require.NoError(t, err) + var certFile bytes.Buffer + require.NoError(t, err) + _, err = certFile.Write(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})) + require.NoError(t, err) + privateKeyBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) + require.NoError(t, err) + var keyFile bytes.Buffer + err = pem.Encode(&keyFile, &pem.Block{Type: "PRIVATE KEY", Bytes: privateKeyBytes}) + require.NoError(t, err) + cert, err := tls.X509KeyPair(certFile.Bytes(), keyFile.Bytes()) + require.NoError(t, err) + return cert } diff --git a/go.mod b/go.mod index 9834e27e5f39c..b33a438eb3d08 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ replace github.com/tcnksm/go-httpstat => github.com/kylecarbs/go-httpstat v0.0.0 // There are a few minor changes we make to Tailscale that we're slowly upstreaming. Compare here: // https://github.com/tailscale/tailscale/compare/main...coder:tailscale:main -replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20220926024748-50f068456c6c +replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20221014173742-9f1da7795630 // Switch to our fork that imports fixes from http://github.com/tailscale/ssh. // See: https://github.com/coder/coder/issues/3371 diff --git a/go.sum b/go.sum index 13fdc5724f6b6..5852582c26c4a 100644 --- a/go.sum +++ b/go.sum @@ -351,8 +351,8 @@ github.com/coder/retry v1.3.0 h1:5lAAwt/2Cm6lVmnfBY7sOMXcBOwcwJhmV5QGSELIVWY= github.com/coder/retry v1.3.0/go.mod h1:tXuRgZgWjUnU5LZPT4lJh4ew2elUhexhlnXzrJWdyFY= github.com/coder/ssh v0.0.0-20220811105153-fcea99919338 h1:tN5GKFT68YLVzJoA8AHuiMNJ0qlhoD3pGN3JY9gxSko= github.com/coder/ssh v0.0.0-20220811105153-fcea99919338/go.mod h1:ZSS+CUoKHDrqVakTfTWUlKSr9MtMFkC4UvtQKD7O914= -github.com/coder/tailscale v1.1.1-0.20220926024748-50f068456c6c h1:xa6lr5Pj87Is26tgpzwBsEGKL7aVz7/fRGgY9QIbf3E= -github.com/coder/tailscale v1.1.1-0.20220926024748-50f068456c6c/go.mod h1:5amxy08qijEa8bcTW2SeIy4MIqcmd7LMsuOxqOlj2Ak= +github.com/coder/tailscale v1.1.1-0.20221014173742-9f1da7795630 h1:FgWWdu0fnFEpUNjW0vOaCuOxOZ/GQzn6oo7p5IMlSA0= +github.com/coder/tailscale v1.1.1-0.20221014173742-9f1da7795630/go.mod h1:5amxy08qijEa8bcTW2SeIy4MIqcmd7LMsuOxqOlj2Ak= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= From 9914840133605ac75f90357ae3801355cd90c91d Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 00:23:22 +0000 Subject: [PATCH 29/79] Add database latency to high availability --- coderd/database/databasefake/databasefake.go | 22 ++++--- coderd/database/db.go | 9 +++ coderd/database/dump.sql | 1 + .../migrations/000061_replicas.up.sql | 2 + coderd/database/models.go | 21 +++--- coderd/database/queries.sql.go | 57 +++++++++------- coderd/database/queries/replicas.sql | 9 +-- codersdk/replicas.go | 2 + enterprise/coderd/replicas.go | 13 ++-- enterprise/replicasync/replicasync.go | 64 ++++++++++-------- site/src/api/typesGenerated.ts | 1 + .../LicenseBanner/LicenseBanner.tsx | 6 +- .../LicenseBannerView.stories.tsx | 10 +++ .../LicenseBanner/LicenseBannerView.tsx | 66 +++++++++++-------- site/src/testHelpers/entities.ts | 3 + .../entitlements/entitlementsXService.ts | 1 + 16 files changed, 180 insertions(+), 107 deletions(-) diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index e58fb990271ca..b4724a9afe0aa 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -113,6 +113,10 @@ type data struct { lastLicenseID int32 } +func (q *fakeQuerier) Ping(_ context.Context) (time.Duration, error) { + return 0, nil +} + // InTx doesn't rollback data properly for in-memory yet. func (q *fakeQuerier) InTx(fn func(database.Store) error) error { q.mutex.Lock() @@ -3170,14 +3174,15 @@ func (q *fakeQuerier) InsertReplica(_ context.Context, arg database.InsertReplic defer q.mutex.Unlock() replica := database.Replica{ - ID: arg.ID, - CreatedAt: arg.CreatedAt, - StartedAt: arg.StartedAt, - UpdatedAt: arg.UpdatedAt, - Hostname: arg.Hostname, - RegionID: arg.RegionID, - RelayAddress: arg.RelayAddress, - Version: arg.Version, + ID: arg.ID, + CreatedAt: arg.CreatedAt, + StartedAt: arg.StartedAt, + UpdatedAt: arg.UpdatedAt, + Hostname: arg.Hostname, + RegionID: arg.RegionID, + RelayAddress: arg.RelayAddress, + Version: arg.Version, + DatabaseLatency: arg.DatabaseLatency, } q.replicas = append(q.replicas, replica) return replica, nil @@ -3199,6 +3204,7 @@ func (q *fakeQuerier) UpdateReplica(_ context.Context, arg database.UpdateReplic replica.RegionID = arg.RegionID replica.Version = arg.Version replica.Error = arg.Error + replica.DatabaseLatency = arg.DatabaseLatency q.replicas[index] = replica return replica, nil } diff --git a/coderd/database/db.go b/coderd/database/db.go index 4cbbdb399f193..020000888f8eb 100644 --- a/coderd/database/db.go +++ b/coderd/database/db.go @@ -12,6 +12,7 @@ import ( "context" "database/sql" "errors" + "time" "github.com/jmoiron/sqlx" "golang.org/x/xerrors" @@ -24,6 +25,7 @@ type Store interface { // customQuerier contains custom queries that are not generated. customQuerier + Ping(ctx context.Context) (time.Duration, error) InTx(func(Store) error) error } @@ -58,6 +60,13 @@ type sqlQuerier struct { db DBTX } +// Ping returns the time it takes to ping the database. +func (q *sqlQuerier) Ping(ctx context.Context) (time.Duration, error) { + start := time.Now() + err := q.sdb.PingContext(ctx) + return time.Since(start), err +} + // InTx performs database operations inside a transaction. func (q *sqlQuerier) InTx(function func(Store) error) error { if _, ok := q.db.(*sqlx.Tx); ok { diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index ca301ac8504b7..1e0a18c1dafef 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -293,6 +293,7 @@ CREATE TABLE replicas ( hostname text NOT NULL, region_id integer NOT NULL, relay_address text NOT NULL, + database_latency integer NOT NULL, version text NOT NULL, error text ); diff --git a/coderd/database/migrations/000061_replicas.up.sql b/coderd/database/migrations/000061_replicas.up.sql index a07587f35a234..b1d1a1ab13ee0 100644 --- a/coderd/database/migrations/000061_replicas.up.sql +++ b/coderd/database/migrations/000061_replicas.up.sql @@ -17,6 +17,8 @@ CREATE TABLE IF NOT EXISTS replicas ( region_id integer NOT NULL, -- An address that should be accessible to other replicas. relay_address text NOT NULL, + -- The latency of the replica to the database in microseconds. + database_latency int NOT NULL, -- Version is the Coder version of the replica. version text NOT NULL, error text diff --git a/coderd/database/models.go b/coderd/database/models.go index 55867a164bd98..b4601ecadeb78 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -540,16 +540,17 @@ type ProvisionerJobLog struct { } type Replica struct { - ID uuid.UUID `db:"id" json:"id"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - StartedAt time.Time `db:"started_at" json:"started_at"` - StoppedAt sql.NullTime `db:"stopped_at" json:"stopped_at"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` - Hostname string `db:"hostname" json:"hostname"` - RegionID int32 `db:"region_id" json:"region_id"` - RelayAddress string `db:"relay_address" json:"relay_address"` - Version string `db:"version" json:"version"` - Error sql.NullString `db:"error" json:"error"` + ID uuid.UUID `db:"id" json:"id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + StartedAt time.Time `db:"started_at" json:"started_at"` + StoppedAt sql.NullTime `db:"stopped_at" json:"stopped_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + Hostname string `db:"hostname" json:"hostname"` + RegionID int32 `db:"region_id" json:"region_id"` + RelayAddress string `db:"relay_address" json:"relay_address"` + DatabaseLatency int32 `db:"database_latency" json:"database_latency"` + Version string `db:"version" json:"version"` + Error sql.NullString `db:"error" json:"error"` } type SiteConfig struct { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 8577903ecc0a2..241474e7e66bd 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -2580,7 +2580,7 @@ func (q *sqlQuerier) DeleteReplicasUpdatedBefore(ctx context.Context, updatedAt } const getReplicaByID = `-- name: GetReplicaByID :one -SELECT id, created_at, started_at, stopped_at, updated_at, hostname, region_id, relay_address, version, error FROM replicas WHERE id = $1 +SELECT id, created_at, started_at, stopped_at, updated_at, hostname, region_id, relay_address, database_latency, version, error FROM replicas WHERE id = $1 ` func (q *sqlQuerier) GetReplicaByID(ctx context.Context, id uuid.UUID) (Replica, error) { @@ -2595,6 +2595,7 @@ func (q *sqlQuerier) GetReplicaByID(ctx context.Context, id uuid.UUID) (Replica, &i.Hostname, &i.RegionID, &i.RelayAddress, + &i.DatabaseLatency, &i.Version, &i.Error, ) @@ -2602,7 +2603,7 @@ func (q *sqlQuerier) GetReplicaByID(ctx context.Context, id uuid.UUID) (Replica, } const getReplicasUpdatedAfter = `-- name: GetReplicasUpdatedAfter :many -SELECT id, created_at, started_at, stopped_at, updated_at, hostname, region_id, relay_address, version, error FROM replicas WHERE updated_at > $1 AND stopped_at IS NULL +SELECT id, created_at, started_at, stopped_at, updated_at, hostname, region_id, relay_address, database_latency, version, error FROM replicas WHERE updated_at > $1 AND stopped_at IS NULL ` func (q *sqlQuerier) GetReplicasUpdatedAfter(ctx context.Context, updatedAt time.Time) ([]Replica, error) { @@ -2623,6 +2624,7 @@ func (q *sqlQuerier) GetReplicasUpdatedAfter(ctx context.Context, updatedAt time &i.Hostname, &i.RegionID, &i.RelayAddress, + &i.DatabaseLatency, &i.Version, &i.Error, ); err != nil { @@ -2648,20 +2650,21 @@ INSERT INTO replicas ( hostname, region_id, relay_address, - version - -) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id, created_at, started_at, stopped_at, updated_at, hostname, region_id, relay_address, version, error + version, + database_latency +) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING id, created_at, started_at, stopped_at, updated_at, hostname, region_id, relay_address, database_latency, version, error ` type InsertReplicaParams struct { - ID uuid.UUID `db:"id" json:"id"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - StartedAt time.Time `db:"started_at" json:"started_at"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` - Hostname string `db:"hostname" json:"hostname"` - RegionID int32 `db:"region_id" json:"region_id"` - RelayAddress string `db:"relay_address" json:"relay_address"` - Version string `db:"version" json:"version"` + ID uuid.UUID `db:"id" json:"id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + StartedAt time.Time `db:"started_at" json:"started_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + Hostname string `db:"hostname" json:"hostname"` + RegionID int32 `db:"region_id" json:"region_id"` + RelayAddress string `db:"relay_address" json:"relay_address"` + Version string `db:"version" json:"version"` + DatabaseLatency int32 `db:"database_latency" json:"database_latency"` } func (q *sqlQuerier) InsertReplica(ctx context.Context, arg InsertReplicaParams) (Replica, error) { @@ -2674,6 +2677,7 @@ func (q *sqlQuerier) InsertReplica(ctx context.Context, arg InsertReplicaParams) arg.RegionID, arg.RelayAddress, arg.Version, + arg.DatabaseLatency, ) var i Replica err := row.Scan( @@ -2685,6 +2689,7 @@ func (q *sqlQuerier) InsertReplica(ctx context.Context, arg InsertReplicaParams) &i.Hostname, &i.RegionID, &i.RelayAddress, + &i.DatabaseLatency, &i.Version, &i.Error, ) @@ -2700,20 +2705,22 @@ UPDATE replicas SET region_id = $6, hostname = $7, version = $8, - error = $9 -WHERE id = $1 RETURNING id, created_at, started_at, stopped_at, updated_at, hostname, region_id, relay_address, version, error + error = $9, + database_latency = $10 +WHERE id = $1 RETURNING id, created_at, started_at, stopped_at, updated_at, hostname, region_id, relay_address, database_latency, version, error ` type UpdateReplicaParams struct { - ID uuid.UUID `db:"id" json:"id"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` - StartedAt time.Time `db:"started_at" json:"started_at"` - StoppedAt sql.NullTime `db:"stopped_at" json:"stopped_at"` - RelayAddress string `db:"relay_address" json:"relay_address"` - RegionID int32 `db:"region_id" json:"region_id"` - Hostname string `db:"hostname" json:"hostname"` - Version string `db:"version" json:"version"` - Error sql.NullString `db:"error" json:"error"` + ID uuid.UUID `db:"id" json:"id"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + StartedAt time.Time `db:"started_at" json:"started_at"` + StoppedAt sql.NullTime `db:"stopped_at" json:"stopped_at"` + RelayAddress string `db:"relay_address" json:"relay_address"` + RegionID int32 `db:"region_id" json:"region_id"` + Hostname string `db:"hostname" json:"hostname"` + Version string `db:"version" json:"version"` + Error sql.NullString `db:"error" json:"error"` + DatabaseLatency int32 `db:"database_latency" json:"database_latency"` } func (q *sqlQuerier) UpdateReplica(ctx context.Context, arg UpdateReplicaParams) (Replica, error) { @@ -2727,6 +2734,7 @@ func (q *sqlQuerier) UpdateReplica(ctx context.Context, arg UpdateReplicaParams) arg.Hostname, arg.Version, arg.Error, + arg.DatabaseLatency, ) var i Replica err := row.Scan( @@ -2738,6 +2746,7 @@ func (q *sqlQuerier) UpdateReplica(ctx context.Context, arg UpdateReplicaParams) &i.Hostname, &i.RegionID, &i.RelayAddress, + &i.DatabaseLatency, &i.Version, &i.Error, ) diff --git a/coderd/database/queries/replicas.sql b/coderd/database/queries/replicas.sql index a7aa5b0aa1dee..5a62527fac107 100644 --- a/coderd/database/queries/replicas.sql +++ b/coderd/database/queries/replicas.sql @@ -13,9 +13,9 @@ INSERT INTO replicas ( hostname, region_id, relay_address, - version - -) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *; + version, + database_latency +) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING *; -- name: UpdateReplica :one UPDATE replicas SET @@ -26,7 +26,8 @@ UPDATE replicas SET region_id = $6, hostname = $7, version = $8, - error = $9 + error = $9, + database_latency = $10 WHERE id = $1 RETURNING *; -- name: DeleteReplicasUpdatedBefore :exec diff --git a/codersdk/replicas.go b/codersdk/replicas.go index 8e698fd3e6345..e74af021ee9a3 100644 --- a/codersdk/replicas.go +++ b/codersdk/replicas.go @@ -23,6 +23,8 @@ type Replica struct { RegionID int32 `json:"region_id"` // Error is the error. Error string `json:"error"` + // DatabaseLatency is the latency in microseconds to the database. + DatabaseLatency int32 `json:"database_latency"` } // Replicas fetches the list of replicas. diff --git a/enterprise/coderd/replicas.go b/enterprise/coderd/replicas.go index f8cb64fe553dc..c07c37243d0ca 100644 --- a/enterprise/coderd/replicas.go +++ b/enterprise/coderd/replicas.go @@ -26,11 +26,12 @@ func (api *API) replicas(rw http.ResponseWriter, r *http.Request) { func convertReplica(replica database.Replica) codersdk.Replica { return codersdk.Replica{ - ID: replica.ID, - Hostname: replica.Hostname, - CreatedAt: replica.CreatedAt, - RelayAddress: replica.RelayAddress, - RegionID: replica.RegionID, - Error: replica.Error.String, + ID: replica.ID, + Hostname: replica.Hostname, + CreatedAt: replica.CreatedAt, + RelayAddress: replica.RelayAddress, + RegionID: replica.RegionID, + Error: replica.Error.String, + DatabaseLatency: replica.DatabaseLatency, } } diff --git a/enterprise/replicasync/replicasync.go b/enterprise/replicasync/replicasync.go index 8b8327038e088..75ba041aaa6e1 100644 --- a/enterprise/replicasync/replicasync.go +++ b/enterprise/replicasync/replicasync.go @@ -48,6 +48,10 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, pubsub data if err != nil { return nil, xerrors.Errorf("get hostname: %w", err) } + databaseLatency, err := db.Ping(ctx) + if err != nil { + return nil, xerrors.Errorf("ping database: %w", err) + } var replica database.Replica _, err = db.GetReplicaByID(ctx, options.ID) if err != nil { @@ -55,29 +59,31 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, pubsub data return nil, xerrors.Errorf("get replica: %w", err) } replica, err = db.InsertReplica(ctx, database.InsertReplicaParams{ - ID: options.ID, - CreatedAt: database.Now(), - StartedAt: database.Now(), - UpdatedAt: database.Now(), - Hostname: hostname, - RegionID: options.RegionID, - RelayAddress: options.RelayAddress, - Version: buildinfo.Version(), + ID: options.ID, + CreatedAt: database.Now(), + StartedAt: database.Now(), + UpdatedAt: database.Now(), + Hostname: hostname, + RegionID: options.RegionID, + RelayAddress: options.RelayAddress, + Version: buildinfo.Version(), + DatabaseLatency: int32(databaseLatency.Microseconds()), }) if err != nil { return nil, xerrors.Errorf("insert replica: %w", err) } } else { replica, err = db.UpdateReplica(ctx, database.UpdateReplicaParams{ - ID: options.ID, - UpdatedAt: database.Now(), - StartedAt: database.Now(), - StoppedAt: sql.NullTime{}, - RelayAddress: options.RelayAddress, - RegionID: options.RegionID, - Hostname: hostname, - Version: buildinfo.Version(), - Error: sql.NullString{}, + ID: options.ID, + UpdatedAt: database.Now(), + StartedAt: database.Now(), + StoppedAt: sql.NullTime{}, + RelayAddress: options.RelayAddress, + RegionID: options.RegionID, + Hostname: hostname, + Version: buildinfo.Version(), + Error: sql.NullString{}, + DatabaseLatency: int32(databaseLatency.Microseconds()), }) if err != nil { return nil, xerrors.Errorf("update replica: %w", err) @@ -268,16 +274,22 @@ func (m *Manager) run(ctx context.Context) error { } } + databaseLatency, err := m.db.Ping(ctx) + if err != nil { + return xerrors.Errorf("ping database: %w", err) + } + replica, err := m.db.UpdateReplica(ctx, database.UpdateReplicaParams{ - ID: m.self.ID, - UpdatedAt: database.Now(), - StartedAt: m.self.StartedAt, - StoppedAt: m.self.StoppedAt, - RelayAddress: m.self.RelayAddress, - RegionID: m.self.RegionID, - Hostname: m.self.Hostname, - Version: m.self.Version, - Error: replicaError, + ID: m.self.ID, + UpdatedAt: database.Now(), + StartedAt: m.self.StartedAt, + StoppedAt: m.self.StoppedAt, + RelayAddress: m.self.RelayAddress, + RegionID: m.self.RegionID, + Hostname: m.self.Hostname, + Version: m.self.Version, + Error: replicaError, + DatabaseLatency: int32(databaseLatency.Microseconds()), }) if err != nil { return xerrors.Errorf("update replica: %w", err) diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 7a8af9278b1eb..92db958074a68 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -338,6 +338,7 @@ export interface DurationFlag { export interface Entitlements { readonly features: Record readonly warnings: string[] + readonly errors: string[] readonly has_license: boolean readonly experimental: boolean readonly trial: boolean diff --git a/site/src/components/LicenseBanner/LicenseBanner.tsx b/site/src/components/LicenseBanner/LicenseBanner.tsx index 8532bfca2ecbe..7ecfc2a2a2fac 100644 --- a/site/src/components/LicenseBanner/LicenseBanner.tsx +++ b/site/src/components/LicenseBanner/LicenseBanner.tsx @@ -8,15 +8,15 @@ export const LicenseBanner: React.FC = () => { const [entitlementsState, entitlementsSend] = useActor( xServices.entitlementsXService, ) - const { warnings } = entitlementsState.context.entitlements + const { errors, warnings } = entitlementsState.context.entitlements /** Gets license data on app mount because LicenseBanner is mounted in App */ useEffect(() => { entitlementsSend("GET_ENTITLEMENTS") }, [entitlementsSend]) - if (warnings.length > 0) { - return + if (errors.length > 0 || warnings.length > 0) { + return } else { return null } diff --git a/site/src/components/LicenseBanner/LicenseBannerView.stories.tsx b/site/src/components/LicenseBanner/LicenseBannerView.stories.tsx index c37653eff7bd5..c7ee69c261e38 100644 --- a/site/src/components/LicenseBanner/LicenseBannerView.stories.tsx +++ b/site/src/components/LicenseBanner/LicenseBannerView.stories.tsx @@ -12,13 +12,23 @@ const Template: Story = (args) => ( export const OneWarning = Template.bind({}) OneWarning.args = { + errors: [], warnings: ["You have exceeded the number of seats in your license."], } export const TwoWarnings = Template.bind({}) TwoWarnings.args = { + errors: [], warnings: [ "You have exceeded the number of seats in your license.", "You are flying too close to the sun.", ], } + +export const OneError = Template.bind({}) +OneError.args = { + errors: [ + "You have multiple replicas but high availability is an Enterprise feature. You will be unable to connect to workspaces.", + ], + warnings: [], +} diff --git a/site/src/components/LicenseBanner/LicenseBannerView.tsx b/site/src/components/LicenseBanner/LicenseBannerView.tsx index 49276b1f0d5ed..792bc191a0a2a 100644 --- a/site/src/components/LicenseBanner/LicenseBannerView.tsx +++ b/site/src/components/LicenseBanner/LicenseBannerView.tsx @@ -2,47 +2,56 @@ import { makeStyles } from "@material-ui/core/styles" import { Expander } from "components/Expander/Expander" import { Pill } from "components/Pill/Pill" import { useState } from "react" +import { colors } from "theme/colors" export const Language = { licenseIssue: "License Issue", licenseIssues: (num: number): string => `${num} License Issues`, - upgrade: "Contact us to upgrade your license.", + upgrade: "Contact sales@coder.com.", exceeded: "It looks like you've exceeded some limits of your license.", lessDetails: "Less", moreDetails: "More", } export interface LicenseBannerViewProps { + errors: string[] warnings: string[] } export const LicenseBannerView: React.FC = ({ + errors, warnings, }) => { const styles = useStyles() const [showDetails, setShowDetails] = useState(false) - if (warnings.length === 1) { + const isError = errors.length > 0 + const messages = [...errors, ...warnings] + const type = isError ? "error" : "warning" + + if (messages.length === 1) { return ( -
- - {warnings[0]} -   - - {Language.upgrade} - +
+ +
+ {messages[0]} +   + + {Language.upgrade} + +
) } else { return ( -
-
-
- - {Language.exceeded} +
+ +
+
    - {warnings.map((warning) => ( -
  • - {warning} + {messages.map((message) => ( +
  • + {message}
    • ))}
@@ -67,14 +76,18 @@ const useStyles = makeStyles((theme) => ({ container: { padding: theme.spacing(1.5), backgroundColor: theme.palette.warning.main, + display: "flex", + alignItems: "center", + + "&.error": { + backgroundColor: colors.red[12], + }, }, flex: { - display: "flex", + display: "column", }, leftContent: { marginRight: theme.spacing(1), - }, - text: { marginLeft: theme.spacing(1), }, link: { @@ -83,9 +96,10 @@ const useStyles = makeStyles((theme) => ({ fontWeight: "bold", }, list: { - margin: theme.spacing(1.5), + padding: theme.spacing(1), + margin: 0, }, listItem: { - margin: theme.spacing(1), + margin: theme.spacing(0.5), }, })) diff --git a/site/src/testHelpers/entities.ts b/site/src/testHelpers/entities.ts index 6e26a4fee5944..7080d2a8d6002 100644 --- a/site/src/testHelpers/entities.ts +++ b/site/src/testHelpers/entities.ts @@ -816,6 +816,7 @@ export const makeMockApiError = ({ }) export const MockEntitlements: TypesGen.Entitlements = { + errors: [], warnings: [], has_license: false, features: {}, @@ -824,6 +825,7 @@ export const MockEntitlements: TypesGen.Entitlements = { } export const MockEntitlementsWithWarnings: TypesGen.Entitlements = { + errors: [], warnings: ["You are over your active user limit.", "And another thing."], has_license: true, experimental: false, @@ -847,6 +849,7 @@ export const MockEntitlementsWithWarnings: TypesGen.Entitlements = { } export const MockEntitlementsWithAuditLog: TypesGen.Entitlements = { + errors: [], warnings: [], has_license: true, experimental: false, diff --git a/site/src/xServices/entitlements/entitlementsXService.ts b/site/src/xServices/entitlements/entitlementsXService.ts index 83ed44d12052d..a1e8bb0d9b895 100644 --- a/site/src/xServices/entitlements/entitlementsXService.ts +++ b/site/src/xServices/entitlements/entitlementsXService.ts @@ -20,6 +20,7 @@ export type EntitlementsEvent = | { type: "HIDE_MOCK_BANNER" } const emptyEntitlements = { + errors: [], warnings: [], features: {}, has_license: false, From c1aa3d230740ab4e2388c14781131b0f76686b33 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 00:36:24 +0000 Subject: [PATCH 30/79] Pipe TLS to DERP mesh --- cli/server.go | 3 +++ coderd/coderd.go | 3 +++ enterprise/coderd/coderd.go | 7 ++++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cli/server.go b/cli/server.go index 1ab1a6228f356..de15b7c63c84b 100644 --- a/cli/server.go +++ b/cli/server.go @@ -322,6 +322,9 @@ func Server(dflags *codersdk.DeploymentFlags, newAPI func(context.Context, *code Experimental: ExperimentalEnabled(cmd), DeploymentFlags: dflags, } + if tlsConfig != nil { + options.TLSCertificates = tlsConfig.Certificates + } if dflags.OAuth2GithubClientSecret.Value != "" { options.GithubOAuth2Config, err = configureGithubOAuth2(accessURLParsed, diff --git a/coderd/coderd.go b/coderd/coderd.go index df5c85f030d09..735190373a4f3 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -1,6 +1,7 @@ package coderd import ( + "crypto/tls" "crypto/x509" "io" "net/http" @@ -76,6 +77,8 @@ type Options struct { TracerProvider trace.TracerProvider AutoImportTemplates []AutoImportTemplate + // TLSCertificates is used to mesh DERP servers securely. + TLSCertificates []tls.Certificate TailnetCoordinator tailnet.Coordinator DERPServer *derp.Server DERPMap *tailcfg.DERPMap diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 612e710395722..8a92c0b1c641a 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -3,6 +3,7 @@ package coderd import ( "context" "crypto/ed25519" + "crypto/tls" "net/http" "sync" "time" @@ -137,7 +138,11 @@ func New(ctx context.Context, options *Options) (*API, error) { if err != nil { return nil, xerrors.Errorf("initialize replica: %w", err) } - api.derpMesh = derpmesh.New(options.Logger.Named("derpmesh"), api.DERPServer, nil) + // nolint:gosec + api.derpMesh = derpmesh.New(options.Logger.Named("derpmesh"), api.DERPServer, &tls.Config{ + Certificates: options.TLSCertificates, + ServerName: options.AccessURL.Host, + }) err = api.updateEntitlements(ctx) if err != nil { From 0cc4263715d02cbf83d7ef8c853d85fa427cb8ad Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 03:41:53 +0000 Subject: [PATCH 31/79] Fix DERP mesh with TLS --- enterprise/coderd/coderd.go | 17 ++++++- enterprise/derpmesh/derpmesh_test.go | 72 +++++++++++----------------- enterprise/tailnet/coordinator.go | 6 ++- go.mod | 2 +- go.sum | 4 +- site/src/api/typesGenerated.ts | 1 + 6 files changed, 53 insertions(+), 49 deletions(-) diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 8a92c0b1c641a..803d13b44b7c4 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -4,6 +4,7 @@ import ( "context" "crypto/ed25519" "crypto/tls" + "crypto/x509" "net/http" "sync" "time" @@ -138,10 +139,22 @@ func New(ctx context.Context, options *Options) (*API, error) { if err != nil { return nil, xerrors.Errorf("initialize replica: %w", err) } + + rootCA := x509.NewCertPool() + for _, certificate := range options.TLSCertificates { + for _, certificatePart := range certificate.Certificate { + certificate, err := x509.ParseCertificate(certificatePart) + if err != nil { + return nil, xerrors.Errorf("parse certificate %s: %w", certificate.Subject.CommonName, err) + } + rootCA.AddCert(certificate) + } + } + // nolint:gosec api.derpMesh = derpmesh.New(options.Logger.Named("derpmesh"), api.DERPServer, &tls.Config{ - Certificates: options.TLSCertificates, - ServerName: options.AccessURL.Host, + ServerName: options.AccessURL.Host, + RootCAs: rootCA, }) err = api.updateEntitlements(ctx) diff --git a/enterprise/derpmesh/derpmesh_test.go b/enterprise/derpmesh/derpmesh_test.go index 139e42566ffb1..353e51dd2983f 100644 --- a/enterprise/derpmesh/derpmesh_test.go +++ b/enterprise/derpmesh/derpmesh_test.go @@ -37,44 +37,27 @@ func TestMain(m *testing.M) { func TestDERPMesh(t *testing.T) { t.Parallel() - t.Run("ExchangeMessages", func(t *testing.T) { - // This tests messages passing through multiple DERP servers. - t.Parallel() - firstServer, firstServerURL, firstTLSName := startDERP(t) - defer firstServer.Close() - secondServer, secondServerURL, secondTLSName := startDERP(t) - firstMesh := derpmesh.New(slogtest.Make(t, nil).Named("first").Leveled(slog.LevelDebug), firstServer, firstTLSName) - firstMesh.SetAddresses([]string{secondServerURL}) - secondMesh := derpmesh.New(slogtest.Make(t, nil).Named("second").Leveled(slog.LevelDebug), secondServer, secondTLSName) - secondMesh.SetAddresses([]string{firstServerURL}) - defer firstMesh.Close() - defer secondMesh.Close() - - first := key.NewNode() - second := key.NewNode() - firstClient, err := derphttp.NewClient(first, secondServerURL, tailnet.Logger(slogtest.Make(t, nil))) - require.NoError(t, err) - secondClient, err := derphttp.NewClient(second, firstServerURL, tailnet.Logger(slogtest.Make(t, nil))) - require.NoError(t, err) - err = secondClient.Connect(context.Background()) - require.NoError(t, err) - - sent := []byte("hello world") - err = firstClient.Send(second.Public(), sent) - require.NoError(t, err) + commonName := "something.org" + rawCert := generateTLSCertificate(t, commonName) + certificate, err := x509.ParseCertificate(rawCert.Certificate[0]) + require.NoError(t, err) + pool := x509.NewCertPool() + pool.AddCert(certificate) + tlsConfig := &tls.Config{ + ServerName: commonName, + RootCAs: pool, + Certificates: []tls.Certificate{rawCert}, + } - got := recvData(t, secondClient) - require.Equal(t, sent, got) - }) t.Run("ExchangeMessages", func(t *testing.T) { // This tests messages passing through multiple DERP servers. t.Parallel() - firstServer, firstServerURL, firstTLSName := startDERP(t) + firstServer, firstServerURL := startDERP(t, tlsConfig) defer firstServer.Close() - secondServer, secondServerURL, secondTLSName := startDERP(t) - firstMesh := derpmesh.New(slogtest.Make(t, nil).Named("first").Leveled(slog.LevelDebug), firstServer, firstTLSName) + secondServer, secondServerURL := startDERP(t, tlsConfig) + firstMesh := derpmesh.New(slogtest.Make(t, nil).Named("first").Leveled(slog.LevelDebug), firstServer, tlsConfig) firstMesh.SetAddresses([]string{secondServerURL}) - secondMesh := derpmesh.New(slogtest.Make(t, nil).Named("second").Leveled(slog.LevelDebug), secondServer, secondTLSName) + secondMesh := derpmesh.New(slogtest.Make(t, nil).Named("second").Leveled(slog.LevelDebug), secondServer, tlsConfig) secondMesh.SetAddresses([]string{firstServerURL}) defer firstMesh.Close() defer secondMesh.Close() @@ -83,8 +66,10 @@ func TestDERPMesh(t *testing.T) { second := key.NewNode() firstClient, err := derphttp.NewClient(first, secondServerURL, tailnet.Logger(slogtest.Make(t, nil))) require.NoError(t, err) + firstClient.TLSConfig = tlsConfig secondClient, err := derphttp.NewClient(second, firstServerURL, tailnet.Logger(slogtest.Make(t, nil))) require.NoError(t, err) + secondClient.TLSConfig = tlsConfig err = secondClient.Connect(context.Background()) require.NoError(t, err) @@ -98,8 +83,8 @@ func TestDERPMesh(t *testing.T) { t.Run("RemoveAddress", func(t *testing.T) { // This tests messages passing through multiple DERP servers. t.Parallel() - server, serverURL, tlsName := startDERP(t) - mesh := derpmesh.New(slogtest.Make(t, nil).Named("first").Leveled(slog.LevelDebug), server, tlsName) + server, serverURL := startDERP(t, tlsConfig) + mesh := derpmesh.New(slogtest.Make(t, nil).Named("first").Leveled(slog.LevelDebug), server, tlsConfig) mesh.SetAddresses([]string{"http://fake.com"}) // This should trigger a removal... mesh.SetAddresses([]string{}) @@ -109,8 +94,10 @@ func TestDERPMesh(t *testing.T) { second := key.NewNode() firstClient, err := derphttp.NewClient(first, serverURL, tailnet.Logger(slogtest.Make(t, nil))) require.NoError(t, err) + firstClient.TLSConfig = tlsConfig secondClient, err := derphttp.NewClient(second, serverURL, tailnet.Logger(slogtest.Make(t, nil))) require.NoError(t, err) + secondClient.TLSConfig = tlsConfig err = secondClient.Connect(context.Background()) require.NoError(t, err) sent := []byte("hello world") @@ -124,8 +111,8 @@ func TestDERPMesh(t *testing.T) { meshes := make([]*derpmesh.Mesh, 0, 20) serverURLs := make([]string, 0, 20) for i := 0; i < 20; i++ { - server, url, tlsName := startDERP(t) - mesh := derpmesh.New(slogtest.Make(t, nil).Named("mesh").Leveled(slog.LevelDebug), server, tlsName) + server, url := startDERP(t, tlsConfig) + mesh := derpmesh.New(slogtest.Make(t, nil).Named("mesh").Leveled(slog.LevelDebug), server, tlsConfig) t.Cleanup(func() { _ = server.Close() _ = mesh.Close() @@ -141,8 +128,10 @@ func TestDERPMesh(t *testing.T) { second := key.NewNode() firstClient, err := derphttp.NewClient(first, serverURLs[9], tailnet.Logger(slogtest.Make(t, nil))) require.NoError(t, err) + firstClient.TLSConfig = tlsConfig secondClient, err := derphttp.NewClient(second, serverURLs[16], tailnet.Logger(slogtest.Make(t, nil))) require.NoError(t, err) + secondClient.TLSConfig = tlsConfig err = secondClient.Connect(context.Background()) require.NoError(t, err) @@ -172,21 +161,18 @@ func recvData(t *testing.T, client *derphttp.Client) []byte { } } -func startDERP(t *testing.T) (*derp.Server, string, *tls.Config) { +func startDERP(t *testing.T, tlsConfig *tls.Config) (*derp.Server, string) { logf := tailnet.Logger(slogtest.Make(t, nil)) d := derp.NewServer(key.NewNode(), logf) d.SetMeshKey("some-key") server := httptest.NewUnstartedServer(derphttp.Handler(d)) - commonName := "something.org" - server.TLS = &tls.Config{ - Certificates: []tls.Certificate{generateTLSCertificate(t, commonName)}, - } - server.Start() + server.TLS = tlsConfig + server.StartTLS() t.Cleanup(func() { _ = d.Close() }) t.Cleanup(server.Close) - return d, server.URL, server.TLS + return d, server.URL } func generateTLSCertificate(t testing.TB, commonName string) tls.Certificate { diff --git a/enterprise/tailnet/coordinator.go b/enterprise/tailnet/coordinator.go index 0643f7a259719..206dc68d6319c 100644 --- a/enterprise/tailnet/coordinator.go +++ b/enterprise/tailnet/coordinator.go @@ -294,7 +294,11 @@ func (c *haCoordinator) hangleAgentUpdate(id uuid.UUID, decoder *json.Decoder) ( func (c *haCoordinator) Close() error { c.mutex.Lock() defer c.mutex.Unlock() - + select { + case <-c.close: + return nil + default: + } close(c.close) wg := sync.WaitGroup{} diff --git a/go.mod b/go.mod index b33a438eb3d08..195a09ae2b8fd 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ replace github.com/tcnksm/go-httpstat => github.com/kylecarbs/go-httpstat v0.0.0 // There are a few minor changes we make to Tailscale that we're slowly upstreaming. Compare here: // https://github.com/tailscale/tailscale/compare/main...coder:tailscale:main -replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20221014173742-9f1da7795630 +replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20221015033036-5861cbbf7bf5 // Switch to our fork that imports fixes from http://github.com/tailscale/ssh. // See: https://github.com/coder/coder/issues/3371 diff --git a/go.sum b/go.sum index 5852582c26c4a..b80c0d4173a5f 100644 --- a/go.sum +++ b/go.sum @@ -351,8 +351,8 @@ github.com/coder/retry v1.3.0 h1:5lAAwt/2Cm6lVmnfBY7sOMXcBOwcwJhmV5QGSELIVWY= github.com/coder/retry v1.3.0/go.mod h1:tXuRgZgWjUnU5LZPT4lJh4ew2elUhexhlnXzrJWdyFY= github.com/coder/ssh v0.0.0-20220811105153-fcea99919338 h1:tN5GKFT68YLVzJoA8AHuiMNJ0qlhoD3pGN3JY9gxSko= github.com/coder/ssh v0.0.0-20220811105153-fcea99919338/go.mod h1:ZSS+CUoKHDrqVakTfTWUlKSr9MtMFkC4UvtQKD7O914= -github.com/coder/tailscale v1.1.1-0.20221014173742-9f1da7795630 h1:FgWWdu0fnFEpUNjW0vOaCuOxOZ/GQzn6oo7p5IMlSA0= -github.com/coder/tailscale v1.1.1-0.20221014173742-9f1da7795630/go.mod h1:5amxy08qijEa8bcTW2SeIy4MIqcmd7LMsuOxqOlj2Ak= +github.com/coder/tailscale v1.1.1-0.20221015033036-5861cbbf7bf5 h1:WVH6e/qK3Wpl0wbmpORD2oQ1qLJborF3fsFHyO1ps0Y= +github.com/coder/tailscale v1.1.1-0.20221015033036-5861cbbf7bf5/go.mod h1:5amxy08qijEa8bcTW2SeIy4MIqcmd7LMsuOxqOlj2Ak= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 92db958074a68..11b4c64e34786 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -538,6 +538,7 @@ export interface Replica { readonly relay_address: string readonly region_id: number readonly error: string + readonly database_latency: number } // From codersdk/error.go From f9177e40ecea7499f4c7d972b9a66f49fbe2689a Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 03:59:20 +0000 Subject: [PATCH 32/79] Add tests for TLS --- coderd/coderdtest/coderdtest.go | 14 ++++- codersdk/workspaceagents.go | 6 ++- enterprise/coderd/coderd.go | 3 +- .../coderd/coderdenttest/coderdenttest.go | 13 ++++- enterprise/coderd/replicas_test.go | 47 +++++++++++++++- enterprise/coderd/workspaceagents_test.go | 9 ++++ enterprise/derpmesh/derpmesh_test.go | 47 +--------------- testutil/certificate.go | 53 +++++++++++++++++++ 8 files changed, 138 insertions(+), 54 deletions(-) create mode 100644 testutil/certificate.go diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index 59b414cad8903..cbbcd7aaa493a 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -7,6 +7,7 @@ import ( "crypto/rand" "crypto/rsa" "crypto/sha256" + "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/base64" @@ -75,6 +76,7 @@ type Options struct { AutobuildTicker <-chan time.Time AutobuildStats chan<- executor.Stats Auditor audit.Auditor + TLSCertificates []tls.Certificate // IncludeProvisionerDaemon when true means to start an in-memory provisionerD IncludeProvisionerDaemon bool @@ -158,7 +160,14 @@ func NewOptions(t *testing.T, options *Options) (*httptest.Server, context.Cance srv.Config.BaseContext = func(_ net.Listener) context.Context { return ctx } - srv.Start() + if options.TLSCertificates != nil { + srv.TLS = &tls.Config{ + Certificates: options.TLSCertificates, + } + srv.StartTLS() + } else { + srv.Start() + } t.Cleanup(srv.Close) tcpAddr, ok := srv.Listener.Addr().(*net.TCPAddr) @@ -201,6 +210,7 @@ func NewOptions(t *testing.T, options *Options) (*httptest.Server, context.Cance APIRateLimit: options.APIRateLimit, Authorizer: options.Authorizer, Telemetry: telemetry.NewNoop(), + TLSCertificates: options.TLSCertificates, DERPMap: &tailcfg.DERPMap{ Regions: map[int]*tailcfg.DERPRegion{ 1: { @@ -215,7 +225,7 @@ func NewOptions(t *testing.T, options *Options) (*httptest.Server, context.Cance DERPPort: derpPort, STUNPort: stunAddr.Port, InsecureForTests: true, - ForceHTTP: true, + ForceHTTP: options.TLSCertificates == nil, }}, }, }, diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index c86b399e189ab..c86944ae2b629 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -315,7 +315,8 @@ func (c *Client) ListenWorkspaceAgentTailnet(ctx context.Context) (net.Conn, err Value: c.SessionToken, }}) httpClient := &http.Client{ - Jar: jar, + Jar: jar, + Transport: c.HTTPClient.Transport, } // nolint:bodyclose conn, res, err := websocket.Dial(ctx, coordinateURL.String(), &websocket.DialOptions{ @@ -380,7 +381,8 @@ func (c *Client) DialWorkspaceAgent(ctx context.Context, agentID uuid.UUID, opti Value: c.SessionToken, }}) httpClient := &http.Client{ - Jar: jar, + Jar: jar, + Transport: c.HTTPClient.Transport, } ctx, cancelFunc := context.WithCancel(ctx) closed := make(chan struct{}) diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 803d13b44b7c4..469205cfa2e96 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -150,10 +150,9 @@ func New(ctx context.Context, options *Options) (*API, error) { rootCA.AddCert(certificate) } } - // nolint:gosec api.derpMesh = derpmesh.New(options.Logger.Named("derpmesh"), api.DERPServer, &tls.Config{ - ServerName: options.AccessURL.Host, + ServerName: options.AccessURL.Hostname(), RootCAs: rootCA, }) diff --git a/enterprise/coderd/coderdenttest/coderdenttest.go b/enterprise/coderd/coderdenttest/coderdenttest.go index a503a22ce459c..02eff4e2acf2e 100644 --- a/enterprise/coderd/coderdenttest/coderdenttest.go +++ b/enterprise/coderd/coderdenttest/coderdenttest.go @@ -4,7 +4,9 @@ import ( "context" "crypto/ed25519" "crypto/rand" + "crypto/tls" "io" + "net/http" "testing" "time" @@ -85,7 +87,16 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c _ = provisionerCloser.Close() _ = coderAPI.Close() }) - return codersdk.New(coderAPI.AccessURL), provisionerCloser, coderAPI + client := codersdk.New(coderAPI.AccessURL) + client.HTTPClient = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + //nolint:gosec + InsecureSkipVerify: true, + }, + }, + } + return client, provisionerCloser, coderAPI } type LicenseOptions struct { diff --git a/enterprise/coderd/replicas_test.go b/enterprise/coderd/replicas_test.go index 0272fb018f3d6..b66bcaef9f976 100644 --- a/enterprise/coderd/replicas_test.go +++ b/enterprise/coderd/replicas_test.go @@ -2,6 +2,7 @@ package coderd_test import ( "context" + "crypto/tls" "testing" "time" @@ -19,7 +20,7 @@ import ( func TestReplicas(t *testing.T) { t.Parallel() - t.Run("WarningsWithoutLicense", func(t *testing.T) { + t.Run("ErrorWithoutLicense", func(t *testing.T) { t.Parallel() db, pubsub := dbtestutil.NewDB(t) firstClient := coderdenttest.New(t, &coderdenttest.Options{ @@ -39,7 +40,7 @@ func TestReplicas(t *testing.T) { secondClient.SessionToken = firstClient.SessionToken ents, err := secondClient.Entitlements(context.Background()) require.NoError(t, err) - require.Len(t, ents.Warnings, 1) + require.Len(t, ents.Errors, 1) _ = secondAPI.Close() ents, err = firstClient.Entitlements(context.Background()) @@ -85,6 +86,48 @@ func TestReplicas(t *testing.T) { return err == nil }, testutil.WaitLong, testutil.IntervalFast) _ = conn.Close() + }) + t.Run("ConnectAcrossMultipleTLS", func(t *testing.T) { + t.Parallel() + db, pubsub := dbtestutil.NewDB(t) + certificates := []tls.Certificate{testutil.GenerateTLSCertificate(t, "localhost")} + firstClient := coderdenttest.New(t, &coderdenttest.Options{ + Options: &coderdtest.Options{ + IncludeProvisionerDaemon: true, + Database: db, + Pubsub: pubsub, + TLSCertificates: certificates, + }, + }) + firstUser := coderdtest.CreateFirstUser(t, firstClient) + coderdenttest.AddLicense(t, firstClient, coderdenttest.LicenseOptions{ + HighAvailability: true, + }) + + secondClient := coderdenttest.New(t, &coderdenttest.Options{ + Options: &coderdtest.Options{ + Database: db, + Pubsub: pubsub, + TLSCertificates: certificates, + }, + }) + secondClient.SessionToken = firstClient.SessionToken + replicas, err := secondClient.Replicas(context.Background()) + require.NoError(t, err) + require.Len(t, replicas, 2) + _, agent := setupWorkspaceAgent(t, firstClient, firstUser, 0) + conn, err := secondClient.DialWorkspaceAgent(context.Background(), agent.ID, &codersdk.DialWorkspaceAgentOptions{ + BlockEndpoints: true, + Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug), + }) + require.NoError(t, err) + require.Eventually(t, func() bool { + ctx, cancelFunc := context.WithTimeout(context.Background(), 3*time.Second) + defer cancelFunc() + _, err = conn.Ping(ctx) + return err == nil + }, testutil.WaitLong, testutil.IntervalFast) + _ = conn.Close() }) } diff --git a/enterprise/coderd/workspaceagents_test.go b/enterprise/coderd/workspaceagents_test.go index 097bab354ba74..18285bcb94317 100644 --- a/enterprise/coderd/workspaceagents_test.go +++ b/enterprise/coderd/workspaceagents_test.go @@ -2,6 +2,7 @@ package coderd_test import ( "context" + "crypto/tls" "fmt" "net/http" "testing" @@ -108,6 +109,14 @@ func setupWorkspaceAgent(t *testing.T, client *codersdk.Client, user codersdk.Cr workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID) coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID) agentClient := codersdk.New(client.URL) + agentClient.HTTPClient = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + //nolint:gosec + InsecureSkipVerify: true, + }, + }, + } agentClient.SessionToken = authToken agentCloser := agent.New(agent.Options{ FetchMetadata: agentClient.WorkspaceAgentMetadata, diff --git a/enterprise/derpmesh/derpmesh_test.go b/enterprise/derpmesh/derpmesh_test.go index 353e51dd2983f..fcf410ac0e574 100644 --- a/enterprise/derpmesh/derpmesh_test.go +++ b/enterprise/derpmesh/derpmesh_test.go @@ -1,22 +1,13 @@ package derpmesh_test import ( - "bytes" "context" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" "errors" "io" - "math/big" - "net" "net/http/httptest" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -29,6 +20,7 @@ import ( "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/enterprise/derpmesh" "github.com/coder/coder/tailnet" + "github.com/coder/coder/testutil" ) func TestMain(m *testing.M) { @@ -38,7 +30,7 @@ func TestMain(m *testing.M) { func TestDERPMesh(t *testing.T) { t.Parallel() commonName := "something.org" - rawCert := generateTLSCertificate(t, commonName) + rawCert := testutil.GenerateTLSCertificate(t, commonName) certificate, err := x509.ParseCertificate(rawCert.Certificate[0]) require.NoError(t, err) pool := x509.NewCertPool() @@ -174,38 +166,3 @@ func startDERP(t *testing.T, tlsConfig *tls.Config) (*derp.Server, string) { t.Cleanup(server.Close) return d, server.URL } - -func generateTLSCertificate(t testing.TB, commonName string) tls.Certificate { - privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - require.NoError(t, err) - template := x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{ - Organization: []string{"Acme Co"}, - CommonName: commonName, - }, - DNSNames: []string{commonName}, - NotBefore: time.Now(), - NotAfter: time.Now().Add(time.Hour * 24 * 180), - - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, - } - - derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) - require.NoError(t, err) - var certFile bytes.Buffer - require.NoError(t, err) - _, err = certFile.Write(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})) - require.NoError(t, err) - privateKeyBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) - require.NoError(t, err) - var keyFile bytes.Buffer - err = pem.Encode(&keyFile, &pem.Block{Type: "PRIVATE KEY", Bytes: privateKeyBytes}) - require.NoError(t, err) - cert, err := tls.X509KeyPair(certFile.Bytes(), keyFile.Bytes()) - require.NoError(t, err) - return cert -} diff --git a/testutil/certificate.go b/testutil/certificate.go new file mode 100644 index 0000000000000..1edc975746958 --- /dev/null +++ b/testutil/certificate.go @@ -0,0 +1,53 @@ +package testutil + +import ( + "bytes" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func GenerateTLSCertificate(t testing.TB, commonName string) tls.Certificate { + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + Organization: []string{"Acme Co"}, + CommonName: commonName, + }, + DNSNames: []string{commonName}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(time.Hour * 24 * 180), + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + require.NoError(t, err) + var certFile bytes.Buffer + require.NoError(t, err) + _, err = certFile.Write(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})) + require.NoError(t, err) + privateKeyBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) + require.NoError(t, err) + var keyFile bytes.Buffer + err = pem.Encode(&keyFile, &pem.Block{Type: "PRIVATE KEY", Bytes: privateKeyBytes}) + require.NoError(t, err) + cert, err := tls.X509KeyPair(certFile.Bytes(), keyFile.Bytes()) + require.NoError(t, err) + return cert +} From ee59d88a087408885a8297afb8eda41b0bd73a54 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 04:09:16 +0000 Subject: [PATCH 33/79] Fix replica sync TLS --- enterprise/coderd/coderd.go | 38 ++++++++++++++++----------- enterprise/coderd/replicas_test.go | 6 +++++ enterprise/replicasync/replicasync.go | 5 ++++ 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 469205cfa2e96..6bbbcb16f33cf 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -129,32 +129,38 @@ func New(ctx context.Context, options *Options) (*API, error) { }) } - var err error - api.replicaManager, err = replicasync.New(ctx, options.Logger, options.Database, options.Pubsub, replicasync.Options{ - // Create a new replica ID for each Coder instance! - ID: uuid.New(), - RelayAddress: options.DERPServerRelayAddress, - RegionID: int32(options.DERPServerRegionID), - }) - if err != nil { - return nil, xerrors.Errorf("initialize replica: %w", err) - } - - rootCA := x509.NewCertPool() + meshRootCA := x509.NewCertPool() for _, certificate := range options.TLSCertificates { for _, certificatePart := range certificate.Certificate { certificate, err := x509.ParseCertificate(certificatePart) if err != nil { return nil, xerrors.Errorf("parse certificate %s: %w", certificate.Subject.CommonName, err) } - rootCA.AddCert(certificate) + meshRootCA.AddCert(certificate) } } - // nolint:gosec - api.derpMesh = derpmesh.New(options.Logger.Named("derpmesh"), api.DERPServer, &tls.Config{ + // This TLS configuration spoofs access from the access URL hostname + // assuming that the certificates provided will cover that hostname. + // + // Replica sync and DERP meshing require accessing replicas via their + // internal IP addresses, and if TLS is configured we use the same + // certificates. + meshTLSConfig := &tls.Config{ ServerName: options.AccessURL.Hostname(), - RootCAs: rootCA, + RootCAs: meshRootCA, + } + var err error + api.replicaManager, err = replicasync.New(ctx, options.Logger, options.Database, options.Pubsub, replicasync.Options{ + // Create a new replica ID for each Coder instance! + ID: uuid.New(), + RelayAddress: options.DERPServerRelayAddress, + RegionID: int32(options.DERPServerRegionID), + TLSConfig: meshTLSConfig, }) + if err != nil { + return nil, xerrors.Errorf("initialize replica: %w", err) + } + api.derpMesh = derpmesh.New(options.Logger.Named("derpmesh"), api.DERPServer, meshTLSConfig) err = api.updateEntitlements(ctx) if err != nil { diff --git a/enterprise/coderd/replicas_test.go b/enterprise/coderd/replicas_test.go index b66bcaef9f976..63bae9ebce9e6 100644 --- a/enterprise/coderd/replicas_test.go +++ b/enterprise/coderd/replicas_test.go @@ -129,5 +129,11 @@ func TestReplicas(t *testing.T) { return err == nil }, testutil.WaitLong, testutil.IntervalFast) _ = conn.Close() + replicas, err = secondClient.Replicas(context.Background()) + require.NoError(t, err) + require.Len(t, replicas, 2) + for _, replica := range replicas { + require.Empty(t, replica.Error) + } }) } diff --git a/enterprise/replicasync/replicasync.go b/enterprise/replicasync/replicasync.go index 75ba041aaa6e1..758a11a84e842 100644 --- a/enterprise/replicasync/replicasync.go +++ b/enterprise/replicasync/replicasync.go @@ -2,6 +2,7 @@ package replicasync import ( "context" + "crypto/tls" "database/sql" "errors" "fmt" @@ -30,6 +31,7 @@ type Options struct { PeerTimeout time.Duration RelayAddress string RegionID int32 + TLSConfig *tls.Config } // New registers the replica with the database and periodically updates to ensure @@ -254,6 +256,9 @@ func (m *Manager) run(ctx context.Context) error { } client := http.Client{ Timeout: m.options.PeerTimeout, + Transport: &http.Transport{ + TLSClientConfig: m.options.TLSConfig, + }, } res, err := client.Do(req) if err != nil { From 8641e58790b067a70ae29fb4e67e89d952b1fff3 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 04:28:50 +0000 Subject: [PATCH 34/79] Fix RootCA for replica meshing --- enterprise/coderd/coderd.go | 5 ++- enterprise/replicasync/replicasync_test.go | 44 ++++++++++++++++++++++ helm/templates/service.yaml | 1 + 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 6bbbcb16f33cf..f836f786463d2 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -146,8 +146,9 @@ func New(ctx context.Context, options *Options) (*API, error) { // internal IP addresses, and if TLS is configured we use the same // certificates. meshTLSConfig := &tls.Config{ - ServerName: options.AccessURL.Hostname(), - RootCAs: meshRootCA, + Certificates: options.TLSCertificates, + RootCAs: meshRootCA, + ServerName: options.AccessURL.Hostname(), } var err error api.replicaManager, err = replicasync.New(ctx, options.Logger, options.Database, options.Pubsub, replicasync.Options{ diff --git a/enterprise/replicasync/replicasync_test.go b/enterprise/replicasync/replicasync_test.go index ccacbeb310c23..faba7345183ff 100644 --- a/enterprise/replicasync/replicasync_test.go +++ b/enterprise/replicasync/replicasync_test.go @@ -2,6 +2,8 @@ package replicasync_test import ( "context" + "crypto/tls" + "crypto/x509" "net/http" "net/http/httptest" "sync" @@ -112,6 +114,48 @@ func TestReplica(t *testing.T) { require.False(t, server.Self().Error.Valid) _ = server.Close() }) + t.Run("ConnectsToPeerReplicaTLS", func(t *testing.T) { + // Ensures that the replica reports a successful status for + // accessing all of its peers. + t.Parallel() + rawCert := testutil.GenerateTLSCertificate(t, "hello.org") + certificate, err := x509.ParseCertificate(rawCert.Certificate[0]) + require.NoError(t, err) + pool := x509.NewCertPool() + pool.AddCert(certificate) + // nolint:gosec + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{rawCert}, + ServerName: "hello.org", + RootCAs: pool, + } + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + srv.TLS = tlsConfig + srv.StartTLS() + defer srv.Close() + db, pubsub := dbtestutil.NewDB(t) + peer, err := db.InsertReplica(context.Background(), database.InsertReplicaParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + StartedAt: database.Now(), + UpdatedAt: database.Now(), + Hostname: "something", + RelayAddress: srv.URL, + }) + require.NoError(t, err) + server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replicasync.Options{ + ID: uuid.New(), + RelayAddress: "http://169.254.169.254", + TLSConfig: tlsConfig, + }) + require.NoError(t, err) + require.Len(t, server.Regional(), 1) + require.Equal(t, peer.ID, server.Regional()[0].ID) + require.False(t, server.Self().Error.Valid) + _ = server.Close() + }) t.Run("ConnectsToFakePeerWithError", func(t *testing.T) { t.Parallel() db, pubsub := dbtestutil.NewDB(t) diff --git a/helm/templates/service.yaml b/helm/templates/service.yaml index 28fe0e9f9aa8c..b9a7e9a2f0886 100644 --- a/helm/templates/service.yaml +++ b/helm/templates/service.yaml @@ -10,6 +10,7 @@ metadata: {{- toYaml .Values.coder.service.annotations | nindent 4 }} spec: type: {{ .Values.coder.service.type }} + sessionAffinity: ClientIP ports: - name: {{ include "coder.portName" . | quote }} port: {{ include "coder.servicePort" . }} From 3dfb796c29ae142be30399ffb4b1dd279f567ca0 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 04:51:04 +0000 Subject: [PATCH 35/79] Remove ID from replicasync --- enterprise/coderd/coderd.go | 6 +- enterprise/coderd/replicas_test.go | 2 +- enterprise/replicasync/replicasync.go | 65 ++++++++-------------- enterprise/replicasync/replicasync_test.go | 53 ++++-------------- 4 files changed, 35 insertions(+), 91 deletions(-) diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index f836f786463d2..371ac12fe21b8 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -13,7 +13,6 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/go-chi/chi/v5" - "github.com/google/uuid" "cdr.dev/slog" "github.com/coder/coder/coderd" @@ -146,14 +145,13 @@ func New(ctx context.Context, options *Options) (*API, error) { // internal IP addresses, and if TLS is configured we use the same // certificates. meshTLSConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, Certificates: options.TLSCertificates, RootCAs: meshRootCA, ServerName: options.AccessURL.Hostname(), } var err error - api.replicaManager, err = replicasync.New(ctx, options.Logger, options.Database, options.Pubsub, replicasync.Options{ - // Create a new replica ID for each Coder instance! - ID: uuid.New(), + api.replicaManager, err = replicasync.New(ctx, options.Logger, options.Database, options.Pubsub, &replicasync.Options{ RelayAddress: options.DERPServerRelayAddress, RegionID: int32(options.DERPServerRegionID), TLSConfig: meshTLSConfig, diff --git a/enterprise/coderd/replicas_test.go b/enterprise/coderd/replicas_test.go index 63bae9ebce9e6..9d6970823befb 100644 --- a/enterprise/coderd/replicas_test.go +++ b/enterprise/coderd/replicas_test.go @@ -123,7 +123,7 @@ func TestReplicas(t *testing.T) { }) require.NoError(t, err) require.Eventually(t, func() bool { - ctx, cancelFunc := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.IntervalMedium) defer cancelFunc() _, err = conn.Ping(ctx) return err == nil diff --git a/enterprise/replicasync/replicasync.go b/enterprise/replicasync/replicasync.go index 758a11a84e842..82e7d74273eeb 100644 --- a/enterprise/replicasync/replicasync.go +++ b/enterprise/replicasync/replicasync.go @@ -26,7 +26,6 @@ var ( ) type Options struct { - ID uuid.UUID UpdateInterval time.Duration PeerTimeout time.Duration RelayAddress string @@ -36,9 +35,9 @@ type Options struct { // New registers the replica with the database and periodically updates to ensure // it's healthy. It contacts all other alive replicas to ensure they are reachable. -func New(ctx context.Context, logger slog.Logger, db database.Store, pubsub database.Pubsub, options Options) (*Manager, error) { - if options.ID == uuid.Nil { - panic("An ID must be provided!") +func New(ctx context.Context, logger slog.Logger, db database.Store, pubsub database.Pubsub, options *Options) (*Manager, error) { + if options == nil { + options = &Options{} } if options.PeerTimeout == 0 { options.PeerTimeout = 3 * time.Second @@ -54,50 +53,29 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, pubsub data if err != nil { return nil, xerrors.Errorf("ping database: %w", err) } - var replica database.Replica - _, err = db.GetReplicaByID(ctx, options.ID) + id := uuid.New() + replica, err := db.InsertReplica(ctx, database.InsertReplicaParams{ + ID: id, + CreatedAt: database.Now(), + StartedAt: database.Now(), + UpdatedAt: database.Now(), + Hostname: hostname, + RegionID: options.RegionID, + RelayAddress: options.RelayAddress, + Version: buildinfo.Version(), + DatabaseLatency: int32(databaseLatency.Microseconds()), + }) if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - return nil, xerrors.Errorf("get replica: %w", err) - } - replica, err = db.InsertReplica(ctx, database.InsertReplicaParams{ - ID: options.ID, - CreatedAt: database.Now(), - StartedAt: database.Now(), - UpdatedAt: database.Now(), - Hostname: hostname, - RegionID: options.RegionID, - RelayAddress: options.RelayAddress, - Version: buildinfo.Version(), - DatabaseLatency: int32(databaseLatency.Microseconds()), - }) - if err != nil { - return nil, xerrors.Errorf("insert replica: %w", err) - } - } else { - replica, err = db.UpdateReplica(ctx, database.UpdateReplicaParams{ - ID: options.ID, - UpdatedAt: database.Now(), - StartedAt: database.Now(), - StoppedAt: sql.NullTime{}, - RelayAddress: options.RelayAddress, - RegionID: options.RegionID, - Hostname: hostname, - Version: buildinfo.Version(), - Error: sql.NullString{}, - DatabaseLatency: int32(databaseLatency.Microseconds()), - }) - if err != nil { - return nil, xerrors.Errorf("update replica: %w", err) - } + return nil, xerrors.Errorf("insert replica: %w", err) } - err = pubsub.Publish(PubsubEvent, []byte(options.ID.String())) + err = pubsub.Publish(PubsubEvent, []byte(id.String())) if err != nil { return nil, xerrors.Errorf("publish new replica: %w", err) } ctx, cancelFunc := context.WithCancel(ctx) server := &Manager{ - options: &options, + id: id, + options: options, db: db, pubsub: pubsub, self: replica, @@ -128,6 +106,7 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, pubsub data // Manager keeps the replica up to date and in sync with other replicas. type Manager struct { + id uuid.UUID options *Options db database.Store pubsub database.Pubsub @@ -196,7 +175,7 @@ func (m *Manager) subscribe(ctx context.Context) error { return } // Don't process updates for ourself! - if id == m.options.ID { + if id == m.id { return } if updating { @@ -233,7 +212,7 @@ func (m *Manager) run(ctx context.Context) error { m.mutex.Lock() m.peers = make([]database.Replica, 0, len(replicas)) for _, replica := range replicas { - if replica.ID == m.options.ID { + if replica.ID == m.id { continue } m.peers = append(m.peers, replica) diff --git a/enterprise/replicasync/replicasync_test.go b/enterprise/replicasync/replicasync_test.go index faba7345183ff..0b42f44791df4 100644 --- a/enterprise/replicasync/replicasync_test.go +++ b/enterprise/replicasync/replicasync_test.go @@ -11,7 +11,6 @@ import ( "time" "github.com/google/uuid" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/goleak" @@ -32,38 +31,15 @@ func TestReplica(t *testing.T) { // This ensures that a new replica is created on New. t.Parallel() db, pubsub := dbtestutil.NewDB(t) - id := uuid.New() + closeChan := make(chan struct{}, 1) cancel, err := pubsub.Subscribe(replicasync.PubsubEvent, func(ctx context.Context, message []byte) { - assert.Equal(t, []byte(id.String()), message) + closeChan <- struct{}{} }) require.NoError(t, err) defer cancel() - server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replicasync.Options{ - ID: id, - }) - require.NoError(t, err) - _ = server.Close() - require.NoError(t, err) - }) - t.Run("UpdatesOnNew", func(t *testing.T) { - // This ensures that a replica is updated when it initially connects - // and immediately publishes it's existence! - t.Parallel() - db, pubsub := dbtestutil.NewDB(t) - id := uuid.New() - _, err := db.InsertReplica(context.Background(), database.InsertReplicaParams{ - ID: id, - }) - require.NoError(t, err) - cancel, err := pubsub.Subscribe(replicasync.PubsubEvent, func(ctx context.Context, message []byte) { - assert.Equal(t, []byte(id.String()), message) - }) - require.NoError(t, err) - defer cancel() - server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replicasync.Options{ - ID: id, - }) + server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, nil) require.NoError(t, err) + <-closeChan _ = server.Close() require.NoError(t, err) }) @@ -80,9 +56,7 @@ func TestReplica(t *testing.T) { Hostname: "something", }) require.NoError(t, err) - _, err = replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replicasync.Options{ - ID: uuid.New(), - }) + _, err = replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, nil) require.Error(t, err) require.Equal(t, "a relay address must be specified when running multiple replicas in the same region", err.Error()) }) @@ -104,8 +78,7 @@ func TestReplica(t *testing.T) { RelayAddress: srv.URL, }) require.NoError(t, err) - server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replicasync.Options{ - ID: uuid.New(), + server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, &replicasync.Options{ RelayAddress: "http://169.254.169.254", }) require.NoError(t, err) @@ -145,8 +118,7 @@ func TestReplica(t *testing.T) { RelayAddress: srv.URL, }) require.NoError(t, err) - server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replicasync.Options{ - ID: uuid.New(), + server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, &replicasync.Options{ RelayAddress: "http://169.254.169.254", TLSConfig: tlsConfig, }) @@ -169,8 +141,7 @@ func TestReplica(t *testing.T) { RelayAddress: "http://169.254.169.254", }) require.NoError(t, err) - server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replicasync.Options{ - ID: uuid.New(), + server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, &replicasync.Options{ PeerTimeout: 1 * time.Millisecond, RelayAddress: "http://169.254.169.254", }) @@ -185,10 +156,7 @@ func TestReplica(t *testing.T) { // Refresh when a new replica appears! t.Parallel() db, pubsub := dbtestutil.NewDB(t) - id := uuid.New() - server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, replicasync.Options{ - ID: id, - }) + server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, nil) require.NoError(t, err) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) @@ -224,8 +192,7 @@ func TestReplica(t *testing.T) { count := 20 wg.Add(count) for i := 0; i < count; i++ { - server, err := replicasync.New(context.Background(), logger, db, pubsub, replicasync.Options{ - ID: uuid.New(), + server, err := replicasync.New(context.Background(), logger, db, pubsub, &replicasync.Options{ RelayAddress: srv.URL, }) require.NoError(t, err) From ec2c1f13403216460e6ab3104c3c5fe5b5355667 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 04:56:58 +0000 Subject: [PATCH 36/79] Fix getting certificates for meshing --- cli/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/server.go b/cli/server.go index de15b7c63c84b..0704aca10b07e 100644 --- a/cli/server.go +++ b/cli/server.go @@ -929,6 +929,7 @@ func configureTLS(tlsMinVersion, tlsClientAuth string, tlsCertFiles, tlsKeyFiles if err != nil { return nil, xerrors.Errorf("load certificates: %w", err) } + tlsConfig.Certificates = certs tlsConfig.GetCertificate = func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { // If there's only one certificate, return it. if len(certs) == 1 { From 590f0f896ae74d9765d7014150936bbc8ed6278d Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 05:22:04 +0000 Subject: [PATCH 37/79] Remove excessive locking --- enterprise/replicasync/replicasync.go | 22 +++++++++--------- enterprise/tailnet/coordinator.go | 32 +++++++++------------------ tailnet/coordinator.go | 24 +++++++++++--------- 3 files changed, 33 insertions(+), 45 deletions(-) diff --git a/enterprise/replicasync/replicasync.go b/enterprise/replicasync/replicasync.go index 82e7d74273eeb..b635a84991e24 100644 --- a/enterprise/replicasync/replicasync.go +++ b/enterprise/replicasync/replicasync.go @@ -143,9 +143,11 @@ func (m *Manager) loop(ctx context.Context) { // subscribe listens for new replica information! func (m *Manager) subscribe(ctx context.Context) error { - needsUpdate := false - updating := false - updateMutex := sync.Mutex{} + var ( + needsUpdate = false + updating = false + updateMutex = sync.Mutex{} + ) // This loop will continually update nodes as updates are processed. // The intent is to always be up to date without spamming the run @@ -199,9 +201,7 @@ func (m *Manager) run(ctx context.Context) error { m.closeMutex.Lock() m.closeWait.Add(1) m.closeMutex.Unlock() - go func() { - m.closeWait.Done() - }() + defer m.closeWait.Done() // Expect replicas to update once every three times the interval... // If they don't, assume death! replicas, err := m.db.GetReplicasUpdatedAfter(ctx, database.Now().Add(-3*m.options.UpdateInterval)) @@ -224,8 +224,7 @@ func (m *Manager) run(ctx context.Context) error { failed := make([]string, 0) for _, peer := range m.Regional() { wg.Add(1) - peer := peer - go func() { + go func(peer database.Replica) { defer wg.Done() req, err := http.NewRequestWithContext(ctx, http.MethodGet, peer.RelayAddress, nil) if err != nil { @@ -247,7 +246,7 @@ func (m *Manager) run(ctx context.Context) error { return } _ = res.Body.Close() - }() + }(peer) } wg.Wait() replicaError := sql.NullString{} @@ -279,11 +278,11 @@ func (m *Manager) run(ctx context.Context) error { return xerrors.Errorf("update replica: %w", err) } m.mutex.Lock() + defer m.mutex.Unlock() if m.self.Error.String != replica.Error.String { // Publish an update occurred! err = m.pubsub.Publish(PubsubEvent, []byte(m.self.ID.String())) if err != nil { - m.mutex.Unlock() return xerrors.Errorf("publish replica update: %w", err) } } @@ -291,7 +290,6 @@ func (m *Manager) run(ctx context.Context) error { if m.callback != nil { go m.callback() } - m.mutex.Unlock() return nil } @@ -306,7 +304,7 @@ func (m *Manager) Self() database.Replica { func (m *Manager) All() []database.Replica { m.mutex.Lock() defer m.mutex.Unlock() - return append(m.peers, m.self) + return append(m.peers[:], m.self) } // Regional returns all replicas in the same region excluding itself. diff --git a/enterprise/tailnet/coordinator.go b/enterprise/tailnet/coordinator.go index 206dc68d6319c..1ccf56f50da11 100644 --- a/enterprise/tailnet/coordinator.go +++ b/enterprise/tailnet/coordinator.go @@ -69,19 +69,19 @@ func (c *haCoordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID // When a new connection is requested, we update it with the latest // node of the agent. This allows the connection to establish. node, ok := c.nodes[agent] + c.mutex.Unlock() if ok { data, err := json.Marshal([]*agpl.Node{node}) if err != nil { - c.mutex.Unlock() return xerrors.Errorf("marshal node: %w", err) } _, err = conn.Write(data) if err != nil { - c.mutex.Unlock() return xerrors.Errorf("write nodes: %w", err) } } + c.mutex.Lock() connectionSockets, ok := c.agentToConnectionSockets[agent] if !ok { connectionSockets = map[uuid.UUID]net.Conn{} @@ -129,28 +129,17 @@ func (c *haCoordinator) handleNextClientMessage(id, agent uuid.UUID, decoder *js } c.mutex.Lock() - defer c.mutex.Unlock() - // Update the node of this client in our in-memory map. If an agent entirely // shuts down and reconnects, it needs to be aware of all clients attempting // to establish connections. c.nodes[id] = &node - // Write the new node from this client to the actively connected agent. - err = c.writeNodeToAgent(agent, &node) - if err != nil { - return xerrors.Errorf("write node to agent: %w", err) - } - - return nil -} - -func (c *haCoordinator) writeNodeToAgent(agent uuid.UUID, node *agpl.Node) error { agentSocket, ok := c.agentSockets[agent] + c.mutex.Unlock() if !ok { // If we don't own the agent locally, send it over pubsub to a node that // owns the agent. - err := c.publishNodesToAgent(agent, []*agpl.Node{node}) + err := c.publishNodesToAgent(agent, []*agpl.Node{&node}) if err != nil { return xerrors.Errorf("publish node to agent") } @@ -159,7 +148,7 @@ func (c *haCoordinator) writeNodeToAgent(agent uuid.UUID, node *agpl.Node) error // Write the new node from this client to the actively // connected agent. - data, err := json.Marshal([]*agpl.Node{node}) + data, err := json.Marshal([]*agpl.Node{&node}) if err != nil { return xerrors.Errorf("marshal nodes: %w", err) } @@ -171,14 +160,13 @@ func (c *haCoordinator) writeNodeToAgent(agent uuid.UUID, node *agpl.Node) error } return xerrors.Errorf("write json: %w", err) } + return nil } // ServeAgent accepts a WebSocket connection to an agent that listens to // incoming connections and publishes node updates. func (c *haCoordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { - c.mutex.Lock() - // Tell clients on other instances to send a callmemaybe to us. err := c.publishAgentHello(id) if err != nil { @@ -203,6 +191,7 @@ func (c *haCoordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { // If an old agent socket is connected, we close it // to avoid any leaks. This shouldn't ever occur because // we expect one agent to be running. + c.mutex.Lock() oldAgentSocket, ok := c.agentSockets[id] if ok { _ = oldAgentSocket.Close() @@ -234,6 +223,8 @@ func (c *haCoordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { } func (c *haCoordinator) nodesSubscribedToAgent(agentID uuid.UUID) []*agpl.Node { + c.mutex.Lock() + defer c.mutex.Unlock() sockets, ok := c.agentToConnectionSockets[agentID] if !ok { return nil @@ -279,12 +270,11 @@ func (c *haCoordinator) hangleAgentUpdate(id uuid.UUID, decoder *json.Decoder) ( for _, connectionSocket := range connectionSockets { connectionSocket := connectionSocket go func() { + defer wg.Done() _ = connectionSocket.SetWriteDeadline(time.Now().Add(5 * time.Second)) _, _ = connectionSocket.Write(data) - wg.Done() }() } - wg.Wait() return &node, nil } @@ -428,9 +418,7 @@ func (c *haCoordinator) runPubsub() error { return } - c.mutex.Lock() nodes := c.nodesSubscribedToAgent(agentUUID) - c.mutex.Unlock() if len(nodes) > 0 { err := c.publishNodesToAgent(agentUUID, nodes) if err != nil { diff --git a/tailnet/coordinator.go b/tailnet/coordinator.go index 96de8d295162e..23531af1260f5 100644 --- a/tailnet/coordinator.go +++ b/tailnet/coordinator.go @@ -127,25 +127,26 @@ type coordinator struct { } // Node returns an in-memory node by ID. +// If the node does not exist, nil is returned. func (c *coordinator) Node(id uuid.UUID) *Node { c.mutex.Lock() defer c.mutex.Unlock() - node := c.nodes[id] - return node + return c.nodes[id] } // ServeClient accepts a WebSocket connection that wants to connect to an agent // with the specified ID. func (c *coordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID) error { c.mutex.Lock() - if c.closed { + c.mutex.Unlock() return xerrors.New("coordinator is closed") } // When a new connection is requested, we update it with the latest // node of the agent. This allows the connection to establish. node, ok := c.nodes[agent] + c.mutex.Unlock() if ok { data, err := json.Marshal([]*Node{node}) if err != nil { @@ -158,6 +159,7 @@ func (c *coordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID) return xerrors.Errorf("write nodes: %w", err) } } + c.mutex.Lock() connectionSockets, ok := c.agentToConnectionSockets[agent] if !ok { connectionSockets = map[uuid.UUID]net.Conn{} @@ -203,7 +205,6 @@ func (c *coordinator) handleNextClientMessage(id, agent uuid.UUID, decoder *json } c.mutex.Lock() - // Update the node of this client in our in-memory map. If an agent entirely // shuts down and reconnects, it needs to be aware of all clients attempting // to establish connections. @@ -237,12 +238,13 @@ func (c *coordinator) handleNextClientMessage(id, agent uuid.UUID, decoder *json // listens to incoming connections and publishes node updates. func (c *coordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { c.mutex.Lock() - if c.closed { + c.mutex.Unlock() return xerrors.New("coordinator is closed") } sockets, ok := c.agentToConnectionSockets[id] + c.mutex.Unlock() if ok { // Publish all nodes that want to connect to the // desired agent ID. @@ -269,6 +271,7 @@ func (c *coordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { // If an old agent socket is connected, we close it // to avoid any leaks. This shouldn't ever occur because // we expect one agent to be running. + c.mutex.Lock() oldAgentSocket, ok := c.agentSockets[id] if ok { _ = oldAgentSocket.Close() @@ -302,17 +305,15 @@ func (c *coordinator) handleNextAgentMessage(id uuid.UUID, decoder *json.Decoder } c.mutex.Lock() - c.nodes[id] = &node connectionSockets, ok := c.agentToConnectionSockets[id] if !ok { c.mutex.Unlock() return nil } - + c.mutex.Unlock() data, err := json.Marshal([]*Node{&node}) if err != nil { - c.mutex.Unlock() return xerrors.Errorf("marshal nodes: %w", err) } @@ -328,7 +329,6 @@ func (c *coordinator) handleNextAgentMessage(id uuid.UUID, decoder *json.Decoder }() } - c.mutex.Unlock() wg.Wait() return nil } @@ -337,9 +337,11 @@ func (c *coordinator) handleNextAgentMessage(id uuid.UUID, decoder *json.Decoder // coordinator from accepting new connections. func (c *coordinator) Close() error { c.mutex.Lock() - defer c.mutex.Unlock() - + if c.closed { + return nil + } c.closed = true + c.mutex.Unlock() wg := sync.WaitGroup{} From d8580d107a16c4802262b9f8919175512c8d6ec6 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 05:24:35 +0000 Subject: [PATCH 38/79] Fix linting --- coderd/coderdtest/coderdtest.go | 1 + enterprise/coderd/replicas_test.go | 3 +-- enterprise/derpmesh/derpmesh_test.go | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index cbbcd7aaa493a..4141e33cf8648 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -163,6 +163,7 @@ func NewOptions(t *testing.T, options *Options) (*httptest.Server, context.Cance if options.TLSCertificates != nil { srv.TLS = &tls.Config{ Certificates: options.TLSCertificates, + MinVersion: tls.VersionTLS12, } srv.StartTLS() } else { diff --git a/enterprise/coderd/replicas_test.go b/enterprise/coderd/replicas_test.go index 9d6970823befb..f9f6e138bd3cc 100644 --- a/enterprise/coderd/replicas_test.go +++ b/enterprise/coderd/replicas_test.go @@ -4,7 +4,6 @@ import ( "context" "crypto/tls" "testing" - "time" "github.com/stretchr/testify/require" @@ -80,7 +79,7 @@ func TestReplicas(t *testing.T) { }) require.NoError(t, err) require.Eventually(t, func() bool { - ctx, cancelFunc := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.IntervalSlow) defer cancelFunc() _, err = conn.Ping(ctx) return err == nil diff --git a/enterprise/derpmesh/derpmesh_test.go b/enterprise/derpmesh/derpmesh_test.go index fcf410ac0e574..d1131d59da25b 100644 --- a/enterprise/derpmesh/derpmesh_test.go +++ b/enterprise/derpmesh/derpmesh_test.go @@ -36,6 +36,7 @@ func TestDERPMesh(t *testing.T) { pool := x509.NewCertPool() pool.AddCert(certificate) tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, ServerName: commonName, RootCAs: pool, Certificates: []tls.Certificate{rawCert}, From ae956fbc00df6bed796e3624b515849002b8bb21 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 05:44:04 +0000 Subject: [PATCH 39/79] Store mesh key in the database --- coderd/coderd.go | 1 - coderd/database/databasefake/databasefake.go | 20 +++++++++++++++-- coderd/database/querier.go | 2 ++ coderd/database/queries.sql.go | 20 +++++++++++++++++ coderd/database/queries/siteconfig.sql | 6 +++++ enterprise/cli/server.go | 23 ++++++++++++++++++++ 6 files changed, 69 insertions(+), 3 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index 735190373a4f3..2fe0a5dc0d08e 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -127,7 +127,6 @@ func New(options *Options) *API { } if options.DERPServer == nil { options.DERPServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger.Named("derp"))) - options.DERPServer.SetMeshKey("todo-kyle-change-this") } if options.Auditor == nil { options.Auditor = audit.NewNop() diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index b4724a9afe0aa..e860c69dd11cb 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -110,10 +110,11 @@ type data struct { replicas []database.Replica deploymentID string + derpMeshKey string lastLicenseID int32 } -func (q *fakeQuerier) Ping(_ context.Context) (time.Duration, error) { +func (*fakeQuerier) Ping(_ context.Context) (time.Duration, error) { return 0, nil } @@ -2890,6 +2891,21 @@ func (q *fakeQuerier) GetDeploymentID(_ context.Context) (string, error) { return q.deploymentID, nil } +func (q *fakeQuerier) InsertDERPMeshKey(_ context.Context, id string) error { + q.mutex.Lock() + defer q.mutex.Unlock() + + q.derpMeshKey = id + return nil +} + +func (q *fakeQuerier) GetDERPMeshKey(_ context.Context) (string, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + + return q.derpMeshKey, nil +} + func (q *fakeQuerier) InsertLicense( _ context.Context, arg database.InsertLicenseParams, ) (database.License, error) { @@ -3156,7 +3172,7 @@ func (q *fakeQuerier) DeleteGroupByID(_ context.Context, id uuid.UUID) error { return sql.ErrNoRows } -func (q *fakeQuerier) DeleteReplicasUpdatedBefore(ctx context.Context, before time.Time) error { +func (q *fakeQuerier) DeleteReplicasUpdatedBefore(_ context.Context, before time.Time) error { q.mutex.Lock() defer q.mutex.Unlock() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 957de26c89e05..7f2f0d942bb10 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -39,6 +39,7 @@ type sqlcQuerier interface { // This function returns roles for authorization purposes. Implied member roles // are included. GetAuthorizationUserRoles(ctx context.Context, userID uuid.UUID) (GetAuthorizationUserRolesRow, error) + GetDERPMeshKey(ctx context.Context) (string, error) GetDeploymentID(ctx context.Context) (string, error) GetFileByHashAndCreator(ctx context.Context, arg GetFileByHashAndCreatorParams) (File, error) GetFileByID(ctx context.Context, id uuid.UUID) (File, error) @@ -125,6 +126,7 @@ type sqlcQuerier interface { // every member of the org. InsertAllUsersGroup(ctx context.Context, organizationID uuid.UUID) (Group, error) InsertAuditLog(ctx context.Context, arg InsertAuditLogParams) (AuditLog, error) + InsertDERPMeshKey(ctx context.Context, value string) error InsertDeploymentID(ctx context.Context, value string) error InsertFile(ctx context.Context, arg InsertFileParams) (File, error) InsertGitSSHKey(ctx context.Context, arg InsertGitSSHKeyParams) (GitSSHKey, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 241474e7e66bd..c40b93426ddee 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -2753,6 +2753,17 @@ func (q *sqlQuerier) UpdateReplica(ctx context.Context, arg UpdateReplicaParams) return i, err } +const getDERPMeshKey = `-- name: GetDERPMeshKey :one +SELECT value FROM site_configs WHERE key = 'derp_mesh_key' +` + +func (q *sqlQuerier) GetDERPMeshKey(ctx context.Context) (string, error) { + row := q.db.QueryRowContext(ctx, getDERPMeshKey) + var value string + err := row.Scan(&value) + return value, err +} + const getDeploymentID = `-- name: GetDeploymentID :one SELECT value FROM site_configs WHERE key = 'deployment_id' ` @@ -2764,6 +2775,15 @@ func (q *sqlQuerier) GetDeploymentID(ctx context.Context) (string, error) { return value, err } +const insertDERPMeshKey = `-- name: InsertDERPMeshKey :exec +INSERT INTO site_configs (key, value) VALUES ('derp_mesh_key', $1) +` + +func (q *sqlQuerier) InsertDERPMeshKey(ctx context.Context, value string) error { + _, err := q.db.ExecContext(ctx, insertDERPMeshKey, value) + return err +} + const insertDeploymentID = `-- name: InsertDeploymentID :exec INSERT INTO site_configs (key, value) VALUES ('deployment_id', $1) ` diff --git a/coderd/database/queries/siteconfig.sql b/coderd/database/queries/siteconfig.sql index 9d3936e23886d..b975d2f68cc3c 100644 --- a/coderd/database/queries/siteconfig.sql +++ b/coderd/database/queries/siteconfig.sql @@ -3,3 +3,9 @@ INSERT INTO site_configs (key, value) VALUES ('deployment_id', $1); -- name: GetDeploymentID :one SELECT value FROM site_configs WHERE key = 'deployment_id'; + +-- name: InsertDERPMeshKey :exec +INSERT INTO site_configs (key, value) VALUES ('derp_mesh_key', $1); + +-- name: GetDERPMeshKey :one +SELECT value FROM site_configs WHERE key = 'derp_mesh_key'; diff --git a/enterprise/cli/server.go b/enterprise/cli/server.go index f3e99c1613ab8..a65b8e8faa6e0 100644 --- a/enterprise/cli/server.go +++ b/enterprise/cli/server.go @@ -2,14 +2,20 @@ package cli import ( "context" + "database/sql" + "errors" "io" "net/url" "github.com/spf13/cobra" "golang.org/x/xerrors" + "tailscale.com/derp" + "tailscale.com/types/key" "github.com/coder/coder/cli/deployment" + "github.com/coder/coder/cryptorand" "github.com/coder/coder/enterprise/coderd" + "github.com/coder/coder/tailnet" agpl "github.com/coder/coder/cli" agplcoderd "github.com/coder/coder/coderd" @@ -25,6 +31,23 @@ func server() *cobra.Command { } } + options.DERPServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger.Named("derp"))) + meshKey, err := options.Database.GetDERPMeshKey(ctx) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + return nil, nil, xerrors.Errorf("get mesh key: %w", err) + } + meshKey, err = cryptorand.String(32) + if err != nil { + return nil, nil, xerrors.Errorf("generate mesh key: %w", err) + } + err = options.Database.InsertDERPMeshKey(ctx, meshKey) + if err != nil { + return nil, nil, xerrors.Errorf("insert mesh key: %w", err) + } + } + options.DERPServer.SetMeshKey(meshKey) + o := &coderd.Options{ AuditLogging: dflags.AuditLogging.Value, BrowserOnly: dflags.BrowserOnly.Value, From d703e2d08aeb82ac80e5adea594ec0951a1e80a6 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 06:37:32 +0000 Subject: [PATCH 40/79] Fix replica key for tests --- coderd/coderdtest/coderdtest.go | 7 +++++++ enterprise/coderd/replicas_test.go | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index 4141e33cf8648..c0361dfa8e2b8 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -37,8 +37,10 @@ import ( "golang.org/x/xerrors" "google.golang.org/api/idtoken" "google.golang.org/api/option" + "tailscale.com/derp" "tailscale.com/net/stun/stuntest" "tailscale.com/tailcfg" + "tailscale.com/types/key" "tailscale.com/types/nettype" "cdr.dev/slog" @@ -59,6 +61,7 @@ import ( "github.com/coder/coder/provisionerd" "github.com/coder/coder/provisionersdk" "github.com/coder/coder/provisionersdk/proto" + "github.com/coder/coder/tailnet" "github.com/coder/coder/testutil" ) @@ -184,6 +187,9 @@ func NewOptions(t *testing.T, options *Options) (*httptest.Server, context.Cance stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{}) t.Cleanup(stunCleanup) + derpServer := derp.NewServer(key.NewNode(), tailnet.Logger(slogtest.Make(t, nil).Named("derp"))) + derpServer.SetMeshKey("test-key") + // match default with cli default if options.SSHKeygenAlgorithm == "" { options.SSHKeygenAlgorithm = gitsshkey.AlgorithmEd25519 @@ -208,6 +214,7 @@ func NewOptions(t *testing.T, options *Options) (*httptest.Server, context.Cance OIDCConfig: options.OIDCConfig, GoogleTokenValidator: options.GoogleTokenValidator, SSHKeygenAlgorithm: options.SSHKeygenAlgorithm, + DERPServer: derpServer, APIRateLimit: options.APIRateLimit, Authorizer: options.Authorizer, Telemetry: telemetry.NewNoop(), diff --git a/enterprise/coderd/replicas_test.go b/enterprise/coderd/replicas_test.go index f9f6e138bd3cc..fae418ab87261 100644 --- a/enterprise/coderd/replicas_test.go +++ b/enterprise/coderd/replicas_test.go @@ -79,7 +79,7 @@ func TestReplicas(t *testing.T) { }) require.NoError(t, err) require.Eventually(t, func() bool { - ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.IntervalSlow) + ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitShort) defer cancelFunc() _, err = conn.Ping(ctx) return err == nil @@ -122,7 +122,7 @@ func TestReplicas(t *testing.T) { }) require.NoError(t, err) require.Eventually(t, func() bool { - ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.IntervalMedium) + ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.IntervalSlow) defer cancelFunc() _, err = conn.Ping(ctx) return err == nil From 9bb021c0e69f07efb4fb499eecad1359575432fc Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 06:38:08 +0000 Subject: [PATCH 41/79] Fix types gen --- site/src/api/api.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/site/src/api/api.ts b/site/src/api/api.ts index 2e60a88b8469c..fb12571fd91ae 100644 --- a/site/src/api/api.ts +++ b/site/src/api/api.ts @@ -28,6 +28,7 @@ export const defaultEntitlements = (): TypesGen.Entitlements => { return { features: features, has_license: false, + errors: [], warnings: [], experimental: false, trial: false, From 76c9e2c959bb7b260ebed3274ce525e5cac813a0 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 06:44:56 +0000 Subject: [PATCH 42/79] Fix unlocking unlocked --- tailnet/coordinator.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tailnet/coordinator.go b/tailnet/coordinator.go index 23531af1260f5..9d722ddeee117 100644 --- a/tailnet/coordinator.go +++ b/tailnet/coordinator.go @@ -263,7 +263,6 @@ func (c *coordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { } _, err = conn.Write(data) if err != nil { - c.mutex.Unlock() return xerrors.Errorf("write nodes: %w", err) } } From 09e87b0aa06da43125ed9cc767c2fd7877657f9f Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 06:57:46 +0000 Subject: [PATCH 43/79] Fix race in tests --- tailnet/coordinator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tailnet/coordinator.go b/tailnet/coordinator.go index 9d722ddeee117..491c0db885224 100644 --- a/tailnet/coordinator.go +++ b/tailnet/coordinator.go @@ -310,7 +310,6 @@ func (c *coordinator) handleNextAgentMessage(id uuid.UUID, decoder *json.Decoder c.mutex.Unlock() return nil } - c.mutex.Unlock() data, err := json.Marshal([]*Node{&node}) if err != nil { return xerrors.Errorf("marshal nodes: %w", err) @@ -328,6 +327,7 @@ func (c *coordinator) handleNextAgentMessage(id uuid.UUID, decoder *json.Decoder }() } + c.mutex.Unlock() wg.Wait() return nil } From 18c0464e7f106d81c3dbdb84f2f7ad48d75fc5e3 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 01:58:05 -0500 Subject: [PATCH 44/79] Update enterprise/derpmesh/derpmesh.go Co-authored-by: Colin Adler --- enterprise/derpmesh/derpmesh.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enterprise/derpmesh/derpmesh.go b/enterprise/derpmesh/derpmesh.go index 8f51343017593..059eac5a107e7 100644 --- a/enterprise/derpmesh/derpmesh.go +++ b/enterprise/derpmesh/derpmesh.go @@ -82,7 +82,7 @@ func (m *Mesh) SetAddresses(addresses []string) { m.mutex.Unlock() } -// addAddress begins meshing with a new address. +// addAddress begins meshing with a new address. It returns false if the address is already being meshed with. // It's expected that this is a full HTTP address with a path. // e.g. http://127.0.0.1:8080/derp func (m *Mesh) addAddress(address string) (bool, error) { From 6f25b2d44b66b12c5c65c20c0095588c3b51347b Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 06:58:54 +0000 Subject: [PATCH 45/79] Rename to syncReplicas --- enterprise/replicasync/replicasync.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/enterprise/replicasync/replicasync.go b/enterprise/replicasync/replicasync.go index b635a84991e24..46123953298de 100644 --- a/enterprise/replicasync/replicasync.go +++ b/enterprise/replicasync/replicasync.go @@ -73,7 +73,7 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, pubsub data return nil, xerrors.Errorf("publish new replica: %w", err) } ctx, cancelFunc := context.WithCancel(ctx) - server := &Manager{ + manager := &Manager{ id: id, options: options, db: db, @@ -83,25 +83,25 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, pubsub data closed: make(chan struct{}), closeCancel: cancelFunc, } - err = server.run(ctx) + err = manager.syncReplicas(ctx) if err != nil { return nil, xerrors.Errorf("run replica: %w", err) } - peers := server.Regional() + peers := manager.Regional() if len(peers) > 0 { - self := server.Self() + self := manager.Self() if self.RelayAddress == "" { return nil, xerrors.Errorf("a relay address must be specified when running multiple replicas in the same region") } } - err = server.subscribe(ctx) + err = manager.subscribe(ctx) if err != nil { return nil, xerrors.Errorf("subscribe: %w", err) } - server.closeWait.Add(1) - go server.loop(ctx) - return server, nil + manager.closeWait.Add(1) + go manager.loop(ctx) + return manager, nil } // Manager keeps the replica up to date and in sync with other replicas. @@ -134,7 +134,7 @@ func (m *Manager) loop(ctx context.Context) { return case <-ticker.C: } - err := m.run(ctx) + err := m.syncReplicas(ctx) if err != nil && !errors.Is(err, context.Canceled) { m.logger.Warn(ctx, "run replica update loop", slog.Error(err)) } @@ -155,7 +155,7 @@ func (m *Manager) subscribe(ctx context.Context) error { // it will reprocess afterwards. var update func() update = func() { - err := m.run(ctx) + err := m.syncReplicas(ctx) if err != nil && !errors.Is(err, context.Canceled) { m.logger.Error(ctx, "run replica from subscribe", slog.Error(err)) } @@ -197,7 +197,7 @@ func (m *Manager) subscribe(ctx context.Context) error { return nil } -func (m *Manager) run(ctx context.Context) error { +func (m *Manager) syncReplicas(ctx context.Context) error { m.closeMutex.Lock() m.closeWait.Add(1) m.closeMutex.Unlock() From 1e85039d346e5e143d5a3ae462681f11d5cff093 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 19:52:19 +0000 Subject: [PATCH 46/79] Reuse http client --- enterprise/replicasync/replicasync.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/enterprise/replicasync/replicasync.go b/enterprise/replicasync/replicasync.go index 46123953298de..317f6dc274bdc 100644 --- a/enterprise/replicasync/replicasync.go +++ b/enterprise/replicasync/replicasync.go @@ -219,6 +219,13 @@ func (m *Manager) syncReplicas(ctx context.Context) error { } m.mutex.Unlock() + client := http.Client{ + Timeout: m.options.PeerTimeout, + Transport: &http.Transport{ + TLSClientConfig: m.options.TLSConfig, + }, + } + defer client.CloseIdleConnections() var wg sync.WaitGroup var mu sync.Mutex failed := make([]string, 0) @@ -232,12 +239,6 @@ func (m *Manager) syncReplicas(ctx context.Context) error { slog.F("relay_address", peer.RelayAddress), slog.Error(err)) return } - client := http.Client{ - Timeout: m.options.PeerTimeout, - Transport: &http.Transport{ - TLSClientConfig: m.options.TLSConfig, - }, - } res, err := client.Do(req) if err != nil { mu.Lock() From ae0aa5f226bdc2e0e692e09fcee05e6a45d0247e Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 20:55:11 +0000 Subject: [PATCH 47/79] Delete old replicas on a CRON --- coderd/database/databasefake/databasefake.go | 12 ------ coderd/database/querier.go | 1 - coderd/database/queries.sql.go | 23 ------------ coderd/database/queries/replicas.sql | 3 -- enterprise/replicasync/replicasync.go | 39 +++++++++++++++----- enterprise/replicasync/replicasync_test.go | 18 +++++++++ 6 files changed, 48 insertions(+), 48 deletions(-) diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index e860c69dd11cb..d95499147066b 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -3238,15 +3238,3 @@ func (q *fakeQuerier) GetReplicasUpdatedAfter(_ context.Context, updatedAt time. } return replicas, nil } - -func (q *fakeQuerier) GetReplicaByID(_ context.Context, id uuid.UUID) (database.Replica, error) { - q.mutex.RLock() - defer q.mutex.RUnlock() - - for _, replica := range q.replicas { - if replica.ID == id { - return replica, nil - } - } - return database.Replica{}, sql.ErrNoRows -} diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 7f2f0d942bb10..8d1ea946ff5b6 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -69,7 +69,6 @@ type sqlcQuerier interface { GetProvisionerJobsByIDs(ctx context.Context, ids []uuid.UUID) ([]ProvisionerJob, error) GetProvisionerJobsCreatedAfter(ctx context.Context, createdAt time.Time) ([]ProvisionerJob, error) GetProvisionerLogsByIDBetween(ctx context.Context, arg GetProvisionerLogsByIDBetweenParams) ([]ProvisionerJobLog, error) - GetReplicaByID(ctx context.Context, id uuid.UUID) (Replica, error) GetReplicasUpdatedAfter(ctx context.Context, updatedAt time.Time) ([]Replica, error) GetTemplateByID(ctx context.Context, id uuid.UUID) (Template, error) GetTemplateByOrganizationAndName(ctx context.Context, arg GetTemplateByOrganizationAndNameParams) (Template, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index c40b93426ddee..ff72247ad1e0f 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -2579,29 +2579,6 @@ func (q *sqlQuerier) DeleteReplicasUpdatedBefore(ctx context.Context, updatedAt return err } -const getReplicaByID = `-- name: GetReplicaByID :one -SELECT id, created_at, started_at, stopped_at, updated_at, hostname, region_id, relay_address, database_latency, version, error FROM replicas WHERE id = $1 -` - -func (q *sqlQuerier) GetReplicaByID(ctx context.Context, id uuid.UUID) (Replica, error) { - row := q.db.QueryRowContext(ctx, getReplicaByID, id) - var i Replica - err := row.Scan( - &i.ID, - &i.CreatedAt, - &i.StartedAt, - &i.StoppedAt, - &i.UpdatedAt, - &i.Hostname, - &i.RegionID, - &i.RelayAddress, - &i.DatabaseLatency, - &i.Version, - &i.Error, - ) - return i, err -} - const getReplicasUpdatedAfter = `-- name: GetReplicasUpdatedAfter :many SELECT id, created_at, started_at, stopped_at, updated_at, hostname, region_id, relay_address, database_latency, version, error FROM replicas WHERE updated_at > $1 AND stopped_at IS NULL ` diff --git a/coderd/database/queries/replicas.sql b/coderd/database/queries/replicas.sql index 5a62527fac107..e87c1f46432f2 100644 --- a/coderd/database/queries/replicas.sql +++ b/coderd/database/queries/replicas.sql @@ -1,9 +1,6 @@ -- name: GetReplicasUpdatedAfter :many SELECT * FROM replicas WHERE updated_at > $1 AND stopped_at IS NULL; --- name: GetReplicaByID :one -SELECT * FROM replicas WHERE id = $1; - -- name: InsertReplica :one INSERT INTO replicas ( id, diff --git a/enterprise/replicasync/replicasync.go b/enterprise/replicasync/replicasync.go index 317f6dc274bdc..d6cd846d6c96f 100644 --- a/enterprise/replicasync/replicasync.go +++ b/enterprise/replicasync/replicasync.go @@ -26,11 +26,12 @@ var ( ) type Options struct { - UpdateInterval time.Duration - PeerTimeout time.Duration - RelayAddress string - RegionID int32 - TLSConfig *tls.Config + CleanupInterval time.Duration + UpdateInterval time.Duration + PeerTimeout time.Duration + RelayAddress string + RegionID int32 + TLSConfig *tls.Config } // New registers the replica with the database and periodically updates to ensure @@ -45,6 +46,11 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, pubsub data if options.UpdateInterval == 0 { options.UpdateInterval = 5 * time.Second } + if options.CleanupInterval == 0 { + // The cleanup interval can be quite long, because it's + // primary purpose is to clean up dead replicas. + options.CleanupInterval = 30 * time.Minute + } hostname, err := os.Hostname() if err != nil { return nil, xerrors.Errorf("get hostname: %w", err) @@ -123,16 +129,31 @@ type Manager struct { callback func() } +// updateInterval is used to determine a replicas state. +// If the replica was updated > the time, it's considered healthy. +// If the replica was updated < the time, it's considered stale. +func (m *Manager) updateInterval() time.Time { + return database.Now().Add(-3 * m.options.UpdateInterval) +} + // loop runs the replica update sequence on an update interval. func (m *Manager) loop(ctx context.Context) { defer m.closeWait.Done() - ticker := time.NewTicker(m.options.UpdateInterval) - defer ticker.Stop() + updateTicker := time.NewTicker(m.options.UpdateInterval) + defer updateTicker.Stop() + deleteTicker := time.NewTicker(m.options.CleanupInterval) + defer deleteTicker.Stop() for { select { case <-ctx.Done(): return - case <-ticker.C: + case <-deleteTicker.C: + err := m.db.DeleteReplicasUpdatedBefore(ctx, m.updateInterval()) + if err != nil { + m.logger.Warn(ctx, "delete old replicas", slog.Error(err)) + } + continue + case <-updateTicker.C: } err := m.syncReplicas(ctx) if err != nil && !errors.Is(err, context.Canceled) { @@ -204,7 +225,7 @@ func (m *Manager) syncReplicas(ctx context.Context) error { defer m.closeWait.Done() // Expect replicas to update once every three times the interval... // If they don't, assume death! - replicas, err := m.db.GetReplicasUpdatedAfter(ctx, database.Now().Add(-3*m.options.UpdateInterval)) + replicas, err := m.db.GetReplicasUpdatedAfter(ctx, m.updateInterval()) if err != nil { return xerrors.Errorf("get replicas: %w", err) } diff --git a/enterprise/replicasync/replicasync_test.go b/enterprise/replicasync/replicasync_test.go index 0b42f44791df4..79acf86865839 100644 --- a/enterprise/replicasync/replicasync_test.go +++ b/enterprise/replicasync/replicasync_test.go @@ -178,6 +178,24 @@ func TestReplica(t *testing.T) { }, testutil.WaitShort, testutil.IntervalFast) _ = server.Close() }) + t.Run("DeletesOld", func(t *testing.T) { + t.Parallel() + db, pubsub := dbtestutil.NewDB(t) + _, err := db.InsertReplica(context.Background(), database.InsertReplicaParams{ + ID: uuid.New(), + UpdatedAt: database.Now().Add(-time.Hour), + }) + require.NoError(t, err) + server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, &replicasync.Options{ + RelayAddress: "google.com", + CleanupInterval: time.Millisecond, + }) + require.NoError(t, err) + defer server.Close() + require.Eventually(t, func() bool { + return len(server.Regional()) == 0 + }, testutil.WaitShort, testutil.IntervalFast) + }) t.Run("TwentyConcurrent", func(t *testing.T) { // Ensures that twenty concurrent replicas can spawn and all // discover each other in parallel! From bd7fb1314256227c578d659911f3f7ba0d37743f Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 23:18:43 +0000 Subject: [PATCH 48/79] Fix race condition in connection tests --- enterprise/coderd/coderd.go | 4 ++-- enterprise/derpmesh/derpmesh.go | 12 +++++++++--- enterprise/derpmesh/derpmesh_test.go | 10 +++++----- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 371ac12fe21b8..1250e6ae129da 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -283,11 +283,11 @@ func (api *API) updateEntitlements(ctx context.Context) error { for _, replica := range api.replicaManager.Regional() { addresses = append(addresses, replica.RelayAddress) } - api.derpMesh.SetAddresses(addresses) + api.derpMesh.SetAddresses(addresses, false) _ = api.updateEntitlements(ctx) }) } else { - api.derpMesh.SetAddresses([]string{}) + api.derpMesh.SetAddresses([]string{}, false) api.replicaManager.SetCallback(func() { // If the amount of replicas change, so should our entitlements. // This is to display a warning in the UI if the user is unlicensed. diff --git a/enterprise/derpmesh/derpmesh.go b/enterprise/derpmesh/derpmesh.go index 059eac5a107e7..530c799908fca 100644 --- a/enterprise/derpmesh/derpmesh.go +++ b/enterprise/derpmesh/derpmesh.go @@ -42,7 +42,10 @@ type Mesh struct { // SetAddresses performs a diff of the incoming addresses and adds // or removes DERP clients from the mesh. -func (m *Mesh) SetAddresses(addresses []string) { +// +// Connect is only used for testing to ensure DERPs are meshed before +// exchanging messages. +func (m *Mesh) SetAddresses(addresses []string, connect bool) { total := make(map[string]struct{}, 0) for _, address := range addresses { addressURL, err := url.Parse(address) @@ -58,7 +61,7 @@ func (m *Mesh) SetAddresses(addresses []string) { address = derpURL.String() total[address] = struct{}{} - added, err := m.addAddress(address) + added, err := m.addAddress(address, connect) if err != nil { m.logger.Error(m.ctx, "failed to add address", slog.F("address", address), slog.Error(err)) continue @@ -85,7 +88,7 @@ func (m *Mesh) SetAddresses(addresses []string) { // addAddress begins meshing with a new address. It returns false if the address is already being meshed with. // It's expected that this is a full HTTP address with a path. // e.g. http://127.0.0.1:8080/derp -func (m *Mesh) addAddress(address string) (bool, error) { +func (m *Mesh) addAddress(address string, connect bool) (bool, error) { m.mutex.Lock() defer m.mutex.Unlock() _, isActive := m.active[address] @@ -102,6 +105,9 @@ func (m *Mesh) addAddress(address string) (bool, error) { var dialer net.Dialer return dialer.DialContext(ctx, network, addr) }) + if connect { + _ = client.Connect(m.ctx) + } ctx, cancelFunc := context.WithCancel(m.ctx) closed := make(chan struct{}) closeFunc := func() { diff --git a/enterprise/derpmesh/derpmesh_test.go b/enterprise/derpmesh/derpmesh_test.go index d1131d59da25b..84875f106c7f2 100644 --- a/enterprise/derpmesh/derpmesh_test.go +++ b/enterprise/derpmesh/derpmesh_test.go @@ -49,9 +49,9 @@ func TestDERPMesh(t *testing.T) { defer firstServer.Close() secondServer, secondServerURL := startDERP(t, tlsConfig) firstMesh := derpmesh.New(slogtest.Make(t, nil).Named("first").Leveled(slog.LevelDebug), firstServer, tlsConfig) - firstMesh.SetAddresses([]string{secondServerURL}) + firstMesh.SetAddresses([]string{secondServerURL}, true) secondMesh := derpmesh.New(slogtest.Make(t, nil).Named("second").Leveled(slog.LevelDebug), secondServer, tlsConfig) - secondMesh.SetAddresses([]string{firstServerURL}) + secondMesh.SetAddresses([]string{firstServerURL}, true) defer firstMesh.Close() defer secondMesh.Close() @@ -78,9 +78,9 @@ func TestDERPMesh(t *testing.T) { t.Parallel() server, serverURL := startDERP(t, tlsConfig) mesh := derpmesh.New(slogtest.Make(t, nil).Named("first").Leveled(slog.LevelDebug), server, tlsConfig) - mesh.SetAddresses([]string{"http://fake.com"}) + mesh.SetAddresses([]string{"http://fake.com"}, false) // This should trigger a removal... - mesh.SetAddresses([]string{}) + mesh.SetAddresses([]string{}, false) defer mesh.Close() first := key.NewNode() @@ -114,7 +114,7 @@ func TestDERPMesh(t *testing.T) { meshes = append(meshes, mesh) } for _, mesh := range meshes { - mesh.SetAddresses(serverURLs) + mesh.SetAddresses(serverURLs, true) } first := key.NewNode() From bb5b347ada043f7ed0a30eb9f6754c8ae25416d4 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 23:26:50 +0000 Subject: [PATCH 49/79] Fix linting --- enterprise/derpmesh/derpmesh.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/enterprise/derpmesh/derpmesh.go b/enterprise/derpmesh/derpmesh.go index 530c799908fca..5de7799aa74eb 100644 --- a/enterprise/derpmesh/derpmesh.go +++ b/enterprise/derpmesh/derpmesh.go @@ -45,6 +45,7 @@ type Mesh struct { // // Connect is only used for testing to ensure DERPs are meshed before // exchanging messages. +// nolint:revive func (m *Mesh) SetAddresses(addresses []string, connect bool) { total := make(map[string]struct{}, 0) for _, address := range addresses { @@ -88,6 +89,7 @@ func (m *Mesh) SetAddresses(addresses []string, connect bool) { // addAddress begins meshing with a new address. It returns false if the address is already being meshed with. // It's expected that this is a full HTTP address with a path. // e.g. http://127.0.0.1:8080/derp +// nolint:revive func (m *Mesh) addAddress(address string, connect bool) (bool, error) { m.mutex.Lock() defer m.mutex.Unlock() From 76e0511efef918d121b7834c11bedf3cea0a4771 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 15 Oct 2022 23:38:15 +0000 Subject: [PATCH 50/79] Fix nil type --- coderd/database/dump.sql | 2 +- .../migrations/000061_replicas.up.sql | 2 +- coderd/database/models.go | 22 +++++++++---------- coderd/database/queries.sql.go | 20 ++++++++--------- enterprise/coderd/replicas.go | 2 +- enterprise/replicasync/replicasync.go | 9 +++----- enterprise/replicasync/replicasync_test.go | 8 +++---- 7 files changed, 31 insertions(+), 34 deletions(-) diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 1e0a18c1dafef..8e31a990a8925 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -295,7 +295,7 @@ CREATE TABLE replicas ( relay_address text NOT NULL, database_latency integer NOT NULL, version text NOT NULL, - error text + error text DEFAULT ''::text NOT NULL ); CREATE TABLE site_configs ( diff --git a/coderd/database/migrations/000061_replicas.up.sql b/coderd/database/migrations/000061_replicas.up.sql index b1d1a1ab13ee0..1400662e30582 100644 --- a/coderd/database/migrations/000061_replicas.up.sql +++ b/coderd/database/migrations/000061_replicas.up.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS replicas ( database_latency int NOT NULL, -- Version is the Coder version of the replica. version text NOT NULL, - error text + error text NOT NULL DEFAULT '' ); -- Associates a provisioner daemon with a replica. diff --git a/coderd/database/models.go b/coderd/database/models.go index b4601ecadeb78..53e074984ac11 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -540,17 +540,17 @@ type ProvisionerJobLog struct { } type Replica struct { - ID uuid.UUID `db:"id" json:"id"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - StartedAt time.Time `db:"started_at" json:"started_at"` - StoppedAt sql.NullTime `db:"stopped_at" json:"stopped_at"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` - Hostname string `db:"hostname" json:"hostname"` - RegionID int32 `db:"region_id" json:"region_id"` - RelayAddress string `db:"relay_address" json:"relay_address"` - DatabaseLatency int32 `db:"database_latency" json:"database_latency"` - Version string `db:"version" json:"version"` - Error sql.NullString `db:"error" json:"error"` + ID uuid.UUID `db:"id" json:"id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + StartedAt time.Time `db:"started_at" json:"started_at"` + StoppedAt sql.NullTime `db:"stopped_at" json:"stopped_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + Hostname string `db:"hostname" json:"hostname"` + RegionID int32 `db:"region_id" json:"region_id"` + RelayAddress string `db:"relay_address" json:"relay_address"` + DatabaseLatency int32 `db:"database_latency" json:"database_latency"` + Version string `db:"version" json:"version"` + Error string `db:"error" json:"error"` } type SiteConfig struct { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 1c10bc259c72c..aa76ddfec52a3 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -2698,16 +2698,16 @@ WHERE id = $1 RETURNING id, created_at, started_at, stopped_at, updated_at, host ` type UpdateReplicaParams struct { - ID uuid.UUID `db:"id" json:"id"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` - StartedAt time.Time `db:"started_at" json:"started_at"` - StoppedAt sql.NullTime `db:"stopped_at" json:"stopped_at"` - RelayAddress string `db:"relay_address" json:"relay_address"` - RegionID int32 `db:"region_id" json:"region_id"` - Hostname string `db:"hostname" json:"hostname"` - Version string `db:"version" json:"version"` - Error sql.NullString `db:"error" json:"error"` - DatabaseLatency int32 `db:"database_latency" json:"database_latency"` + ID uuid.UUID `db:"id" json:"id"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + StartedAt time.Time `db:"started_at" json:"started_at"` + StoppedAt sql.NullTime `db:"stopped_at" json:"stopped_at"` + RelayAddress string `db:"relay_address" json:"relay_address"` + RegionID int32 `db:"region_id" json:"region_id"` + Hostname string `db:"hostname" json:"hostname"` + Version string `db:"version" json:"version"` + Error string `db:"error" json:"error"` + DatabaseLatency int32 `db:"database_latency" json:"database_latency"` } func (q *sqlQuerier) UpdateReplica(ctx context.Context, arg UpdateReplicaParams) (Replica, error) { diff --git a/enterprise/coderd/replicas.go b/enterprise/coderd/replicas.go index c07c37243d0ca..906597f257f04 100644 --- a/enterprise/coderd/replicas.go +++ b/enterprise/coderd/replicas.go @@ -31,7 +31,7 @@ func convertReplica(replica database.Replica) codersdk.Replica { CreatedAt: replica.CreatedAt, RelayAddress: replica.RelayAddress, RegionID: replica.RegionID, - Error: replica.Error.String, + Error: replica.Error, DatabaseLatency: replica.DatabaseLatency, } } diff --git a/enterprise/replicasync/replicasync.go b/enterprise/replicasync/replicasync.go index d6cd846d6c96f..aa8eba46613ff 100644 --- a/enterprise/replicasync/replicasync.go +++ b/enterprise/replicasync/replicasync.go @@ -271,12 +271,9 @@ func (m *Manager) syncReplicas(ctx context.Context) error { }(peer) } wg.Wait() - replicaError := sql.NullString{} + replicaError := "" if len(failed) > 0 { - replicaError = sql.NullString{ - Valid: true, - String: fmt.Sprintf("Failed to dial peers: %s", strings.Join(failed, ", ")), - } + replicaError = fmt.Sprintf("Failed to dial peers: %s", strings.Join(failed, ", ")) } databaseLatency, err := m.db.Ping(ctx) @@ -301,7 +298,7 @@ func (m *Manager) syncReplicas(ctx context.Context) error { } m.mutex.Lock() defer m.mutex.Unlock() - if m.self.Error.String != replica.Error.String { + if m.self.Error != replica.Error { // Publish an update occurred! err = m.pubsub.Publish(PubsubEvent, []byte(m.self.ID.String())) if err != nil { diff --git a/enterprise/replicasync/replicasync_test.go b/enterprise/replicasync/replicasync_test.go index 79acf86865839..40e087a7616ce 100644 --- a/enterprise/replicasync/replicasync_test.go +++ b/enterprise/replicasync/replicasync_test.go @@ -84,7 +84,7 @@ func TestReplica(t *testing.T) { require.NoError(t, err) require.Len(t, server.Regional(), 1) require.Equal(t, peer.ID, server.Regional()[0].ID) - require.False(t, server.Self().Error.Valid) + require.Empty(t, server.Self().Error) _ = server.Close() }) t.Run("ConnectsToPeerReplicaTLS", func(t *testing.T) { @@ -125,7 +125,7 @@ func TestReplica(t *testing.T) { require.NoError(t, err) require.Len(t, server.Regional(), 1) require.Equal(t, peer.ID, server.Regional()[0].ID) - require.False(t, server.Self().Error.Valid) + require.Empty(t, server.Self().Error) _ = server.Close() }) t.Run("ConnectsToFakePeerWithError", func(t *testing.T) { @@ -148,8 +148,8 @@ func TestReplica(t *testing.T) { require.NoError(t, err) require.Len(t, server.Regional(), 1) require.Equal(t, peer.ID, server.Regional()[0].ID) - require.True(t, server.Self().Error.Valid) - require.Contains(t, server.Self().Error.String, "Failed to dial peers") + require.NotEmpty(t, server.Self().Error) + require.Contains(t, server.Self().Error, "Failed to dial peers") _ = server.Close() }) t.Run("RefreshOnPublish", func(t *testing.T) { From 1ff5f7d81cad45cf6c6346f6763df3dd8ad1bdb5 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 00:51:49 +0000 Subject: [PATCH 51/79] Move pubsub to in-memory for twenty test --- enterprise/replicasync/replicasync_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/enterprise/replicasync/replicasync_test.go b/enterprise/replicasync/replicasync_test.go index 40e087a7616ce..2e3d1deafc68c 100644 --- a/enterprise/replicasync/replicasync_test.go +++ b/enterprise/replicasync/replicasync_test.go @@ -16,6 +16,7 @@ import ( "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/coderd/database" + "github.com/coder/coder/coderd/database/databasefake" "github.com/coder/coder/coderd/database/dbtestutil" "github.com/coder/coder/enterprise/replicasync" "github.com/coder/coder/testutil" @@ -200,7 +201,8 @@ func TestReplica(t *testing.T) { // Ensures that twenty concurrent replicas can spawn and all // discover each other in parallel! t.Parallel() - db, pubsub := dbtestutil.NewDB(t) + db := databasefake.New() + pubsub := database.NewPubsubInMemory() logger := slogtest.Make(t, nil) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) From b732184e0a52d7c121d5b4a1d696c7721510d57f Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 00:54:19 +0000 Subject: [PATCH 52/79] Add comment for configuration tweaking --- enterprise/replicasync/replicasync_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/enterprise/replicasync/replicasync_test.go b/enterprise/replicasync/replicasync_test.go index 2e3d1deafc68c..b6eb45bb9d316 100644 --- a/enterprise/replicasync/replicasync_test.go +++ b/enterprise/replicasync/replicasync_test.go @@ -201,6 +201,9 @@ func TestReplica(t *testing.T) { // Ensures that twenty concurrent replicas can spawn and all // discover each other in parallel! t.Parallel() + // This doesn't use the database fake because creating + // this many PostgreSQL connections takes some + // configuration tweaking. db := databasefake.New() pubsub := database.NewPubsubInMemory() logger := slogtest.Make(t, nil) From 38465ac4f9306bac1669335c4155b2440838d361 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 01:12:32 +0000 Subject: [PATCH 53/79] Fix leak with transport --- coderd/wsconncache/wsconncache_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/coderd/wsconncache/wsconncache_test.go b/coderd/wsconncache/wsconncache_test.go index 003d3cddb8b7a..d4345ce9d5f05 100644 --- a/coderd/wsconncache/wsconncache_test.go +++ b/coderd/wsconncache/wsconncache_test.go @@ -128,7 +128,9 @@ func TestCache(t *testing.T) { return } defer release() - proxy.Transport = conn.HTTPTransport() + transport := conn.HTTPTransport() + defer transport.CloseIdleConnections() + proxy.Transport = transport res := httptest.NewRecorder() proxy.ServeHTTP(res, req) resp := res.Result() From 72555e2d8cd81cd7dd02498f13f59071d6ef1705 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 01:17:47 +0000 Subject: [PATCH 54/79] Fix close leak in derpmesh --- enterprise/derpmesh/derpmesh.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/enterprise/derpmesh/derpmesh.go b/enterprise/derpmesh/derpmesh.go index 5de7799aa74eb..3982542167073 100644 --- a/enterprise/derpmesh/derpmesh.go +++ b/enterprise/derpmesh/derpmesh.go @@ -93,6 +93,9 @@ func (m *Mesh) SetAddresses(addresses []string, connect bool) { func (m *Mesh) addAddress(address string, connect bool) (bool, error) { m.mutex.Lock() defer m.mutex.Unlock() + if m.isClosed() { + return false, nil + } _, isActive := m.active[address] if isActive { return false, nil @@ -142,10 +145,8 @@ func (m *Mesh) removeAddress(address string) bool { func (m *Mesh) Close() error { m.mutex.Lock() defer m.mutex.Unlock() - select { - case <-m.closed: + if m.isClosed() { return nil - default: } close(m.closed) for _, cancelFunc := range m.active { @@ -153,3 +154,12 @@ func (m *Mesh) Close() error { } return nil } + +func (m *Mesh) isClosed() bool { + select { + case <-m.closed: + return true + default: + } + return false +} From e54072a53a80d78c54ea82897301f8120d35ad15 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 01:30:09 +0000 Subject: [PATCH 55/79] Fix race when creating server --- coderd/coderdtest/coderdtest.go | 8 +++++--- enterprise/coderd/coderdenttest/coderdenttest.go | 5 +++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index de49afaa1c269..4a7c3e38b69e1 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -170,9 +170,6 @@ func NewOptions(t *testing.T, options *Options) (*httptest.Server, context.Cance Certificates: options.TLSCertificates, MinVersion: tls.VersionTLS12, } - srv.StartTLS() - } else { - srv.Start() } t.Cleanup(srv.Close) @@ -266,6 +263,11 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c // We set the handler after server creation for the access URL. coderAPI := coderd.New(newOptions) srv.Config.Handler = coderAPI.RootHandler + if newOptions.TLSCertificates != nil { + srv.StartTLS() + } else { + srv.Start() + } var provisionerCloser io.Closer = nopcloser{} if options.IncludeProvisionerDaemon { provisionerCloser = NewProvisionerDaemon(t, coderAPI) diff --git a/enterprise/coderd/coderdenttest/coderdenttest.go b/enterprise/coderd/coderdenttest/coderdenttest.go index 02eff4e2acf2e..fbffa683bea8b 100644 --- a/enterprise/coderd/coderdenttest/coderdenttest.go +++ b/enterprise/coderd/coderdenttest/coderdenttest.go @@ -77,6 +77,11 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c }) assert.NoError(t, err) srv.Config.Handler = coderAPI.AGPL.RootHandler + if oop.TLSCertificates != nil { + srv.StartTLS() + } else { + srv.Start() + } var provisionerCloser io.Closer = nopcloser{} if options.IncludeProvisionerDaemon { provisionerCloser = coderdtest.NewProvisionerDaemon(t, coderAPI.AGPL) From 27d5f40619e47cc5729dce114fba34b771daf612 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 02:48:52 +0000 Subject: [PATCH 56/79] Remove handler update --- coderd/coderdtest/coderdtest.go | 8 +++----- enterprise/coderd/coderdenttest/coderdenttest.go | 5 ----- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index 4a7c3e38b69e1..de49afaa1c269 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -170,6 +170,9 @@ func NewOptions(t *testing.T, options *Options) (*httptest.Server, context.Cance Certificates: options.TLSCertificates, MinVersion: tls.VersionTLS12, } + srv.StartTLS() + } else { + srv.Start() } t.Cleanup(srv.Close) @@ -263,11 +266,6 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c // We set the handler after server creation for the access URL. coderAPI := coderd.New(newOptions) srv.Config.Handler = coderAPI.RootHandler - if newOptions.TLSCertificates != nil { - srv.StartTLS() - } else { - srv.Start() - } var provisionerCloser io.Closer = nopcloser{} if options.IncludeProvisionerDaemon { provisionerCloser = NewProvisionerDaemon(t, coderAPI) diff --git a/enterprise/coderd/coderdenttest/coderdenttest.go b/enterprise/coderd/coderdenttest/coderdenttest.go index fbffa683bea8b..02eff4e2acf2e 100644 --- a/enterprise/coderd/coderdenttest/coderdenttest.go +++ b/enterprise/coderd/coderdenttest/coderdenttest.go @@ -77,11 +77,6 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c }) assert.NoError(t, err) srv.Config.Handler = coderAPI.AGPL.RootHandler - if oop.TLSCertificates != nil { - srv.StartTLS() - } else { - srv.Start() - } var provisionerCloser io.Closer = nopcloser{} if options.IncludeProvisionerDaemon { provisionerCloser = coderdtest.NewProvisionerDaemon(t, coderAPI.AGPL) From 4d0b1d86854a41a4616578c5655e00d65f2cfffa Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 03:23:53 +0000 Subject: [PATCH 57/79] Skip test on Windows --- enterprise/derpmesh/derpmesh_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/enterprise/derpmesh/derpmesh_test.go b/enterprise/derpmesh/derpmesh_test.go index 84875f106c7f2..1c1d658bb03c2 100644 --- a/enterprise/derpmesh/derpmesh_test.go +++ b/enterprise/derpmesh/derpmesh_test.go @@ -7,6 +7,7 @@ import ( "errors" "io" "net/http/httptest" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -101,6 +102,9 @@ func TestDERPMesh(t *testing.T) { }) t.Run("TwentyMeshes", func(t *testing.T) { t.Parallel() + if runtime.GOOS == "windows" { + t.Skip("This test is races on Windows... I think because it's too slow.") + } meshes := make([]*derpmesh.Mesh, 0, 20) serverURLs := make([]string, 0, 20) for i := 0; i < 20; i++ { From 129f5ba6511615e0a8920164ce56b90efb41e214 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 03:39:45 +0000 Subject: [PATCH 58/79] Fix DERP mesh test --- enterprise/derpmesh/derpmesh_test.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/enterprise/derpmesh/derpmesh_test.go b/enterprise/derpmesh/derpmesh_test.go index 1c1d658bb03c2..7ca844c57e6fc 100644 --- a/enterprise/derpmesh/derpmesh_test.go +++ b/enterprise/derpmesh/derpmesh_test.go @@ -7,8 +7,8 @@ import ( "errors" "io" "net/http/httptest" - "runtime" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -102,9 +102,6 @@ func TestDERPMesh(t *testing.T) { }) t.Run("TwentyMeshes", func(t *testing.T) { t.Parallel() - if runtime.GOOS == "windows" { - t.Skip("This test is races on Windows... I think because it's too slow.") - } meshes := make([]*derpmesh.Mesh, 0, 20) serverURLs := make([]string, 0, 20) for i := 0; i < 20; i++ { @@ -132,12 +129,28 @@ func TestDERPMesh(t *testing.T) { err = secondClient.Connect(context.Background()) require.NoError(t, err) + closed := make(chan struct{}) + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() sent := []byte("hello world") - err = firstClient.Send(second.Public(), sent) - require.NoError(t, err) + go func() { + defer close(closed) + ticker := time.NewTicker(time.Second) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + err = firstClient.Send(second.Public(), sent) + require.NoError(t, err) + } + }() got := recvData(t, secondClient) require.Equal(t, sent, got) + cancelFunc() + <-closed }) } From 4e5d30e6267ca70ea318ca2ffe84fea369725617 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 04:47:04 +0000 Subject: [PATCH 59/79] Wrap HTTP handler replacement in mutex --- coderd/coderdtest/coderdtest.go | 115 ++++++++++-------- .../coderd/coderdenttest/coderdenttest.go | 4 +- 2 files changed, 66 insertions(+), 53 deletions(-) diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index de49afaa1c269..31785ff7e7950 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -24,6 +24,7 @@ import ( "regexp" "strconv" "strings" + "sync" "testing" "time" @@ -127,7 +128,7 @@ func newWithCloser(t *testing.T, options *Options) (*codersdk.Client, io.Closer) return client, closer } -func NewOptions(t *testing.T, options *Options) (*httptest.Server, context.CancelFunc, *coderd.Options) { +func NewOptions(t *testing.T, options *Options) (func(http.Handler), context.CancelFunc, *coderd.Options) { if options == nil { options = &Options{} } @@ -161,7 +162,15 @@ func NewOptions(t *testing.T, options *Options) (*httptest.Server, context.Cance ).WithStatsChannel(options.AutobuildStats) lifecycleExecutor.Run() - srv := httptest.NewUnstartedServer(nil) + var mutex sync.RWMutex + var handler http.Handler + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mutex.RLock() + defer mutex.RUnlock() + if handler != nil { + handler.ServeHTTP(w, r) + } + })) srv.Config.BaseContext = func(_ net.Listener) context.Context { return ctx } @@ -204,55 +213,59 @@ func NewOptions(t *testing.T, options *Options) (*httptest.Server, context.Cance require.NoError(t, err) } - return srv, cancelFunc, &coderd.Options{ - AgentConnectionUpdateFrequency: 150 * time.Millisecond, - // Force a long disconnection timeout to ensure - // agents are not marked as disconnected during slow tests. - AgentInactiveDisconnectTimeout: testutil.WaitShort, - AccessURL: serverURL, - AppHostname: options.AppHostname, - AppHostnameRegex: appHostnameRegex, - Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug), - CacheDir: t.TempDir(), - Database: options.Database, - Pubsub: options.Pubsub, - - Auditor: options.Auditor, - AWSCertificates: options.AWSCertificates, - AzureCertificates: options.AzureCertificates, - GithubOAuth2Config: options.GithubOAuth2Config, - OIDCConfig: options.OIDCConfig, - GoogleTokenValidator: options.GoogleTokenValidator, - SSHKeygenAlgorithm: options.SSHKeygenAlgorithm, - DERPServer: derpServer, - APIRateLimit: options.APIRateLimit, - Authorizer: options.Authorizer, - Telemetry: telemetry.NewNoop(), - TLSCertificates: options.TLSCertificates, - DERPMap: &tailcfg.DERPMap{ - Regions: map[int]*tailcfg.DERPRegion{ - 1: { - EmbeddedRelay: true, - RegionID: 1, - RegionCode: "coder", - RegionName: "Coder", - Nodes: []*tailcfg.DERPNode{{ - Name: "1a", - RegionID: 1, - IPv4: "127.0.0.1", - DERPPort: derpPort, - STUNPort: stunAddr.Port, - InsecureForTests: true, - ForceHTTP: options.TLSCertificates == nil, - }}, + return func(h http.Handler) { + mutex.Lock() + handler = h + mutex.Unlock() + }, cancelFunc, &coderd.Options{ + AgentConnectionUpdateFrequency: 150 * time.Millisecond, + // Force a long disconnection timeout to ensure + // agents are not marked as disconnected during slow tests. + AgentInactiveDisconnectTimeout: testutil.WaitShort, + AccessURL: serverURL, + AppHostname: options.AppHostname, + AppHostnameRegex: appHostnameRegex, + Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug), + CacheDir: t.TempDir(), + Database: options.Database, + Pubsub: options.Pubsub, + + Auditor: options.Auditor, + AWSCertificates: options.AWSCertificates, + AzureCertificates: options.AzureCertificates, + GithubOAuth2Config: options.GithubOAuth2Config, + OIDCConfig: options.OIDCConfig, + GoogleTokenValidator: options.GoogleTokenValidator, + SSHKeygenAlgorithm: options.SSHKeygenAlgorithm, + DERPServer: derpServer, + APIRateLimit: options.APIRateLimit, + Authorizer: options.Authorizer, + Telemetry: telemetry.NewNoop(), + TLSCertificates: options.TLSCertificates, + DERPMap: &tailcfg.DERPMap{ + Regions: map[int]*tailcfg.DERPRegion{ + 1: { + EmbeddedRelay: true, + RegionID: 1, + RegionCode: "coder", + RegionName: "Coder", + Nodes: []*tailcfg.DERPNode{{ + Name: "1a", + RegionID: 1, + IPv4: "127.0.0.1", + DERPPort: derpPort, + STUNPort: stunAddr.Port, + InsecureForTests: true, + ForceHTTP: options.TLSCertificates == nil, + }}, + }, }, }, - }, - AutoImportTemplates: options.AutoImportTemplates, - MetricsCacheRefreshInterval: options.MetricsCacheRefreshInterval, - AgentStatsRefreshInterval: options.AgentStatsRefreshInterval, - DeploymentFlags: options.DeploymentFlags, - } + AutoImportTemplates: options.AutoImportTemplates, + MetricsCacheRefreshInterval: options.MetricsCacheRefreshInterval, + AgentStatsRefreshInterval: options.AgentStatsRefreshInterval, + DeploymentFlags: options.DeploymentFlags, + } } // NewWithAPI constructs an in-memory API instance and returns a client to talk to it. @@ -262,10 +275,10 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c if options == nil { options = &Options{} } - srv, cancelFunc, newOptions := NewOptions(t, options) + setHandler, cancelFunc, newOptions := NewOptions(t, options) // We set the handler after server creation for the access URL. coderAPI := coderd.New(newOptions) - srv.Config.Handler = coderAPI.RootHandler + setHandler(coderAPI.APIHandler) var provisionerCloser io.Closer = nopcloser{} if options.IncludeProvisionerDaemon { provisionerCloser = NewProvisionerDaemon(t, coderAPI) diff --git a/enterprise/coderd/coderdenttest/coderdenttest.go b/enterprise/coderd/coderdenttest/coderdenttest.go index 02eff4e2acf2e..a8595b5bc6ede 100644 --- a/enterprise/coderd/coderdenttest/coderdenttest.go +++ b/enterprise/coderd/coderdenttest/coderdenttest.go @@ -62,7 +62,7 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c if options.Options == nil { options.Options = &coderdtest.Options{} } - srv, cancelFunc, oop := coderdtest.NewOptions(t, options.Options) + setHandler, cancelFunc, oop := coderdtest.NewOptions(t, options.Options) coderAPI, err := coderd.New(context.Background(), &coderd.Options{ RBAC: true, AuditLogging: options.AuditLogging, @@ -76,7 +76,7 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c Keys: Keys, }) assert.NoError(t, err) - srv.Config.Handler = coderAPI.AGPL.RootHandler + setHandler(coderAPI.AGPL.RootHandler) var provisionerCloser io.Closer = nopcloser{} if options.IncludeProvisionerDaemon { provisionerCloser = coderdtest.NewProvisionerDaemon(t, coderAPI.AGPL) From 0359a7e9a79debe4f4e28fdde81faa27094e153e Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 14:44:24 +0000 Subject: [PATCH 60/79] Fix error message for relay --- enterprise/replicasync/replicasync.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/enterprise/replicasync/replicasync.go b/enterprise/replicasync/replicasync.go index aa8eba46613ff..4aeabd2a05742 100644 --- a/enterprise/replicasync/replicasync.go +++ b/enterprise/replicasync/replicasync.go @@ -178,7 +178,7 @@ func (m *Manager) subscribe(ctx context.Context) error { update = func() { err := m.syncReplicas(ctx) if err != nil && !errors.Is(err, context.Canceled) { - m.logger.Error(ctx, "run replica from subscribe", slog.Error(err)) + m.logger.Warn(ctx, "run replica from subscribe", slog.Error(err)) } updateMutex.Lock() if needsUpdate { @@ -256,7 +256,7 @@ func (m *Manager) syncReplicas(ctx context.Context) error { defer wg.Done() req, err := http.NewRequestWithContext(ctx, http.MethodGet, peer.RelayAddress, nil) if err != nil { - m.logger.Error(ctx, "create http request for relay probe", + m.logger.Warn(ctx, "create http request for relay probe", slog.F("relay_address", peer.RelayAddress), slog.Error(err)) return } From f364d1fe7523af406fb047828ba93fd6c2b139df Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 14:51:48 +0000 Subject: [PATCH 61/79] Fix API handler for normal tests --- coderd/coderdtest/coderdtest.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index 31785ff7e7950..2a7184e2ca05d 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -215,8 +215,8 @@ func NewOptions(t *testing.T, options *Options) (func(http.Handler), context.Can return func(h http.Handler) { mutex.Lock() + defer mutex.Unlock() handler = h - mutex.Unlock() }, cancelFunc, &coderd.Options{ AgentConnectionUpdateFrequency: 150 * time.Millisecond, // Force a long disconnection timeout to ensure @@ -278,7 +278,7 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c setHandler, cancelFunc, newOptions := NewOptions(t, options) // We set the handler after server creation for the access URL. coderAPI := coderd.New(newOptions) - setHandler(coderAPI.APIHandler) + setHandler(coderAPI.RootHandler) var provisionerCloser io.Closer = nopcloser{} if options.IncludeProvisionerDaemon { provisionerCloser = NewProvisionerDaemon(t, coderAPI) From 423a47e1dd9aa227a3a48ada5ac343a732bb9d01 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 15:25:34 +0000 Subject: [PATCH 62/79] Fix speedtest --- agent/agent_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/agent/agent_test.go b/agent/agent_test.go index e1269d6003922..e10eee7f111a0 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -483,9 +483,7 @@ func TestAgent(t *testing.T) { t.Run("Speedtest", func(t *testing.T) { t.Parallel() - if testing.Short() { - t.Skip("The minimum duration for a speedtest is hardcoded in Tailscale to 5s!") - } + t.Skip("This test is relatively flakey because of Tailscale's speedtest code...") derpMap := tailnettest.RunDERPAndSTUN(t) conn, _ := setupAgent(t, codersdk.WorkspaceAgentMetadata{ DERPMap: derpMap, From c3a77fe2d048b88d4e6e1f5d1ad8475735a18f18 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 16:21:13 +0000 Subject: [PATCH 63/79] Fix replica resend --- enterprise/derpmesh/derpmesh_test.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/enterprise/derpmesh/derpmesh_test.go b/enterprise/derpmesh/derpmesh_test.go index 7ca844c57e6fc..2878346d4ee43 100644 --- a/enterprise/derpmesh/derpmesh_test.go +++ b/enterprise/derpmesh/derpmesh_test.go @@ -94,11 +94,28 @@ func TestDERPMesh(t *testing.T) { secondClient.TLSConfig = tlsConfig err = secondClient.Connect(context.Background()) require.NoError(t, err) + + closed := make(chan struct{}) + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() sent := []byte("hello world") - err = firstClient.Send(second.Public(), sent) - require.NoError(t, err) + go func() { + defer close(closed) + ticker := time.NewTicker(50 * time.Millisecond) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + err = firstClient.Send(second.Public(), sent) + require.NoError(t, err) + } + }() got := recvData(t, secondClient) require.Equal(t, sent, got) + cancelFunc() + <-closed }) t.Run("TwentyMeshes", func(t *testing.T) { t.Parallel() @@ -135,7 +152,7 @@ func TestDERPMesh(t *testing.T) { sent := []byte("hello world") go func() { defer close(closed) - ticker := time.NewTicker(time.Second) + ticker := time.NewTicker(50 * time.Millisecond) for { select { case <-ctx.Done(): From 729f8a07acda2301d0fa60dc350b3838da956a04 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 16:34:38 +0000 Subject: [PATCH 64/79] Fix derpmesh send --- enterprise/derpmesh/derpmesh_test.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/enterprise/derpmesh/derpmesh_test.go b/enterprise/derpmesh/derpmesh_test.go index 2878346d4ee43..7fad141238442 100644 --- a/enterprise/derpmesh/derpmesh_test.go +++ b/enterprise/derpmesh/derpmesh_test.go @@ -67,12 +67,28 @@ func TestDERPMesh(t *testing.T) { err = secondClient.Connect(context.Background()) require.NoError(t, err) + closed := make(chan struct{}) + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() sent := []byte("hello world") - err = firstClient.Send(second.Public(), sent) - require.NoError(t, err) + go func() { + defer close(closed) + ticker := time.NewTicker(50 * time.Millisecond) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + err = firstClient.Send(second.Public(), sent) + require.NoError(t, err) + } + }() got := recvData(t, secondClient) require.Equal(t, sent, got) + cancelFunc() + <-closed }) t.Run("RemoveAddress", func(t *testing.T) { // This tests messages passing through multiple DERP servers. From ae0bc5df1e0538fc4e4a0155044bd37724dc2b23 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 21:17:12 +0000 Subject: [PATCH 65/79] Ping async --- codersdk/agentconn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codersdk/agentconn.go b/codersdk/agentconn.go index e75edf1ca6bb0..ddfb9541a186a 100644 --- a/codersdk/agentconn.go +++ b/codersdk/agentconn.go @@ -135,7 +135,7 @@ type AgentConn struct { func (c *AgentConn) Ping(ctx context.Context) (time.Duration, error) { errCh := make(chan error, 1) durCh := make(chan time.Duration, 1) - c.Conn.Ping(TailnetIP, tailcfg.PingDisco, func(pr *ipnstate.PingResult) { + go c.Conn.Ping(TailnetIP, tailcfg.PingDisco, func(pr *ipnstate.PingResult) { if pr.Err != "" { errCh <- xerrors.New(pr.Err) return From d7d50db6dd6cec7adefbf4153cc075a3588c4be3 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 21:23:26 +0000 Subject: [PATCH 66/79] Increase wait time of template version jobd --- coderd/coderdtest/coderdtest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index 2a7184e2ca05d..5cf307d842e90 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -497,7 +497,7 @@ func AwaitTemplateVersionJob(t *testing.T, client *codersdk.Client, version uuid var err error templateVersion, err = client.TemplateVersion(context.Background(), version) return assert.NoError(t, err) && templateVersion.Job.CompletedAt != nil - }, testutil.WaitShort, testutil.IntervalFast) + }, testutil.WaitMedium, testutil.IntervalFast) return templateVersion } From 77d23dc113a1fef2670ec9a898a94c9d6eb0ec18 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 22:24:18 +0000 Subject: [PATCH 67/79] Fix race when closing replica sync --- enterprise/replicasync/replicasync.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/enterprise/replicasync/replicasync.go b/enterprise/replicasync/replicasync.go index 4aeabd2a05742..0534c55246824 100644 --- a/enterprise/replicasync/replicasync.go +++ b/enterprise/replicasync/replicasync.go @@ -362,7 +362,8 @@ func (m *Manager) Close() error { m.closeCancel() m.closeWait.Wait() m.closeMutex.Unlock() - + m.mutex.Lock() + defer m.mutex.Unlock() ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Second) defer cancelFunc() _, err := m.db.UpdateReplica(ctx, database.UpdateReplicaParams{ From 435bbbb364a47bffeada924e45772059a4250022 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 16 Oct 2022 22:57:10 +0000 Subject: [PATCH 68/79] Add name to client --- enterprise/coderd/replicas_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enterprise/coderd/replicas_test.go b/enterprise/coderd/replicas_test.go index fae418ab87261..7a3e130cf7770 100644 --- a/enterprise/coderd/replicas_test.go +++ b/enterprise/coderd/replicas_test.go @@ -118,7 +118,7 @@ func TestReplicas(t *testing.T) { _, agent := setupWorkspaceAgent(t, firstClient, firstUser, 0) conn, err := secondClient.DialWorkspaceAgent(context.Background(), agent.ID, &codersdk.DialWorkspaceAgentOptions{ BlockEndpoints: true, - Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug), + Logger: slogtest.Make(t, nil).Named("client").Leveled(slog.LevelDebug), }) require.NoError(t, err) require.Eventually(t, func() bool { From 9b7c41afd462daf8f42d57d865b07e33d744f141 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 17 Oct 2022 00:27:24 +0000 Subject: [PATCH 69/79] Log the derpmap being used --- agent/agent.go | 1 + 1 file changed, 1 insertion(+) diff --git a/agent/agent.go b/agent/agent.go index 6d0a9a952f44b..f7c5598b7b710 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -170,6 +170,7 @@ func (a *agent) runTailnet(ctx context.Context, derpMap *tailcfg.DERPMap) { if a.isClosed() { return } + a.logger.Debug(ctx, "running tailnet with derpmap", slog.F("derpmap", derpMap)) if a.network != nil { a.network.SetDERPMap(derpMap) return From 961540291ae4137848850dfd2c04bf78a3bd1079 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 17 Oct 2022 00:30:43 +0000 Subject: [PATCH 70/79] Don't connect if DERP is empty --- tailnet/conn.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tailnet/conn.go b/tailnet/conn.go index e41ed60a527f3..7c572a55e1b66 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -344,9 +344,14 @@ func (c *Conn) UpdateNodes(nodes []*Node) error { // reason. TODO: @kylecarbs debug this! KeepAlive: ok && peerStatus.Active, } + // If no preferred DERP is provided, don't set an IP! + if node.PreferredDERP == 0 { + peerNode.DERP = "" + } if c.blockEndpoints { peerNode.Endpoints = nil } + c.logger.Debug(context.Background(), "adding node", slog.F("node", peerNode)) c.peerMap[node.ID] = peerNode } c.netMap.Peers = make([]*tailcfg.Node, 0, len(c.peerMap)) From bcb97ac7087b105b78c087ac89203024e51192f5 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 17 Oct 2022 01:30:29 +0000 Subject: [PATCH 71/79] Improve agent coordinator logging --- coderd/workspaceagents.go | 2 ++ tailnet/conn.go | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index b93369b0cb9cb..fb7f765cc7519 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -378,6 +378,7 @@ func (api *API) dialWorkspaceAgentTailnet(r *http.Request, agentID uuid.UUID) (* go func() { err := (*api.TailnetCoordinator.Load()).ServeClient(serverConn, uuid.New(), agentID) if err != nil { + api.Logger.Warn(r.Context(), "tailnet coordinator client error", slog.Error(err)) _ = conn.Close() } }() @@ -516,6 +517,7 @@ func (api *API) workspaceAgentCoordinate(rw http.ResponseWriter, r *http.Request defer close(closeChan) err := (*api.TailnetCoordinator.Load()).ServeAgent(wsNetConn, workspaceAgent.ID) if err != nil { + api.Logger.Warn(ctx, "tailnet coordinator agent error", slog.Error(err)) _ = conn.Close(websocket.StatusInternalError, err.Error()) return } diff --git a/tailnet/conn.go b/tailnet/conn.go index 7c572a55e1b66..2f2549718880d 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -328,6 +328,8 @@ func (c *Conn) UpdateNodes(nodes []*Node) error { delete(c.peerMap, peer.ID) } for _, node := range nodes { + c.logger.Debug(context.Background(), "adding node", slog.F("node", node)) + peerStatus, ok := status.Peer[node.Key] peerNode := &tailcfg.Node{ ID: node.ID, @@ -351,7 +353,6 @@ func (c *Conn) UpdateNodes(nodes []*Node) error { if c.blockEndpoints { peerNode.Endpoints = nil } - c.logger.Debug(context.Background(), "adding node", slog.F("node", peerNode)) c.peerMap[node.ID] = peerNode } c.netMap.Peers = make([]*tailcfg.Node, 0, len(c.peerMap)) From e2f6a1939f819feb3a1785c31e90a44e239b6f35 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 17 Oct 2022 01:35:07 +0000 Subject: [PATCH 72/79] Fix lock in coordinator --- enterprise/tailnet/coordinator.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/enterprise/tailnet/coordinator.go b/enterprise/tailnet/coordinator.go index 1ccf56f50da11..4bfae463e202a 100644 --- a/enterprise/tailnet/coordinator.go +++ b/enterprise/tailnet/coordinator.go @@ -178,12 +178,10 @@ func (c *haCoordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { if len(nodes) > 0 { data, err := json.Marshal(nodes) if err != nil { - c.mutex.Unlock() return xerrors.Errorf("marshal json: %w", err) } _, err = conn.Write(data) if err != nil { - c.mutex.Unlock() return xerrors.Errorf("write nodes: %w", err) } } @@ -250,17 +248,16 @@ func (c *haCoordinator) hangleAgentUpdate(id uuid.UUID, decoder *json.Decoder) ( } c.mutex.Lock() - defer c.mutex.Unlock() - c.nodes[id] = &node - connectionSockets, ok := c.agentToConnectionSockets[id] if !ok { + c.mutex.Unlock() return &node, nil } data, err := json.Marshal([]*agpl.Node{&node}) if err != nil { + c.mutex.Unlock() return nil, xerrors.Errorf("marshal nodes: %w", err) } @@ -275,6 +272,7 @@ func (c *haCoordinator) hangleAgentUpdate(id uuid.UUID, decoder *json.Decoder) ( _, _ = connectionSocket.Write(data) }() } + c.mutex.Unlock() wg.Wait() return &node, nil } @@ -394,12 +392,12 @@ func (c *haCoordinator) runPubsub() error { } c.mutex.Lock() - defer c.mutex.Unlock() - agentSocket, ok := c.agentSockets[agentUUID] if !ok { + c.mutex.Unlock() return } + c.mutex.Unlock() // We get a single node over pubsub, so turn into an array. _, err = agentSocket.Write(nodeJSON) @@ -410,7 +408,6 @@ func (c *haCoordinator) runPubsub() error { c.log.Error(ctx, "send callmemaybe to agent", slog.Error(err)) return } - case "agenthello": agentUUID, err := uuid.ParseBytes(agentID) if err != nil { @@ -426,7 +423,6 @@ func (c *haCoordinator) runPubsub() error { return } } - case "agentupdate": agentUUID, err := uuid.ParseBytes(agentID) if err != nil { @@ -440,7 +436,6 @@ func (c *haCoordinator) runPubsub() error { c.log.Error(ctx, "handle agent update", slog.Error(err)) return } - default: c.log.Error(ctx, "unknown peer event", slog.F("name", string(eventType))) } From c855c9ba60d511153ea9d1f30f57f6e8ccca626b Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 17 Oct 2022 01:53:50 +0000 Subject: [PATCH 73/79] Fix relay addr --- enterprise/replicasync/replicasync_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/enterprise/replicasync/replicasync_test.go b/enterprise/replicasync/replicasync_test.go index b6eb45bb9d316..b7709c1f6f814 100644 --- a/enterprise/replicasync/replicasync_test.go +++ b/enterprise/replicasync/replicasync_test.go @@ -134,17 +134,17 @@ func TestReplica(t *testing.T) { db, pubsub := dbtestutil.NewDB(t) peer, err := db.InsertReplica(context.Background(), database.InsertReplicaParams{ ID: uuid.New(), - CreatedAt: database.Now(), - StartedAt: database.Now(), - UpdatedAt: database.Now(), + CreatedAt: database.Now().Add(time.Minute), + StartedAt: database.Now().Add(time.Minute), + UpdatedAt: database.Now().Add(time.Minute), Hostname: "something", - // Fake address to hit! - RelayAddress: "http://169.254.169.254", + // Fake address to dial! + RelayAddress: "http://127.0.0.1:1", }) require.NoError(t, err) server, err := replicasync.New(context.Background(), slogtest.Make(t, nil), db, pubsub, &replicasync.Options{ PeerTimeout: 1 * time.Millisecond, - RelayAddress: "http://169.254.169.254", + RelayAddress: "http://127.0.0.1:1", }) require.NoError(t, err) require.Len(t, server.Regional(), 1) From a0e5cab653e6000e149404838fa90e8a27813ae6 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 17 Oct 2022 02:04:41 +0000 Subject: [PATCH 74/79] Fix race when updating durations --- coderd/activitybump_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderd/activitybump_test.go b/coderd/activitybump_test.go index dec5ec42f6556..e498b98fa0c80 100644 --- a/coderd/activitybump_test.go +++ b/coderd/activitybump_test.go @@ -72,7 +72,7 @@ func TestWorkspaceActivityBump(t *testing.T) { "deadline %v never updated", firstDeadline, ) - require.WithinDuration(t, database.Now().Add(time.Hour), workspace.LatestBuild.Deadline.Time, time.Second) + require.WithinDuration(t, database.Now().Add(time.Hour), workspace.LatestBuild.Deadline.Time, 3*time.Second) } } From 9878fc51d4b6a464e87bcb5f59f6891e1e39d5d9 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 17 Oct 2022 02:37:28 +0000 Subject: [PATCH 75/79] Fix client publish race --- enterprise/tailnet/coordinator.go | 60 +++++++++++++++++++++++--- enterprise/tailnet/coordinator_test.go | 11 ++--- 2 files changed, 61 insertions(+), 10 deletions(-) diff --git a/enterprise/tailnet/coordinator.go b/enterprise/tailnet/coordinator.go index 4bfae463e202a..2f284cb00ff61 100644 --- a/enterprise/tailnet/coordinator.go +++ b/enterprise/tailnet/coordinator.go @@ -56,8 +56,8 @@ type haCoordinator struct { // Node returns an in-memory node by ID. func (c *haCoordinator) Node(id uuid.UUID) *agpl.Node { - c.mutex.RLock() - defer c.mutex.RUnlock() + c.mutex.Lock() + defer c.mutex.Unlock() node := c.nodes[id] return node } @@ -79,6 +79,11 @@ func (c *haCoordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID if err != nil { return xerrors.Errorf("write nodes: %w", err) } + } else { + err := c.publishClientHello(agent) + if err != nil { + return xerrors.Errorf("publish client hello: %w", err) + } } c.mutex.Lock() @@ -205,7 +210,7 @@ func (c *haCoordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { decoder := json.NewDecoder(conn) for { - node, err := c.hangleAgentUpdate(id, decoder) + node, err := c.handleAgentUpdate(id, decoder) if err != nil { if errors.Is(err, io.EOF) { return nil @@ -240,7 +245,17 @@ func (c *haCoordinator) nodesSubscribedToAgent(agentID uuid.UUID) []*agpl.Node { return nodes } -func (c *haCoordinator) hangleAgentUpdate(id uuid.UUID, decoder *json.Decoder) (*agpl.Node, error) { +func (c *haCoordinator) handleClientHello(id uuid.UUID) error { + c.mutex.Lock() + node, ok := c.nodes[id] + c.mutex.Unlock() + if !ok { + return nil + } + return c.publishAgentToNodes(id, node) +} + +func (c *haCoordinator) handleAgentUpdate(id uuid.UUID, decoder *json.Decoder) (*agpl.Node, error) { var node agpl.Node err := decoder.Decode(&node) if err != nil { @@ -343,6 +358,18 @@ func (c *haCoordinator) publishAgentHello(id uuid.UUID) error { return nil } +func (c *haCoordinator) publishClientHello(id uuid.UUID) error { + msg, err := c.formatClientHello(id) + if err != nil { + return xerrors.Errorf("format client hello: %w", err) + } + err = c.pubsub.Publish("wireguard_peers", msg) + if err != nil { + return xerrors.Errorf("publish client hello: %w", err) + } + return nil +} + func (c *haCoordinator) publishAgentToNodes(id uuid.UUID, node *agpl.Node) error { msg, err := c.formatAgentUpdate(id, node) if err != nil { @@ -408,6 +435,18 @@ func (c *haCoordinator) runPubsub() error { c.log.Error(ctx, "send callmemaybe to agent", slog.Error(err)) return } + case "clienthello": + agentUUID, err := uuid.ParseBytes(agentID) + if err != nil { + c.log.Error(ctx, "invalid agent id", slog.F("id", string(agentID))) + return + } + + err = c.handleClientHello(agentUUID) + if err != nil { + c.log.Error(ctx, "handle agent request node", slog.Error(err)) + return + } case "agenthello": agentUUID, err := uuid.ParseBytes(agentID) if err != nil { @@ -431,7 +470,7 @@ func (c *haCoordinator) runPubsub() error { } decoder := json.NewDecoder(bytes.NewReader(nodeJSON)) - _, err = c.hangleAgentUpdate(agentUUID, decoder) + _, err = c.handleAgentUpdate(agentUUID, decoder) if err != nil { c.log.Error(ctx, "handle agent update", slog.Error(err)) return @@ -478,6 +517,17 @@ func (c *haCoordinator) formatAgentHello(id uuid.UUID) ([]byte, error) { return buf.Bytes(), nil } +// format: |clienthello|| +func (c *haCoordinator) formatClientHello(id uuid.UUID) ([]byte, error) { + buf := bytes.Buffer{} + + buf.WriteString(c.id.String() + "|") + buf.WriteString("clienthello|") + buf.WriteString(id.String() + "|") + + return buf.Bytes(), nil +} + // format: |agentupdate|| func (c *haCoordinator) formatAgentUpdate(id uuid.UUID, node *agpl.Node) ([]byte, error) { buf := bytes.Buffer{} diff --git a/enterprise/tailnet/coordinator_test.go b/enterprise/tailnet/coordinator_test.go index 83fac250b2916..86cee94dbdf5b 100644 --- a/enterprise/tailnet/coordinator_test.go +++ b/enterprise/tailnet/coordinator_test.go @@ -11,6 +11,7 @@ import ( "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/coderd/database" + "github.com/coder/coder/coderd/database/dbtestutil" "github.com/coder/coder/enterprise/tailnet" agpl "github.com/coder/coder/tailnet" "github.com/coder/coder/testutil" @@ -167,16 +168,12 @@ func TestCoordinatorHA(t *testing.T) { t.Run("AgentWithClient", func(t *testing.T) { t.Parallel() - pubsub := database.NewPubsubInMemory() + _, pubsub := dbtestutil.NewDB(t) coordinator1, err := tailnet.NewCoordinator(slogtest.Make(t, nil), pubsub) require.NoError(t, err) defer coordinator1.Close() - coordinator2, err := tailnet.NewCoordinator(slogtest.Make(t, nil), pubsub) - require.NoError(t, err) - defer coordinator2.Close() - agentWS, agentServerWS := net.Pipe() defer agentWS.Close() agentNodeChan := make(chan []*agpl.Node) @@ -196,6 +193,10 @@ func TestCoordinatorHA(t *testing.T) { return coordinator1.Node(agentID) != nil }, testutil.WaitShort, testutil.IntervalFast) + coordinator2, err := tailnet.NewCoordinator(slogtest.Make(t, nil), pubsub) + require.NoError(t, err) + defer coordinator2.Close() + clientWS, clientServerWS := net.Pipe() defer clientWS.Close() defer clientServerWS.Close() From 7a40bf801f89fc09d35fa1bc9716504f09595f2c Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 17 Oct 2022 02:52:02 +0000 Subject: [PATCH 76/79] Run pubsub loop in a queue --- enterprise/tailnet/coordinator.go | 202 +++++++++++++++++------------- 1 file changed, 113 insertions(+), 89 deletions(-) diff --git a/enterprise/tailnet/coordinator.go b/enterprise/tailnet/coordinator.go index 2f284cb00ff61..f001d4a9643dd 100644 --- a/enterprise/tailnet/coordinator.go +++ b/enterprise/tailnet/coordinator.go @@ -21,17 +21,19 @@ import ( // NewCoordinator creates a new high availability coordinator // that uses PostgreSQL pubsub to exchange handshakes. func NewCoordinator(logger slog.Logger, pubsub database.Pubsub) (agpl.Coordinator, error) { + ctx, cancelFunc := context.WithCancel(context.Background()) coord := &haCoordinator{ id: uuid.New(), log: logger, pubsub: pubsub, + closeFunc: cancelFunc, close: make(chan struct{}), nodes: map[uuid.UUID]*agpl.Node{}, agentSockets: map[uuid.UUID]net.Conn{}, agentToConnectionSockets: map[uuid.UUID]map[uuid.UUID]net.Conn{}, } - if err := coord.runPubsub(); err != nil { + if err := coord.runPubsub(ctx); err != nil { return nil, xerrors.Errorf("run coordinator pubsub: %w", err) } @@ -39,11 +41,12 @@ func NewCoordinator(logger slog.Logger, pubsub database.Pubsub) (agpl.Coordinato } type haCoordinator struct { - id uuid.UUID - log slog.Logger - mutex sync.RWMutex - pubsub database.Pubsub - close chan struct{} + id uuid.UUID + log slog.Logger + mutex sync.RWMutex + pubsub database.Pubsub + close chan struct{} + closeFunc context.CancelFunc // nodes maps agent and connection IDs their respective node. nodes map[uuid.UUID]*agpl.Node @@ -303,6 +306,7 @@ func (c *haCoordinator) Close() error { default: } close(c.close) + c.closeFunc() wg := sync.WaitGroup{} @@ -384,111 +388,131 @@ func (c *haCoordinator) publishAgentToNodes(id uuid.UUID, node *agpl.Node) error return nil } -func (c *haCoordinator) runPubsub() error { +func (c *haCoordinator) runPubsub(ctx context.Context) error { + messageQueue := make(chan []byte, 64) cancelSub, err := c.pubsub.Subscribe("wireguard_peers", func(ctx context.Context, message []byte) { - sp := bytes.Split(message, []byte("|")) - if len(sp) != 4 { - c.log.Error(ctx, "invalid wireguard peer message", slog.F("msg", string(message))) + select { + case messageQueue <- message: + case <-ctx.Done(): return } + }) + if err != nil { + return xerrors.Errorf("subscribe wireguard peers") + } + go func() { + for { + var message []byte + select { + case <-ctx.Done(): + return + case message = <-messageQueue: + } + c.handlePubsubMessage(ctx, message) + } + }() + + go func() { + defer cancelSub() + <-c.close + }() + + return nil +} + +func (c *haCoordinator) handlePubsubMessage(ctx context.Context, message []byte) { + sp := bytes.Split(message, []byte("|")) + if len(sp) != 4 { + c.log.Error(ctx, "invalid wireguard peer message", slog.F("msg", string(message))) + return + } + + var ( + coordinatorID = sp[0] + eventType = sp[1] + agentID = sp[2] + nodeJSON = sp[3] + ) - var ( - coordinatorID = sp[0] - eventType = sp[1] - agentID = sp[2] - nodeJSON = sp[3] - ) + sender, err := uuid.ParseBytes(coordinatorID) + if err != nil { + c.log.Error(ctx, "invalid sender id", slog.F("id", string(coordinatorID)), slog.F("msg", string(message))) + return + } - sender, err := uuid.ParseBytes(coordinatorID) + // We sent this message! + if sender == c.id { + return + } + + switch string(eventType) { + case "callmemaybe": + agentUUID, err := uuid.ParseBytes(agentID) if err != nil { - c.log.Error(ctx, "invalid sender id", slog.F("id", string(coordinatorID)), slog.F("msg", string(message))) + c.log.Error(ctx, "invalid agent id", slog.F("id", string(agentID))) return } - // We sent this message! - if sender == c.id { + c.mutex.Lock() + agentSocket, ok := c.agentSockets[agentUUID] + if !ok { + c.mutex.Unlock() return } + c.mutex.Unlock() - switch string(eventType) { - case "callmemaybe": - agentUUID, err := uuid.ParseBytes(agentID) - if err != nil { - c.log.Error(ctx, "invalid agent id", slog.F("id", string(agentID))) - return - } - - c.mutex.Lock() - agentSocket, ok := c.agentSockets[agentUUID] - if !ok { - c.mutex.Unlock() - return - } - c.mutex.Unlock() - - // We get a single node over pubsub, so turn into an array. - _, err = agentSocket.Write(nodeJSON) - if err != nil { - if errors.Is(err, io.EOF) { - return - } - c.log.Error(ctx, "send callmemaybe to agent", slog.Error(err)) - return - } - case "clienthello": - agentUUID, err := uuid.ParseBytes(agentID) - if err != nil { - c.log.Error(ctx, "invalid agent id", slog.F("id", string(agentID))) + // We get a single node over pubsub, so turn into an array. + _, err = agentSocket.Write(nodeJSON) + if err != nil { + if errors.Is(err, io.EOF) { return } + c.log.Error(ctx, "send callmemaybe to agent", slog.Error(err)) + return + } + case "clienthello": + agentUUID, err := uuid.ParseBytes(agentID) + if err != nil { + c.log.Error(ctx, "invalid agent id", slog.F("id", string(agentID))) + return + } - err = c.handleClientHello(agentUUID) - if err != nil { - c.log.Error(ctx, "handle agent request node", slog.Error(err)) - return - } - case "agenthello": - agentUUID, err := uuid.ParseBytes(agentID) - if err != nil { - c.log.Error(ctx, "invalid agent id", slog.F("id", string(agentID))) - return - } + err = c.handleClientHello(agentUUID) + if err != nil { + c.log.Error(ctx, "handle agent request node", slog.Error(err)) + return + } + case "agenthello": + agentUUID, err := uuid.ParseBytes(agentID) + if err != nil { + c.log.Error(ctx, "invalid agent id", slog.F("id", string(agentID))) + return + } - nodes := c.nodesSubscribedToAgent(agentUUID) - if len(nodes) > 0 { - err := c.publishNodesToAgent(agentUUID, nodes) - if err != nil { - c.log.Error(ctx, "publish nodes to agent", slog.Error(err)) - return - } - } - case "agentupdate": - agentUUID, err := uuid.ParseBytes(agentID) + nodes := c.nodesSubscribedToAgent(agentUUID) + if len(nodes) > 0 { + err := c.publishNodesToAgent(agentUUID, nodes) if err != nil { - c.log.Error(ctx, "invalid agent id", slog.F("id", string(agentID))) + c.log.Error(ctx, "publish nodes to agent", slog.Error(err)) return } + } + case "agentupdate": + agentUUID, err := uuid.ParseBytes(agentID) + if err != nil { + c.log.Error(ctx, "invalid agent id", slog.F("id", string(agentID))) + return + } - decoder := json.NewDecoder(bytes.NewReader(nodeJSON)) - _, err = c.handleAgentUpdate(agentUUID, decoder) - if err != nil { - c.log.Error(ctx, "handle agent update", slog.Error(err)) - return - } - default: - c.log.Error(ctx, "unknown peer event", slog.F("name", string(eventType))) + decoder := json.NewDecoder(bytes.NewReader(nodeJSON)) + _, err = c.handleAgentUpdate(agentUUID, decoder) + if err != nil { + c.log.Error(ctx, "handle agent update", slog.Error(err)) + return } - }) - if err != nil { - return xerrors.Errorf("subscribe wireguard peers") + default: + c.log.Error(ctx, "unknown peer event", slog.F("name", string(eventType))) } - - go func() { - defer cancelSub() - <-c.close - }() - - return nil } // format: |callmemaybe|| From 08b9681baac814f970b455432d845a6028a80779 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 17 Oct 2022 03:03:16 +0000 Subject: [PATCH 77/79] Store agent nodes in order --- enterprise/tailnet/coordinator.go | 7 +++++++ tailnet/conn.go | 1 + tailnet/coordinator.go | 2 ++ 3 files changed, 10 insertions(+) diff --git a/enterprise/tailnet/coordinator.go b/enterprise/tailnet/coordinator.go index f001d4a9643dd..da3845f70b4c3 100644 --- a/enterprise/tailnet/coordinator.go +++ b/enterprise/tailnet/coordinator.go @@ -266,6 +266,13 @@ func (c *haCoordinator) handleAgentUpdate(id uuid.UUID, decoder *json.Decoder) ( } c.mutex.Lock() + oldNode := c.nodes[id] + if oldNode != nil { + if oldNode.AsOf.After(node.AsOf) { + c.mutex.Unlock() + return oldNode, nil + } + } c.nodes[id] = &node connectionSockets, ok := c.agentToConnectionSockets[id] if !ok { diff --git a/tailnet/conn.go b/tailnet/conn.go index 2f2549718880d..e3af3786ec92f 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -435,6 +435,7 @@ func (c *Conn) sendNode() { } node := &Node{ ID: c.netMap.SelfNode.ID, + AsOf: c.lastStatus, Key: c.netMap.SelfNode.Key, Addresses: c.netMap.SelfNode.Addresses, AllowedIPs: c.netMap.SelfNode.AllowedIPs, diff --git a/tailnet/coordinator.go b/tailnet/coordinator.go index 491c0db885224..52c1fa1e66ec4 100644 --- a/tailnet/coordinator.go +++ b/tailnet/coordinator.go @@ -37,6 +37,8 @@ type Coordinator interface { type Node struct { // ID is used to identify the connection. ID tailcfg.NodeID `json:"id"` + // AsOf is the time the node was created. + AsOf time.Time `json:"as_of"` // Key is the Wireguard public key of the node. Key key.NodePublic `json:"key"` // DiscoKey is used for discovery messages over DERP to establish peer-to-peer connections. From 79991a939139c64d0b66582d32aeb7078befb43c Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 17 Oct 2022 03:18:19 +0000 Subject: [PATCH 78/79] Fix coordinator locking --- tailnet/coordinator.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tailnet/coordinator.go b/tailnet/coordinator.go index 52c1fa1e66ec4..4216bbc624d48 100644 --- a/tailnet/coordinator.go +++ b/tailnet/coordinator.go @@ -246,7 +246,6 @@ func (c *coordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { } sockets, ok := c.agentToConnectionSockets[id] - c.mutex.Unlock() if ok { // Publish all nodes that want to connect to the // desired agent ID. @@ -258,21 +257,21 @@ func (c *coordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { } nodes = append(nodes, node) } + c.mutex.Unlock() data, err := json.Marshal(nodes) if err != nil { - c.mutex.Unlock() return xerrors.Errorf("marshal json: %w", err) } _, err = conn.Write(data) if err != nil { return xerrors.Errorf("write nodes: %w", err) } + c.mutex.Lock() } // If an old agent socket is connected, we close it // to avoid any leaks. This shouldn't ever occur because // we expect one agent to be running. - c.mutex.Lock() oldAgentSocket, ok := c.agentSockets[id] if ok { _ = oldAgentSocket.Close() From 020171b65e4a19faebc97f2983cf740c8d53755f Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 17 Oct 2022 04:06:01 +0000 Subject: [PATCH 79/79] Check for closed pipe --- enterprise/tailnet/coordinator.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/enterprise/tailnet/coordinator.go b/enterprise/tailnet/coordinator.go index da3845f70b4c3..5749d9ef47c7a 100644 --- a/enterprise/tailnet/coordinator.go +++ b/enterprise/tailnet/coordinator.go @@ -121,7 +121,7 @@ func (c *haCoordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID for { err := c.handleNextClientMessage(id, agent, decoder) if err != nil { - if errors.Is(err, io.EOF) { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrClosedPipe) { return nil } return xerrors.Errorf("handle next client message: %w", err) @@ -163,7 +163,7 @@ func (c *haCoordinator) handleNextClientMessage(id, agent uuid.UUID, decoder *js _, err = agentSocket.Write(data) if err != nil { - if errors.Is(err, io.EOF) { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrClosedPipe) { return nil } return xerrors.Errorf("write json: %w", err) @@ -215,7 +215,7 @@ func (c *haCoordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { for { node, err := c.handleAgentUpdate(id, decoder) if err != nil { - if errors.Is(err, io.EOF) { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrClosedPipe) { return nil } return xerrors.Errorf("handle next agent message: %w", err) @@ -471,7 +471,7 @@ func (c *haCoordinator) handlePubsubMessage(ctx context.Context, message []byte) // We get a single node over pubsub, so turn into an array. _, err = agentSocket.Write(nodeJSON) if err != nil { - if errors.Is(err, io.EOF) { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrClosedPipe) { return } c.log.Error(ctx, "send callmemaybe to agent", slog.Error(err))