coder · dannykopping · Mar 12, 2025 · Mar 12, 2025 · Mar 12, 2025 · Mar 12, 2025
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -34,7 +34,7 @@ env:
 jobs:
   # build-dylib is a separate job to build the dylib on macOS.
   build-dylib:
-    runs-on: ${{ github.repository_owner == 'coder' && 'depot-macos-latest' || 'macos-latest' }}
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-macos-latest' || 'macos-latest'  }}
     steps:
       # Harden Runner doesn't work on macOS.
       - name: Checkout
@@ -279,9 +279,9 @@ jobs:
         env:
           EV_SIGNING_CERT: ${{ secrets.EV_SIGNING_CERT }}
 
-      - name: Test migrations from current ref to main
-        run: |
-          POSTGRES_VERSION=13 make test-migrations
+#      - name: Test migrations from current ref to main
+#        run: |
+#          POSTGRES_VERSION=13 make test-migrations
 
       # Setup GCloud for signing Windows binaries.
       - name: Authenticate to Google Cloud

diff --git a/agent/agent.go b/agent/agent.go
@@ -36,6 +36,9 @@ import (
 	"tailscale.com/util/clientmetric"
 
 	"cdr.dev/slog"
+
+	"github.com/coder/retry"
+
 	"github.com/coder/clistat"
 	"github.com/coder/coder/v2/agent/agentcontainers"
 	"github.com/coder/coder/v2/agent/agentexec"
@@ -53,7 +56,6 @@ import (
 	"github.com/coder/coder/v2/tailnet"
 	tailnetproto "github.com/coder/coder/v2/tailnet/proto"
 	"github.com/coder/quartz"
-	"github.com/coder/retry"
 )
 
 const (
@@ -365,9 +367,11 @@ func (a *agent) runLoop() {
 		if ctx.Err() != nil {
 			// Context canceled errors may come from websocket pings, so we
 			// don't want to use `errors.Is(err, context.Canceled)` here.
+			a.logger.Warn(ctx, "runLoop exited with error", slog.Error(ctx.Err()))
 			return
 		}
 		if a.isClosed() {
+			a.logger.Warn(ctx, "runLoop exited because agent is closed")
 			return
 		}
 		if errors.Is(err, io.EOF) {
@@ -1048,7 +1052,11 @@ func (a *agent) run() (retErr error) {
 		return a.statsReporter.reportLoop(ctx, aAPI)
 	})
 
-	return connMan.wait()
+	err = connMan.wait()
+	if err != nil {
+		a.logger.Warn(context.Background(), "connection manager errored", slog.Error(err))
+	}
+	return err
 }
 
 // handleManifest returns a function that fetches and processes the manifest
@@ -2005,7 +2013,7 @@ func (a *apiConnRoutineManager) startAgentAPI(
 	a.eg.Go(func() error {
 		logger.Debug(ctx, "starting agent routine")
 		err := f(ctx, a.aAPI)
-		if xerrors.Is(err, context.Canceled) && ctx.Err() != nil {
+		if errors.Is(err, context.Canceled) && ctx.Err() != nil {
 			logger.Debug(ctx, "swallowing context canceled")
 			// Don't propagate context canceled errors to the error group, because we don't want the
 			// graceful context being canceled to halt the work of routines with
@@ -2042,7 +2050,7 @@ func (a *apiConnRoutineManager) startTailnetAPI(
 	a.eg.Go(func() error {
 		logger.Debug(ctx, "starting tailnet routine")
 		err := f(ctx, a.tAPI)
-		if xerrors.Is(err, context.Canceled) && ctx.Err() != nil {
+		if errors.Is(err, context.Canceled) && ctx.Err() != nil {
 			logger.Debug(ctx, "swallowing context canceled")
 			// Don't propagate context canceled errors to the error group, because we don't want the
 			// graceful context being canceled to halt the work of routines with

diff --git a/agent/agenttest/client.go b/agent/agenttest/client.go
@@ -21,6 +21,7 @@ import (
 	"tailscale.com/tailcfg"
 
 	"cdr.dev/slog"
+
 	agentproto "github.com/coder/coder/v2/agent/proto"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"

diff --git a/agent/metrics.go b/agent/metrics.go
@@ -20,6 +20,7 @@ type agentMetrics struct {
 	// took to run. This is reported once per agent.
 	startupScriptSeconds *prometheus.GaugeVec
 	currentConnections   *prometheus.GaugeVec
+	manifestsReceived    prometheus.Counter
 }
 
 func newAgentMetrics(registerer prometheus.Registerer) *agentMetrics {
@@ -54,11 +55,20 @@ func newAgentMetrics(registerer prometheus.Registerer) *agentMetrics {
 	}, []string{"connection_type"})
 	registerer.MustRegister(currentConnections)
 
+	manifestsReceived := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "coderd",
+		Subsystem: "agentstats",
+		Name:      "manifests_received",
+		Help:      "The number of manifests this agent has received from the control plane.",
+	})
+	registerer.MustRegister(manifestsReceived)
+
 	return &agentMetrics{
 		connectionsTotal:      connectionsTotal,
 		reconnectingPTYErrors: reconnectingPTYErrors,
 		startupScriptSeconds:  startupScriptSeconds,
 		currentConnections:    currentConnections,
+		manifestsReceived:     manifestsReceived,
 	}
 }
 

diff --git a/agent/reaper/reaper_unix.go b/agent/reaper/reaper_unix.go
@@ -3,6 +3,7 @@
 package reaper
 
 import (
+	"fmt"
 	"os"
 	"os/signal"
 	"syscall"
@@ -29,6 +30,10 @@ func catchSignals(pid int, sigs []os.Signal) {
 		s := <-sc
 		sig, ok := s.(syscall.Signal)
 		if ok {
+			// TODO:
+			// Tried using a logger here but the I/O streams are already closed at this point...
+			// Why is os.Stderr still working then?
+			_, _ = fmt.Fprintf(os.Stderr, "reaper caught %q signal, killing process %v\n", sig.String(), pid)
 			_ = syscall.Kill(pid, sig)
 		}
 	}

diff --git a/cli/agent.go b/cli/agent.go
@@ -19,12 +19,16 @@
 	"golang.org/x/xerrors"
 	"gopkg.in/natefinch/lumberjack.v2"
 
+	"github.com/coder/retry"
+
 	"github.com/prometheus/client_golang/prometheus"
 
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/sloghuman"
 	"cdr.dev/slog/sloggers/slogjson"
 	"cdr.dev/slog/sloggers/slogstackdriver"
+	"github.com/coder/serpent"
+
 	"github.com/coder/coder/v2/agent"
 	"github.com/coder/coder/v2/agent/agentcontainers"
 	"github.com/coder/coder/v2/agent/agentexec"
@@ -34,7 +38,6 @@
 	"github.com/coder/coder/v2/cli/clilog"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
-	"github.com/coder/serpent"
 )
 
 func (r *RootCmd) workspaceAgent() *serpent.Command {
@@ -63,8 +66,10 @@
 		// This command isn't useful to manually execute.
 		Hidden: true,
 		Handler: func(inv *serpent.Invocation) error {
-			ctx, cancel := context.WithCancel(inv.Context())
-			defer cancel()
+			ctx, cancel := context.WithCancelCause(inv.Context())
+			defer func() {
+				cancel(xerrors.New("defer"))
+			}()
 
 			var (
 				ignorePorts = map[int]string{}
@@ -281,7 +286,6 @@
 				return xerrors.Errorf("add executable to $PATH: %w", err)
 			}
 
-			prometheusRegistry := prometheus.NewRegistry()
 			subsystemsRaw := inv.Environ.Get(agent.EnvAgentSubsystem)
 			subsystems := []codersdk.AgentSubsystem{}
 			for _, s := range strings.Split(subsystemsRaw, ",") {
@@ -328,46 +332,90 @@
 				containerLister = agentcontainers.NewDocker(execer)
 			}
 
-			agnt := agent.New(agent.Options{
-				Client:        client,
-				Logger:        logger,
-				LogDir:        logDir,
-				ScriptDataDir: scriptDataDir,
-				// #nosec G115 - Safe conversion as tailnet listen port is within uint16 range (0-65535)
-				TailnetListenPort: uint16(tailnetListenPort),
-				ExchangeToken: func(ctx context.Context) (string, error) {
-					if exchangeToken == nil {
-						return client.SDK.SessionToken(), nil
+			// TODO: timeout ok?
+			reinitCtx, reinitCancel := context.WithTimeout(context.Background(), time.Hour*24)
+			defer reinitCancel()
+			reinitEvents := make(chan agentsdk.ReinitializationResponse)
+
+			go func() {
+				// Retry to wait for reinit, main context cancels the retrier.
+				for retrier := retry.New(100*time.Millisecond, 10*time.Second); retrier.Wait(ctx); {
+					select {
+					case <-reinitCtx.Done():
+						return
+					default:
 					}
-					resp, err := exchangeToken(ctx)
+
+					err := client.WaitForReinit(reinitCtx, reinitEvents)
 					if err != nil {
-						return "", err
+						logger.Error(ctx, "failed to wait for reinit instructions, will retry", slog.Error(err))
 					}
-					client.SetSessionToken(resp.SessionToken)
-					return resp.SessionToken, nil
-				},
-				EnvironmentVariables: environmentVariables,
-				IgnorePorts:          ignorePorts,
-				SSHMaxTimeout:        sshMaxTimeout,
-				Subsystems:           subsystems,
-
-				PrometheusRegistry: prometheusRegistry,
-				BlockFileTransfer:  blockFileTransfer,
-				Execer:             execer,
-				ContainerLister:    containerLister,
-
-				ExperimentalDevcontainersEnabled: experimentalDevcontainersEnabled,
-			})
-
-			promHandler := agent.PrometheusMetricsHandler(prometheusRegistry, logger)
-			prometheusSrvClose := ServeHandler(ctx, logger, promHandler, prometheusAddress, "prometheus")
-			defer prometheusSrvClose()
-
-			debugSrvClose := ServeHandler(ctx, logger, agnt.HTTPDebug(), debugAddress, "debug")
-			defer debugSrvClose()
-
-			<-ctx.Done()
-			return agnt.Close()
+				}
+			}()
+
+			var (
+				lastErr  error
+				mustExit bool
+			)
+			for {
+				prometheusRegistry := prometheus.NewRegistry()
+
+				agnt := agent.New(agent.Options{
+					Client:        client,
+					Logger:        logger,
+					LogDir:        logDir,
+					ScriptDataDir: scriptDataDir,
+					// #nosec G115 - Safe conversion as tailnet listen port is within uint16 range (0-65535)
+					TailnetListenPort: uint16(tailnetListenPort),
+					ExchangeToken: func(ctx context.Context) (string, error) {
+						if exchangeToken == nil {
+							return client.SDK.SessionToken(), nil
+						}
+						resp, err := exchangeToken(ctx)
+						if err != nil {
+							return "", err
+						}
+						client.SetSessionToken(resp.SessionToken)
+						return resp.SessionToken, nil
+					},
+					EnvironmentVariables: environmentVariables,
+					IgnorePorts:          ignorePorts,
+					SSHMaxTimeout:        sshMaxTimeout,
+					Subsystems:           subsystems,
+
+					PrometheusRegistry:               prometheusRegistry,
+					BlockFileTransfer:                blockFileTransfer,
+					Execer:                           execer,
+					ContainerLister:                  containerLister,
+					ExperimentalDevcontainersEnabled: experimentalDevcontainersEnabled,
+				})
+
+				promHandler := agent.PrometheusMetricsHandler(prometheusRegistry, logger)
+				prometheusSrvClose := ServeHandler(ctx, logger, promHandler, prometheusAddress, "prometheus")
+
+				debugSrvClose := ServeHandler(ctx, logger, agnt.HTTPDebug(), debugAddress, "debug")
+
+				select {
+				case <-ctx.Done():
+					logger.Warn(ctx, "agent shutting down", slog.Error(ctx.Err()), slog.F("cause", context.Cause(ctx)))
+					mustExit = true
+				case event := <-reinitEvents:
+					logger.Warn(ctx, "agent received instruction to reinitialize",
+						slog.F("message", event.Message), slog.F("reason", event.Reason))
+				}
+
+				lastErr = agnt.Close()
+				debugSrvClose()
+				prometheusSrvClose()
+
+				if mustExit {
+					reinitCancel()
+					break
+				}
+
+				logger.Info(ctx, "reinitializing...")
+			}
+			return lastErr
 		},
 	}
 

diff --git a/cli/testdata/coder_server_--help.golden b/cli/testdata/coder_server_--help.golden
@@ -670,6 +670,12 @@ workspaces stopping during the day due to template scheduling.
           must be *. Only one hour and minute can be specified (ranges or comma
           separated values are not supported).
 
+WORKSPACE PREBUILDS OPTIONS: 
+Configure how workspace prebuilds behave.
+
+      --workspace-prebuilds-reconciliation-interval duration, $CODER_WORKSPACE_PREBUILDS_RECONCILIATION_INTERVAL (default: 15s)
+          How often to reconcile workspace prebuilds state.
+
 ⚠️ DANGEROUS OPTIONS: 
       --dangerous-allow-path-app-sharing bool, $CODER_DANGEROUS_ALLOW_PATH_APP_SHARING
           Allow workspace apps that are not served from subdomains to be shared.

diff --git a/cli/testdata/server-config.yaml.golden b/cli/testdata/server-config.yaml.golden
@@ -688,3 +688,15 @@ notifications:
   # How often to query the database for queued notifications.
   # (default: 15s, type: duration)
   fetchInterval: 15s
+# Configure how workspace prebuilds behave.
+workspace_prebuilds:
+  # How often to reconcile workspace prebuilds state.
+  # (default: 15s, type: duration)
+  reconciliation_interval: 15s
+  # Interval to increase reconciliation backoff by when unrecoverable errors occur.
+  # (default: 15s, type: duration)
+  reconciliation_backoff_interval: 15s
+  # Interval to look back to determine number of failed builds, which influences
+  # backoff.
+  # (default: 1h0m0s, type: duration)
+  reconciliation_backoff_lookback_period: 1h0m0s
diff --git a/coderd/agentapi/api.go b/coderd/agentapi/api.go
@@ -109,6 +109,8 @@ func New(opts Options) *API {
 		Database:                 opts.Database,
 		DerpMapFn:                opts.DerpMapFn,
 		WorkspaceID:              opts.WorkspaceID,
+		Log:                      opts.Log.Named("manifests"),
+		Pubsub:                   opts.Pubsub,
 	}
 
 	api.AnnouncementBannerAPI = &AnnouncementBannerAPI{

diff --git a/coderd/agentapi/manifest.go b/coderd/agentapi/manifest.go
@@ -8,6 +8,10 @@ import (
 	"strings"
 	"time"
 
+	"cdr.dev/slog"
+
+	"github.com/coder/coder/v2/coderd/database/pubsub"
+
 	"github.com/google/uuid"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
@@ -35,6 +39,8 @@ type ManifestAPI struct {
 	AgentFn   func(context.Context) (database.WorkspaceAgent, error)
 	Database  database.Store
 	DerpMapFn func() *tailcfg.DERPMap
+	Pubsub    pubsub.Pubsub
+	Log       slog.Logger
 }
 
 func (a *ManifestAPI) GetManifest(ctx context.Context, _ *agentproto.GetManifestRequest) (*agentproto.Manifest, error) {