coder · evgeniy-scherbina · Apr 17, 2025 · Mar 12, 2025 · Mar 13, 2025 · Mar 13, 2025
diff --git a/.gitignore b/.gitignore
@@ -79,3 +79,6 @@ result
 
 # Zed
 .zed_server
+
+# dlv debug binaries for go tests
+__debug_bin*
diff --git a/cli/ssh_test.go b/cli/ssh_test.go
@@ -1977,7 +1977,9 @@ Expire-Date: 0
 	tpty.WriteLine("gpg --list-keys && echo gpg-''-listkeys-command-done")
 	listKeysOutput := tpty.ExpectMatch("gpg--listkeys-command-done")
 	require.Contains(t, listKeysOutput, "[ultimate] Coder Test <test@coder.com>")
-	require.Contains(t, listKeysOutput, "[ultimate] Dean Sheather (work key) <dean@coder.com>")
+	// It's fine that this key is expired. We're just testing that the key trust
+	// gets synced properly.
+	require.Contains(t, listKeysOutput, "[ expired] Dean Sheather (work key) <dean@coder.com>")
 
 	// Try to sign something. This demonstrates that the forwarding is
 	// working as expected, since the workspace doesn't have access to the

diff --git a/cli/testdata/coder_list_--output_json.golden b/cli/testdata/coder_list_--output_json.golden
@@ -67,7 +67,8 @@
         "count": 0,
         "available": 0,
         "most_recently_seen": null
-      }
+      },
+      "template_version_preset_id": null
     },
     "latest_app_status": null,
     "outdated": false,

diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go
diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json
diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go
@@ -9806,19 +9806,20 @@ func (q *FakeQuerier) InsertWorkspaceBuild(_ context.Context, arg database.Inser
 	defer q.mutex.Unlock()
 
 	workspaceBuild := database.WorkspaceBuild{
-		ID:                arg.ID,
-		CreatedAt:         arg.CreatedAt,
-		UpdatedAt:         arg.UpdatedAt,
-		WorkspaceID:       arg.WorkspaceID,
-		TemplateVersionID: arg.TemplateVersionID,
-		BuildNumber:       arg.BuildNumber,
-		Transition:        arg.Transition,
-		InitiatorID:       arg.InitiatorID,
-		JobID:             arg.JobID,
-		ProvisionerState:  arg.ProvisionerState,
-		Deadline:          arg.Deadline,
-		MaxDeadline:       arg.MaxDeadline,
-		Reason:            arg.Reason,
+		ID:                      arg.ID,
+		CreatedAt:               arg.CreatedAt,
+		UpdatedAt:               arg.UpdatedAt,
+		WorkspaceID:             arg.WorkspaceID,
+		TemplateVersionID:       arg.TemplateVersionID,
+		BuildNumber:             arg.BuildNumber,
+		Transition:              arg.Transition,
+		InitiatorID:             arg.InitiatorID,
+		JobID:                   arg.JobID,
+		ProvisionerState:        arg.ProvisionerState,
+		Deadline:                arg.Deadline,
+		MaxDeadline:             arg.MaxDeadline,
+		Reason:                  arg.Reason,
+		TemplateVersionPresetID: arg.TemplateVersionPresetID,
 	}
 	q.workspaceBuilds = append(q.workspaceBuilds, workspaceBuild)
 	return nil

diff --git a/coderd/database/lock.go b/coderd/database/lock.go
@@ -13,7 +13,6 @@ const (
 	LockIDNotificationsReportGenerator
 	LockIDCryptoKeyRotation
 	LockIDReconcileTemplatePrebuilds
-	LockIDDeterminePrebuildsState
 )
 
 // GenLockID generates a unique and consistent lock ID from a given string.

diff --git a/coderd/database/querier.go b/coderd/database/querier.go
diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go
diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql
@@ -57,9 +57,9 @@ WHERE (b.transition = 'start'::workspace_transition
 	AND b.job_status = 'succeeded'::provisioner_job_status);
 
 -- name: CountInProgressPrebuilds :many
--- CountInProgressPrebuilds returns the number of in-progress prebuilds, grouped by template version ID and transition.
+-- CountInProgressPrebuilds returns the number of in-progress prebuilds, grouped by preset ID and transition.
 -- Prebuild considered in-progress if it's in the "starting", "stopping", or "deleting" state.
-SELECT t.id AS template_id, wpb.template_version_id, wpb.transition, COUNT(wpb.transition)::int AS count
+SELECT t.id AS template_id, wpb.template_version_id, wpb.transition, COUNT(wpb.transition)::int AS count, wlb.template_version_preset_id as preset_id
 FROM workspace_latest_builds wlb
 		INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id
 		-- We only need these counts for active template versions.
@@ -70,7 +70,7 @@ FROM workspace_latest_builds wlb
 		-- prebuilds that are still building.
 		INNER JOIN templates t ON t.active_version_id = wlb.template_version_id
 WHERE wlb.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status)
-GROUP BY t.id, wpb.template_version_id, wpb.transition;
+GROUP BY t.id, wpb.template_version_id, wpb.transition, wlb.template_version_preset_id;
 
 -- GetPresetsBackoff groups workspace builds by preset ID.
 -- Each preset is associated with exactly one template version ID.

diff --git a/coderd/prebuilds/api.go b/coderd/prebuilds/api.go
@@ -0,0 +1,53 @@
+package prebuilds
+
+import (
+	"context"
+
+	"github.com/coder/coder/v2/coderd/database"
+)
+
+// ReconciliationOrchestrator manages the lifecycle of prebuild reconciliation.
+// It runs a continuous loop to check and reconcile prebuild states, and can be stopped gracefully.
+type ReconciliationOrchestrator interface {
+	Reconciler
+
+	// RunLoop starts a continuous reconciliation loop that periodically calls ReconcileAll
+	// to ensure all prebuilds are in their desired states. The loop runs until the context
+	// is canceled or Stop is called.
+	RunLoop(ctx context.Context)
+
+	// Stop gracefully shuts down the orchestrator with the given cause.
+	// The cause is used for logging and error reporting.
+	Stop(ctx context.Context, cause error)
+}
+
+// Reconciler defines the core operations for managing prebuilds.
+// It provides both high-level orchestration (ReconcileAll) and lower-level operations
+// for more fine-grained control (SnapshotState, ReconcilePreset, CalculateActions).
+// All database operations must be performed within repeatable-read transactions
+// to ensure consistency.
+type Reconciler interface {
+	// ReconcileAll orchestrates the reconciliation of all prebuilds across all templates.
+	// It takes a global snapshot of the system state and then reconciles each preset
+	// in parallel, creating or deleting prebuilds as needed to reach their desired states.
+	// For more fine-grained control, you can use the lower-level methods SnapshotState
+	// and ReconcilePreset directly.
+	ReconcileAll(ctx context.Context) error
+
+	// SnapshotState captures the current state of all prebuilds across templates.
+	// It creates a global database snapshot that can be viewed as a collection of PresetSnapshots,
+	// each representing the state of prebuilds for a specific preset.
+	// MUST be called inside a repeatable-read transaction.
+	SnapshotState(ctx context.Context, store database.Store) (*GlobalSnapshot, error)
+
+	// ReconcilePreset handles a single PresetSnapshot, determining and executing
+	// the required actions (creating or deleting prebuilds) based on the current state.
+	// MUST be called inside a repeatable-read transaction.
+	ReconcilePreset(ctx context.Context, snapshot PresetSnapshot) error
+
+	// CalculateActions determines what actions are needed to reconcile a preset's prebuilds
+	// to their desired state. This includes creating new prebuilds, deleting excess ones,
+	// or waiting due to backoff periods.
+	// MUST be called inside a repeatable-read transaction.
+	CalculateActions(ctx context.Context, state PresetSnapshot) (*ReconciliationActions, error)
+}
diff --git a/coderd/prebuilds/global_snapshot.go b/coderd/prebuilds/global_snapshot.go
@@ -0,0 +1,66 @@
+package prebuilds
+
+import (
+	"github.com/google/uuid"
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/util/slice"
+)
+
+// GlobalSnapshot represents a full point-in-time snapshot of state relating to prebuilds across all templates.
+type GlobalSnapshot struct {
+	Presets             []database.GetTemplatePresetsWithPrebuildsRow
+	RunningPrebuilds    []database.GetRunningPrebuiltWorkspacesRow
+	PrebuildsInProgress []database.CountInProgressPrebuildsRow
+	Backoffs            []database.GetPresetsBackoffRow
+}
+
+func NewGlobalSnapshot(
+	presets []database.GetTemplatePresetsWithPrebuildsRow,
+	runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow,
+	prebuildsInProgress []database.CountInProgressPrebuildsRow,
+	backoffs []database.GetPresetsBackoffRow,
+) GlobalSnapshot {
+	return GlobalSnapshot{
+		Presets:             presets,
+		RunningPrebuilds:    runningPrebuilds,
+		PrebuildsInProgress: prebuildsInProgress,
+		Backoffs:            backoffs,
+	}
+}
+
+func (s GlobalSnapshot) FilterByPreset(presetID uuid.UUID) (*PresetSnapshot, error) {
+	preset, found := slice.Find(s.Presets, func(preset database.GetTemplatePresetsWithPrebuildsRow) bool {
+		return preset.ID == presetID
+	})
+	if !found {
+		return nil, xerrors.Errorf("no preset found with ID %q", presetID)
+	}
+
+	running := slice.Filter(s.RunningPrebuilds, func(prebuild database.GetRunningPrebuiltWorkspacesRow) bool {
+		if !prebuild.CurrentPresetID.Valid {
+			return false
+		}
+		return prebuild.CurrentPresetID.UUID == preset.ID
+	})
+
+	inProgress := slice.Filter(s.PrebuildsInProgress, func(prebuild database.CountInProgressPrebuildsRow) bool {
+		return prebuild.PresetID.UUID == preset.ID
+	})
+
+	var backoffPtr *database.GetPresetsBackoffRow
+	backoff, found := slice.Find(s.Backoffs, func(row database.GetPresetsBackoffRow) bool {
+		return row.PresetID == preset.ID
+	})
+	if found {
+		backoffPtr = &backoff
+	}
+
+	return &PresetSnapshot{
+		Preset:     preset,
+		Running:    running,
+		InProgress: inProgress,
+		Backoff:    backoffPtr,
+	}, nil
+}
diff --git a/coderd/prebuilds/noop.go b/coderd/prebuilds/noop.go
@@ -0,0 +1,34 @@
+package prebuilds
+
+import (
+	"context"
+
+	"github.com/coder/coder/v2/coderd/database"
+)
+
+type NoopReconciler struct{}
+
+func NewNoopReconciler() *NoopReconciler {
+	return &NoopReconciler{}
+}
+
+func (NoopReconciler) RunLoop(context.Context)     {}
+func (NoopReconciler) Stop(context.Context, error) {}
+
+func (NoopReconciler) ReconcileAll(context.Context) error {
+	return nil
+}
+
+func (NoopReconciler) SnapshotState(context.Context, database.Store) (*GlobalSnapshot, error) {
+	return &GlobalSnapshot{}, nil
+}
+
+func (NoopReconciler) ReconcilePreset(context.Context, PresetSnapshot) error {
+	return nil
+}
+
+func (NoopReconciler) CalculateActions(context.Context, PresetSnapshot) (*ReconciliationActions, error) {
+	return &ReconciliationActions{}, nil
+}
+
+var _ ReconciliationOrchestrator = NoopReconciler{}