fix: using prebuild status

coder · evgeniy-scherbina · May 21, 2025 · May 6, 2025 · May 12, 2025 · May 12, 2025
commit 0e4600cc84a3d9afb465ebc3625807ee1f7f1bad
diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go
@@ -4178,6 +4178,10 @@ func (q *querier) UpdateOrganizationDeletedByID(ctx context.Context, arg databas
 	return deleteQ(q.log, q.auth, q.db.GetOrganizationByID, deleteF)(ctx, arg.ID)
 }
 
+func (q *querier) UpdatePrebuildStatus(ctx context.Context, arg database.UpdatePrebuildStatusParams) error {
+	panic("not implemented")
+}
+
 func (q *querier) UpdateProvisionerDaemonLastSeenAt(ctx context.Context, arg database.UpdateProvisionerDaemonLastSeenAtParams) error {
 	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerDaemon); err != nil {
 		return err

diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go
@@ -10867,6 +10867,15 @@ func (q *FakeQuerier) UpdateOrganizationDeletedByID(_ context.Context, arg datab
 	return sql.ErrNoRows
 }
 
+func (q *FakeQuerier) UpdatePrebuildStatus(ctx context.Context, arg database.UpdatePrebuildStatusParams) error {
+	err := validateDatabaseType(arg)
+	if err != nil {
+		return err
+	}
+
+	panic("not implemented")
+}
+
 func (q *FakeQuerier) UpdateProvisionerDaemonLastSeenAt(_ context.Context, arg database.UpdateProvisionerDaemonLastSeenAtParams) error {
 	err := validateDatabaseType(arg)
 	if err != nil {

diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go
diff --git a/coderd/database/querier.go b/coderd/database/querier.go
diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go
diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql
@@ -33,6 +33,7 @@ SELECT
 		tvp.id,
 		tvp.name,
 		tvp.desired_instances       AS desired_instances,
+		tvp.prebuild_status,
 		t.deleted,
 		t.deprecated != ''          AS deprecated
 FROM templates t

diff --git a/coderd/database/queries/presets.sql b/coderd/database/queries/presets.sql
@@ -25,6 +25,11 @@ SELECT
 	unnest(@values :: TEXT[])
 RETURNING *;
 
+-- name: UpdatePrebuildStatus :exec
+UPDATE template_version_presets
+SET prebuild_status = @status
+WHERE id = @preset_id;
+
 -- name: GetPresetsByTemplateVersionID :many
 SELECT
 	*

diff --git a/enterprise/coderd/prebuilds/reconcile.go b/enterprise/coderd/prebuilds/reconcile.go
@@ -361,49 +361,32 @@ func (c *StoreReconciler) ReconcilePreset(ctx context.Context, ps prebuilds.Pres
 		slog.F("preset_name", ps.Preset.Name),
 	)
 
-	if ps.IsHardLimited {
+	// If the preset was previously hard-limited, log it and exit early.
+	if ps.Preset.PrebuildStatus.PrebuildStatus == database.PrebuildStatusHardLimited {
 		logger.Warn(ctx, "skipping hard limited preset", slog.F("preset_id", ps.Preset.ID), slog.F("name", ps.Preset.Name))
+		return nil
+	}
 
-		// TODO: rename ctx?
-		// nolint:gocritic // Necessary to query all the required data.
-		ctx := dbauthz.AsSystemRestricted(ctx)
-
-		// TODO(yevhenii): move into separate function
-		// Send notification to template admins.
-		if c.notifEnq == nil {
-			c.logger.Warn(ctx, "notification enqueuer not set, cannot send resource replacement notification(s)")
-			return nil
-		}
+	// If the preset reached the hard failure limit for the first time during this iteration:
+	// - Mark it as hard-limited in the database
+	// - Send notifications to template admins
+	if ps.IsHardLimited {
+		logger.Warn(ctx, "skipping hard limited preset", slog.F("preset_id", ps.Preset.ID), slog.F("name", ps.Preset.Name))
 
-		// TODO(yevhenii): remove owner from the list
-		templateAdmins, err := c.store.GetUsers(ctx, database.GetUsersParams{
-			RbacRole: []string{codersdk.RoleTemplateAdmin, codersdk.RoleOwner},
+		err := c.store.UpdatePrebuildStatus(ctx, database.UpdatePrebuildStatusParams{
+			Status: database.NullPrebuildStatus{
+				PrebuildStatus: database.PrebuildStatusHardLimited,
+				Valid:          true,
+			},
+			PresetID: ps.Preset.ID,
 		})
 		if err != nil {
-			return xerrors.Errorf("fetch template admins: %w", err)
+			return err
 		}
 
-		for _, templateAdmin := range templateAdmins {
-			if _, err := c.notifEnq.EnqueueWithData(ctx, templateAdmin.ID, notifications.PrebuildFailureLimitReached,
-				map[string]string{
-					"org":              ps.Preset.OrganizationName,
-					"template":         ps.Preset.TemplateName,
-					"template_version": ps.Preset.TemplateVersionName,
-					"preset":           ps.Preset.Name,
-				},
-				map[string]any{},
-				"prebuilds_reconciler",
-				// Associate this notification with all the related entities.
-				ps.Preset.TemplateID, ps.Preset.TemplateVersionID, ps.Preset.ID,
-			); err != nil {
-				c.logger.Error(ctx,
-					"failed to send notification",
-					slog.Error(err),
-					slog.F("template_admin_id", templateAdmin.ID.String()),
-				)
-
-				continue
-			}
+		err = c.notifyPrebuildFailureLimitReached(ctx, ps)
+		if err != nil {
+			return err
 		}
 
 		return nil
@@ -502,6 +485,52 @@ func (c *StoreReconciler) ReconcilePreset(ctx context.Context, ps prebuilds.Pres
 	}
 }
 
+func (c *StoreReconciler) notifyPrebuildFailureLimitReached(ctx context.Context, ps prebuilds.PresetSnapshot) error {
+	// TODO: rename ctx?
+	// nolint:gocritic // Necessary to query all the required data.
+	ctx = dbauthz.AsSystemRestricted(ctx)
+
+	// TODO(yevhenii): move into separate function
+	// Send notification to template admins.
+	if c.notifEnq == nil {
+		c.logger.Warn(ctx, "notification enqueuer not set, cannot send resource replacement notification(s)")
+		return nil
+	}
+
+	// TODO(yevhenii): remove owner from the list
+	templateAdmins, err := c.store.GetUsers(ctx, database.GetUsersParams{
+		RbacRole: []string{codersdk.RoleTemplateAdmin, codersdk.RoleOwner},
+	})
+	if err != nil {
+		return xerrors.Errorf("fetch template admins: %w", err)
+	}
+
+	for _, templateAdmin := range templateAdmins {
+		if _, err := c.notifEnq.EnqueueWithData(ctx, templateAdmin.ID, notifications.PrebuildFailureLimitReached,
+			map[string]string{
+				"org":              ps.Preset.OrganizationName,
+				"template":         ps.Preset.TemplateName,
+				"template_version": ps.Preset.TemplateVersionName,
+				"preset":           ps.Preset.Name,
+			},
+			map[string]any{},
+			"prebuilds_reconciler",
+			// Associate this notification with all the related entities.
+			ps.Preset.TemplateID, ps.Preset.TemplateVersionID, ps.Preset.ID,
+		); err != nil {
+			c.logger.Error(ctx,
+				"failed to send notification",
+				slog.Error(err),
+				slog.F("template_admin_id", templateAdmin.ID.String()),
+			)
+
+			continue
+		}
+	}
+
+	return nil
+}
+
 func (c *StoreReconciler) CalculateActions(ctx context.Context, snapshot prebuilds.PresetSnapshot) (*prebuilds.ReconciliationActions, error) {
 	if ctx.Err() != nil {
 		return nil, ctx.Err()

diff --git a/enterprise/coderd/prebuilds/reconcile_test.go b/enterprise/coderd/prebuilds/reconcile_test.go
@@ -733,6 +733,11 @@ func TestSkippingHardLimitedPresets(t *testing.T) {
 			// The outcome depends on whether the hard limit has been reached
 			require.NoError(t, controller.ReconcileAll(ctx))
 
+			// These two additional calls to ReconcileAll should not trigger any notifications.
+			// A notification is only sent once.
+			require.NoError(t, controller.ReconcileAll(ctx))
+			require.NoError(t, controller.ReconcileAll(ctx))
+
 			// Verify the final state after reconciliation
 			workspaces, err = db.GetWorkspacesByTemplateID(ctx, template.ID)
 			require.NoError(t, err)