feat: add notification for prebuilds failure

evgeniy-scherbina · evgeniy-scherbina · commit 5c60065db5b1 · 2025-05-15T18:36:03.000Z
diff --git a/coderd/database/migrations/000325_prebuild_failure_limit_notification.down.sql b/coderd/database/migrations/000325_prebuild_failure_limit_notification.down.sql
@@ -0,0 +1 @@
+DELETE FROM notification_templates WHERE id = '414d9331-c1fc-4761-b40c-d1f4702279eb';
diff --git a/coderd/database/migrations/000325_prebuild_failure_limit_notification.up.sql b/coderd/database/migrations/000325_prebuild_failure_limit_notification.up.sql
@@ -0,0 +1,10 @@
+INSERT INTO notification_templates
+(id, name, title_template, body_template, "group", actions)
+VALUES ('414d9331-c1fc-4761-b40c-d1f4702279eb',
+		'Prebuild Failure Limit Reached',
+		E'There is a problem creating prebuilt workspaces for the preset',
+		$$
+			The number of failed prebuilds has reached the hard limit for template **{{ .Labels.template }}** and preset **{{ .Labels.preset }}**
+		$$,
+		'Template Events',
+		'[]'::jsonb);
diff --git a/coderd/notifications/events.go b/coderd/notifications/events.go
@@ -40,6 +40,7 @@ var (
 
 	TemplateWorkspaceBuildsFailedReport = uuid.MustParse("34a20db2-e9cc-4a93-b0e4-8569699d7a00")
 	TemplateWorkspaceResourceReplaced   = uuid.MustParse("89d9745a-816e-4695-a17f-3d0a229e2b8d")
+	PrebuildFailureLimitReached         = uuid.MustParse("414d9331-c1fc-4761-b40c-d1f4702279eb")
 )
 
 // Notification-related events.
diff --git a/enterprise/coderd/prebuilds/reconcile.go b/enterprise/coderd/prebuilds/reconcile.go
@@ -362,7 +362,50 @@ func (c *StoreReconciler) ReconcilePreset(ctx context.Context, ps prebuilds.Pres
 	)
 
 	if ps.IsHardLimited {
-		logger.Debug(ctx, "skipping hard limited preset", slog.F("preset_id", ps.Preset.ID), slog.F("name", ps.Preset.Name))
+		logger.Warn(ctx, "skipping hard limited preset", slog.F("preset_id", ps.Preset.ID), slog.F("name", ps.Preset.Name))
+
+		// TODO: rename ctx?
+		// nolint:gocritic // Necessary to query all the required data.
+		ctx := dbauthz.AsSystemRestricted(ctx)
+
+		// TODO(yevhenii): move into separate function
+		// Send notification to template admins.
+		if c.notifEnq == nil {
+			c.logger.Warn(ctx, "notification enqueuer not set, cannot send resource replacement notification(s)")
+			return nil
+		}
+
+		// TODO(yevhenii): remove owner from the list
+		templateAdmins, err := c.store.GetUsers(ctx, database.GetUsersParams{
+			RbacRole: []string{codersdk.RoleTemplateAdmin, codersdk.RoleOwner},
+		})
+		if err != nil {
+			return xerrors.Errorf("fetch template admins: %w", err)
+		}
+
+		for _, templateAdmin := range templateAdmins {
+			if _, err := c.notifEnq.EnqueueWithData(ctx, templateAdmin.ID, notifications.PrebuildFailureLimitReached,
+				map[string]string{
+					"org":              ps.Preset.OrganizationName,
+					"template":         ps.Preset.TemplateName,
+					"template_version": ps.Preset.TemplateVersionName,
+					"preset":           ps.Preset.Name,
+				},
+				map[string]any{},
+				"prebuilds_reconciler",
+				// Associate this notification with all the related entities.
+				ps.Preset.TemplateID, ps.Preset.TemplateVersionID, ps.Preset.ID,
+			); err != nil {
+				c.logger.Error(ctx,
+					"failed to send notification",
+					slog.Error(err),
+					slog.F("template_admin_id", templateAdmin.ID.String()),
+				)
+
+				continue
+			}
+		}
+
 		return nil
 	}
 

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+DELETE FROM notification_templates WHERE id = '414d9331-c1fc-4761-b40c-d1f4702279eb';`
Original file line number	Diff line number	Diff line change
`@@ -40,6 +40,7 @@ var (`
`40`	`40`
`41`	`41`	`TemplateWorkspaceBuildsFailedReport = uuid.MustParse("34a20db2-e9cc-4a93-b0e4-8569699d7a00")`
`42`	`42`	`TemplateWorkspaceResourceReplaced = uuid.MustParse("89d9745a-816e-4695-a17f-3d0a229e2b8d")`
	`43`	`+ PrebuildFailureLimitReached = uuid.MustParse("414d9331-c1fc-4761-b40c-d1f4702279eb")`
`43`	`44`	`)`
`44`	`45`
`45`	`46`	`// Notification-related events.`