coder · dannykopping · Apr 25, 2025 · Apr 25, 2025 · Apr 25, 2025 · Apr 25, 2025
diff --git a/coderd/database/migrations/000318_resource_replacements_notification.down.sql b/coderd/database/migrations/000318_resource_replacements_notification.down.sql
@@ -0,0 +1 @@
+DELETE FROM notification_templates WHERE id = '89d9745a-816e-4695-a17f-3d0a229e2b8d';
diff --git a/coderd/database/migrations/000318_resource_replacements_notification.up.sql b/coderd/database/migrations/000318_resource_replacements_notification.up.sql
@@ -0,0 +1,27 @@
+INSERT INTO notification_templates
+	(id, name, title_template, body_template, "group", actions)
+VALUES ('89d9745a-816e-4695-a17f-3d0a229e2b8d',
+		'Prebuilt Workspace Resource Replaced',
+		E'There might be a problem with a recently claimed prebuilt workspace',
+		$$
+Workspace **{{.Labels.workspace}}** was claimed from a prebuilt workspace by **{{.Labels.claimant}}**.
+During the claim, Terraform destroyed and recreated the following resources
+because one or more immutable attributes changed:
+
+{{range $resource, $paths := .Data.replacements -}}
+- _{{ $resource }}_  was replaced due to changes to _{{ $paths }}_
+{{end}}
+
+When Terraform must change an immutable attribute, it replaces the entire resource.
+If you’re using prebuilds to speed up provisioning, unexpected replacements will slow down
+workspace startup—even when claiming a prebuilt environment.
+
+For tips on preventing replacements and improving claim performance, see [this guide](https://coder.com/docs/TODO).
+$$,
+		'Workspace Events',
+		'[
+		{
+			"label": "View workspace build",
+			"url": "{{base_url}}/@{{.Labels.claimant}}/{{.Labels.workspace}}/builds/{{.Labels.workspace_build_num}}"
+		}
+	]'::jsonb);
diff --git a/coderd/notifications/events.go b/coderd/notifications/events.go
@@ -18,6 +18,7 @@ var (
 	TemplateWorkspaceManualBuildFailed = uuid.MustParse("2faeee0f-26cb-4e96-821c-85ccb9f71513")
 	TemplateWorkspaceOutOfMemory       = uuid.MustParse("a9d027b4-ac49-4fb1-9f6d-45af15f64e7a")
 	TemplateWorkspaceOutOfDisk         = uuid.MustParse("f047f6a3-5713-40f7-85aa-0394cce9fa3a")
+	TemplateWorkspaceResourceReplaced  = uuid.MustParse("89d9745a-816e-4695-a17f-3d0a229e2b8d")
 )
 
 // Account-related events.

diff --git a/coderd/notifications/notifications_test.go b/coderd/notifications/notifications_test.go
@@ -35,6 +35,9 @@ import (
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
+	"github.com/coder/quartz"
+	"github.com/coder/serpent"
+
 	"github.com/coder/coder/v2/coderd/coderdtest"
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbauthz"
@@ -48,8 +51,6 @@ import (
 	"github.com/coder/coder/v2/coderd/util/syncmap"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/testutil"
-	"github.com/coder/quartz"
-	"github.com/coder/serpent"
 )
 
 // updateGoldenFiles is a flag that can be set to update golden files.
@@ -1226,6 +1227,25 @@ func TestNotificationTemplates_Golden(t *testing.T) {
 				Labels:       map[string]string{},
 			},
 		},
+		{
+			name: "TemplateWorkspaceResourceReplaced",
+			id:   notifications.TemplateWorkspaceResourceReplaced,
+			payload: types.MessagePayload{
+				UserName:     "Bobby",
+				UserEmail:    "bobby@coder.com",
+				UserUsername: "bobby",
+				Labels: map[string]string{
+					"workspace":           "my-workspace",
+					"workspace_build_num": "2",
+					"claimant":            "prebuilds-claimer",
+				},
+				Data: map[string]any{
+					"replacements": map[string]string{
+						"docker_container[0]": "env, hostname",
+					},
+				},
+			},
+		},
 	}
 
 	// We must have a test case for every notification_template. This is enforced below:

diff --git a/...ifications/testdata/rendered-templates/smtp/TemplateWorkspaceResourceReplaced.html.golden b/...ifications/testdata/rendered-templates/smtp/TemplateWorkspaceResourceReplaced.html.golden
@@ -0,0 +1,112 @@
+From: system@coder.com
+To: bobby@coder.com
+Subject: There might be a problem with a recently claimed prebuilt workspace
+Message-Id: 02ee4935-73be-4fa1-a290-ff9999026b13@blush-whale-48
+Date: Fri, 11 Oct 2024 09:03:06 +0000
+Content-Type: multipart/alternative;  boundary=bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
+MIME-Version: 1.0
+
+--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/plain; charset=UTF-8
+
+Hi Bobby,
+
+Workspace my-workspace was claimed from a prebuilt workspace by prebuilds-c=
+laimer.
+During the claim, Terraform destroyed and recreated the following resources
+because one or more immutable attributes changed:
+
+docker_container[0]  was replaced due to changes to env, hostname
+
+When Terraform must change an immutable attribute, it replaces the entire r=
+esource.
+If you=E2=80=99re using prebuilds to speed up provisioning, unexpected repl=
+acements will slow down
+workspace startup=E2=80=94even when claiming a prebuilt environment.
+
+For tips on preventing replacements and improving claim performance, see th=
+is guide (https://coder.com/docs/TODO).
+
+
+View workspace build: http://test.com/@prebuilds-claimer/my-workspace/build=
+s/2
+
+--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/html; charset=UTF-8
+
+<!doctype html>
+<html lang=3D"en">
+  <head>
+    <meta charset=3D"UTF-8" />
+    <meta name=3D"viewport" content=3D"width=3Ddevice-width, initial-scale=
+=3D1.0" />
+    <title>There might be a problem with a recently claimed prebuilt worksp=
+ace</title>
+  </head>
+  <body style=3D"margin: 0; padding: 0; font-family: -apple-system, system-=
+ui, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 'Ubuntu', 'Cantarel=
+l', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', sans-serif; color: #020617=
+; background: #f8fafc;">
+    <div style=3D"max-width: 600px; margin: 20px auto; padding: 60px; borde=
+r: 1px solid #e2e8f0; border-radius: 8px; background-color: #fff; text-alig=
+n: left; font-size: 14px; line-height: 1.5;">
+      <div style=3D"text-align: center;">
+        <img src=3D"https://coder.com/coder-logo-horizontal.png" alt=3D"Cod=
+er Logo" style=3D"height: 40px;" />
+      </div>
+      <h1 style=3D"text-align: center; font-size: 24px; font-weight: 400; m=
+argin: 8px 0 32px; line-height: 1.5;">
+        There might be a problem with a recently claimed prebuilt workspace
+      </h1>
+      <div style=3D"line-height: 1.5;">
+        <p>Hi Bobby,</p>
+        <p>Workspace <strong>my-workspace</strong> was claimed from a prebu=
+ilt workspace by <strong>prebuilds-claimer</strong>.<br>
+During the claim, Terraform destroyed and recreated the following resources=
+<br>
+because one or more immutable attributes changed:</p>
+
+<ul>
+<li>_docker<em>container[0]</em>  was replaced due to changes to <em>env, h=
+ostname</em><br>
+</li>
+</ul>
+
+<p>When Terraform must change an immutable attribute, it replaces the entir=
+e resource.<br>
+If you=E2=80=99re using prebuilds to speed up provisioning, unexpected repl=
+acements will slow down<br>
+workspace startup=E2=80=94even when claiming a prebuilt environment.</p>
+
+<p>For tips on preventing replacements and improving claim performance, see=
+ <a href=3D"https://coder.com/docs/TODO">this guide</a>.</p>
+      </div>
+      <div style=3D"text-align: center; margin-top: 32px;">
+       =20
+        <a href=3D"http://test.com/@prebuilds-claimer/my-workspace/builds/2=
+" style=3D"display: inline-block; padding: 13px 24px; background-color: #02=
+0617; color: #f8fafc; text-decoration: none; border-radius: 8px; margin: 0 =
+4px;">
+          View workspace build
+        </a>
+       =20
+      </div>
+      <div style=3D"border-top: 1px solid #e2e8f0; color: #475569; font-siz=
+e: 12px; margin-top: 64px; padding-top: 24px; line-height: 1.6;">
+        <p>&copy;&nbsp;2024&nbsp;Coder. All rights reserved&nbsp;-&nbsp;<a =
+href=3D"http://test.com" style=3D"color: #2563eb; text-decoration: none;">h=
+ttp://test.com</a></p>
+        <p><a href=3D"http://test.com/settings/notifications" style=3D"colo=
+r: #2563eb; text-decoration: none;">Click here to manage your notification =
+settings</a></p>
+        <p><a href=3D"http://test.com/settings/notifications?disabled=3D89d=
+9745a-816e-4695-a17f-3d0a229e2b8d" style=3D"color: #2563eb; text-decoration=
+: none;">Stop receiving emails like this</a></p>
+      </div>
+    </div>
+  </body>
+</html>
+
+--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4--
diff --git a/...cations/testdata/rendered-templates/webhook/TemplateWorkspaceResourceReplaced.json.golden b/...cations/testdata/rendered-templates/webhook/TemplateWorkspaceResourceReplaced.json.golden
@@ -0,0 +1,34 @@
+{
+  "_version": "1.1",
+  "msg_id": "00000000-0000-0000-0000-000000000000",
+  "payload": {
+    "_version": "1.2",
+    "notification_name": "Prebuilt Workspace Resource Replaced",
+    "notification_template_id": "00000000-0000-0000-0000-000000000000",
+    "user_id": "00000000-0000-0000-0000-000000000000",
+    "user_email": "bobby@coder.com",
+    "user_name": "Bobby",
+    "user_username": "bobby",
+    "actions": [
+      {
+        "label": "View workspace build",
+        "url": "http://test.com/@prebuilds-claimer/my-workspace/builds/2"
+      }
+    ],
+    "labels": {
+      "claimant": "prebuilds-claimer",
+      "workspace": "my-workspace",
+      "workspace_build_num": "2"
+    },
+    "data": {
+      "replacements": {
+        "docker_container[0]": "env, hostname"
+      }
+    },
+    "targets": null
+  },
+  "title": "There might be a problem with a recently claimed prebuilt workspace",
+  "title_markdown": "There might be a problem with a recently claimed prebuilt workspace",
+  "body": "Workspace my-workspace was claimed from a prebuilt workspace by prebuilds-claimer.\nDuring the claim, Terraform destroyed and recreated the following resources\nbecause one or more immutable attributes changed:\n\ndocker_container[0]  was replaced due to changes to env, hostname\n\nWhen Terraform must change an immutable attribute, it replaces the entire resource.\nIf you’re using prebuilds to speed up provisioning, unexpected replacements will slow down\nworkspace startup—even when claiming a prebuilt environment.\n\nFor tips on preventing replacements and improving claim performance, see this guide (https://coder.com/docs/TODO).",
+  "body_markdown": "\nWorkspace **my-workspace** was claimed from a prebuilt workspace by **prebuilds-claimer**.\nDuring the claim, Terraform destroyed and recreated the following resources\nbecause one or more immutable attributes changed:\n\n- _docker_container[0]_  was replaced due to changes to _env, hostname_\n\n\nWhen Terraform must change an immutable attribute, it replaces the entire resource.\nIf you’re using prebuilds to speed up provisioning, unexpected replacements will slow down\nworkspace startup—even when claiming a prebuilt environment.\n\nFor tips on preventing replacements and improving claim performance, see [this guide](https://coder.com/docs/TODO).\n"
+}
diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go
@@ -617,6 +617,11 @@ func (s *server) acquireProtoJob(ctx context.Context, job database.ProvisionerJo
 			}
 		}
 
+		var prebuildClaimedForUserID string
+		if input.PrebuildClaimedByUser != uuid.Nil {
+			prebuildClaimedForUserID = input.PrebuildClaimedByUser.String()
+		}
+
 		protoJob.Type = &proto.AcquiredJob_WorkspaceBuild_{
 			WorkspaceBuild: &proto.AcquiredJob_WorkspaceBuild{
 				WorkspaceBuildId:      workspaceBuild.ID.String(),
@@ -646,6 +651,7 @@ func (s *server) acquireProtoJob(ctx context.Context, job database.ProvisionerJo
 					WorkspaceOwnerLoginType:       string(owner.LoginType),
 					WorkspaceOwnerRbacRoles:       ownerRbacRoles,
 					IsPrebuild:                    input.IsPrebuild,
+					PrebuildClaimForUserId:        prebuildClaimedForUserID,
 				},
 				LogLevel: input.LogLevel,
 			},
@@ -1722,6 +1728,10 @@ func (s *server) CompleteJob(ctx context.Context, completed *proto.CompletedJob)
 			})
 		}
 
+		if resourceReplacements := completed.GetWorkspaceBuild().GetResourceReplacements(); len(resourceReplacements) > 0 {
+			s.notifyPrebuiltWorkspaceResourceReplacement(ctx, workspace, workspaceBuild, input.PrebuildClaimedByUser, resourceReplacements)
+		}
+
 		msg, err := json.Marshal(wspubsub.WorkspaceEvent{
 			Kind:        wspubsub.WorkspaceEventKindStateChange,
 			WorkspaceID: workspace.ID,
@@ -1830,6 +1840,67 @@ func (s *server) notifyWorkspaceDeleted(ctx context.Context, workspace database.
 	}
 }
 
+func (s *server) notifyPrebuiltWorkspaceResourceReplacement(ctx context.Context, workspace database.Workspace, build database.WorkspaceBuild, claimantID uuid.UUID, replacements []*sdkproto.ResourceReplacement) {
+	if claimantID == uuid.Nil {
+		// This is not a prebuild claim.
+		return
+	}
+
+	claimant, err := s.Database.GetUserByID(ctx, claimantID)
+	if err != nil {
+		s.Logger.Warn(ctx, "failed to find claimant by ID, cannot send prebuilt workspace resource replacement notification",
+			slog.F("claimant_id", claimantID.String()), slog.Error(err))
+		return
+	}
+
+	templateAdmins, err := findTemplateAdmins(ctx, s.Database)
+	if err != nil {
+		s.Logger.Warn(ctx, "failed to find template admins, cannot send prebuilt workspace resource replacement notification",
+			slog.F("claimant_id", claimantID.String()), slog.Error(err))
+		return
+	}
+
+	repls := make(map[string]string, len(replacements))
+	for _, repl := range replacements {
+		repls[repl.GetResource()] = strings.Join(repl.GetPaths(), ", ")
+	}
+
+	for _, templateAdmin := range templateAdmins {
+		if _, err := s.NotificationsEnqueuer.EnqueueWithData(ctx, templateAdmin.ID, notifications.TemplateWorkspaceResourceReplaced,
+			map[string]string{
+				"workspace":           workspace.Name,
+				"workspace_build_num": fmt.Sprintf("%d", build.BuildNumber),
+				"claimant":            claimant.Username,
+			},
+			map[string]any{
+				"replacements": repls,
+			}, "provisionerdserver",
+			// Associate this notification with all the related entities.
+			workspace.ID, workspace.OwnerID, workspace.TemplateID, workspace.OrganizationID,
+		); err != nil {
+			s.Logger.Warn(ctx, "failed to notify of prebuilt workspace resource replacement", slog.Error(err))
+			break
+		}
+	}
+}
+
+// findTemplateAdmins fetches all users with template admin permission, including owners.
+func findTemplateAdmins(ctx context.Context, store database.Store) ([]database.GetUsersRow, error) {
+	owners, err := store.GetUsers(ctx, database.GetUsersParams{
+		RbacRole: []string{codersdk.RoleOwner},
+	})
+	if err != nil {
+		return nil, xerrors.Errorf("get owners: %w", err)
+	}
+	templateAdmins, err := store.GetUsers(ctx, database.GetUsersParams{
+		RbacRole: []string{codersdk.RoleTemplateAdmin},
+	})
+	if err != nil {
+		return nil, xerrors.Errorf("get template admins: %w", err)
+	}
+	return append(owners, templateAdmins...), nil
+}
+
 func (s *server) startTrace(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
 	return s.Tracer.Start(ctx, name, append(opts, trace.WithAttributes(
 		semconv.ServiceNameKey.String("coderd.provisionerd"),

diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go
@@ -1166,5 +1166,5 @@ func (api *API) setupPrebuilds(featureEnabled bool) (agplprebuilds.Reconciliatio
 
 	reconciler := prebuilds.NewStoreReconciler(api.Database, api.Pubsub, api.DeploymentValues.Prebuilds,
 		api.Logger.Named("prebuilds"), quartz.NewReal())
-	return reconciler, prebuilds.EnterpriseClaimer{}
+	return reconciler, prebuilds.NewEnterpriseClaimer(api.Database)
 }
diff --git a/enterprise/coderd/coderd_test.go b/enterprise/coderd/coderd_test.go
@@ -331,7 +331,7 @@ func TestEntitlements_Prebuilds(t *testing.T) {
 
 			if tc.expectedEnabled {
 				require.IsType(t, &prebuilds.StoreReconciler{}, *reconciler)
-				require.IsType(t, prebuilds.EnterpriseClaimer{}, *claimer)
+				require.IsType(t, &prebuilds.EnterpriseClaimer{}, *claimer)
 			} else {
 				require.Equal(t, &agplprebuilds.DefaultReconciler, reconciler)
 				require.Equal(t, &agplprebuilds.DefaultClaimer, claimer)
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		DELETE FROM notification_templates WHERE id = '89d9745a-816e-4695-a17f-3d0a229e2b8d';