diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 94e60db47cb30..98e701957a92e 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -487,6 +487,16 @@ var ( rbac.ResourceFile.Type: { policy.ActionRead, }, + // Needs to be able to add the prebuilds system user to the "prebuilds" group in each organization that needs prebuilt workspaces + // so that prebuilt workspaces can be scheduled and owned in those organizations. + rbac.ResourceGroup.Type: { + policy.ActionRead, + policy.ActionCreate, + policy.ActionUpdate, + }, + rbac.ResourceGroupMember.Type: { + policy.ActionRead, + }, }), }, }), diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 3a41cf63c1630..5f993e0095b05 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -6609,16 +6609,19 @@ WHERE organization_id = $1 ELSE true END + -- Filter by system type + AND CASE WHEN $2::bool THEN TRUE ELSE is_system = false END ORDER BY -- Deterministic and consistent ordering of all users. This is to ensure consistent pagination. - LOWER(username) ASC OFFSET $2 + LOWER(username) ASC OFFSET $3 LIMIT -- A null limit means "no limit", so 0 means return all - NULLIF($3 :: int, 0) + NULLIF($4 :: int, 0) ` type PaginatedOrganizationMembersParams struct { OrganizationID uuid.UUID `db:"organization_id" json:"organization_id"` + IncludeSystem bool `db:"include_system" json:"include_system"` OffsetOpt int32 `db:"offset_opt" json:"offset_opt"` LimitOpt int32 `db:"limit_opt" json:"limit_opt"` } @@ -6634,7 +6637,12 @@ type PaginatedOrganizationMembersRow struct { } func (q *sqlQuerier) PaginatedOrganizationMembers(ctx context.Context, arg PaginatedOrganizationMembersParams) ([]PaginatedOrganizationMembersRow, error) { - rows, err := q.db.QueryContext(ctx, paginatedOrganizationMembers, arg.OrganizationID, arg.OffsetOpt, arg.LimitOpt) + rows, err := q.db.QueryContext(ctx, paginatedOrganizationMembers, + arg.OrganizationID, + arg.IncludeSystem, + arg.OffsetOpt, + arg.LimitOpt, + ) if err != nil { return nil, err } diff --git a/coderd/database/queries/organizationmembers.sql b/coderd/database/queries/organizationmembers.sql index 9d570bc1c49ee..1c0af011776e3 100644 --- a/coderd/database/queries/organizationmembers.sql +++ b/coderd/database/queries/organizationmembers.sql @@ -89,6 +89,8 @@ WHERE organization_id = @organization_id ELSE true END + -- Filter by system type + AND CASE WHEN @include_system::bool THEN TRUE ELSE is_system = false END ORDER BY -- Deterministic and consistent ordering of all users. This is to ensure consistent pagination. LOWER(username) ASC OFFSET @offset_opt diff --git a/coderd/members.go b/coderd/members.go index 0bd5bb1fbc8bd..371b58015b83b 100644 --- a/coderd/members.go +++ b/coderd/members.go @@ -203,6 +203,7 @@ func (api *API) paginatedMembers(rw http.ResponseWriter, r *http.Request) { paginatedMemberRows, err := api.Database.PaginatedOrganizationMembers(ctx, database.PaginatedOrganizationMembersParams{ OrganizationID: organization.ID, + IncludeSystem: false, // #nosec G115 - Pagination limits are small and fit in int32 LimitOpt: int32(paginationParams.Limit), // #nosec G115 - Pagination offsets are small and fit in int32 diff --git a/docs/admin/templates/extending-templates/prebuilt-workspaces.md b/docs/admin/templates/extending-templates/prebuilt-workspaces.md index 739e13d9130e5..bf80ca479254a 100644 --- a/docs/admin/templates/extending-templates/prebuilt-workspaces.md +++ b/docs/admin/templates/extending-templates/prebuilt-workspaces.md @@ -233,12 +233,18 @@ The system always maintains the desired number of prebuilt workspaces for the ac ### Managing resource quotas -Prebuilt workspaces can be used in conjunction with [resource quotas](../../users/quotas.md). +To help prevent unexpected infrastructure costs, prebuilt workspaces can be used in conjunction with [resource quotas](../../users/quotas.md). Because unclaimed prebuilt workspaces are owned by the `prebuilds` user, you can: 1. Configure quotas for any group that includes this user. 1. Set appropriate limits to balance prebuilt workspace availability with resource constraints. +When prebuilt workspaces are configured for an organization, Coder creates a "prebuilds" group in that organization and adds the prebuilds user to it. This group has a default quota allowance of 0, which you should adjust based on your needs: + +- **Set a quota allowance** on the "prebuilds" group to control how many prebuilt workspaces can be provisioned +- **Monitor usage** to ensure the quota is appropriate for your desired number of prebuilt instances +- **Adjust as needed** based on your template costs and desired prebuilt workspace pool size + If a quota is exceeded, the prebuilt workspace will fail provisioning the same way other workspaces do. ### Template configuration best practices diff --git a/enterprise/coderd/prebuilds/membership.go b/enterprise/coderd/prebuilds/membership.go index 079711bcbcc49..f843d33f7f106 100644 --- a/enterprise/coderd/prebuilds/membership.go +++ b/enterprise/coderd/prebuilds/membership.go @@ -12,6 +12,11 @@ import ( "github.com/coder/quartz" ) +const ( + PrebuiltWorkspacesGroupName = "coderprebuiltworkspaces" + PrebuiltWorkspacesGroupDisplayName = "Prebuilt Workspaces" +) + // StoreMembershipReconciler encapsulates the responsibility of ensuring that the prebuilds system user is a member of all // organizations for which prebuilt workspaces are requested. This is necessary because our data model requires that such // prebuilt workspaces belong to a member of the organization of their eventual claimant. @@ -27,11 +32,16 @@ func NewStoreMembershipReconciler(store database.Store, clock quartz.Clock) Stor } } -// ReconcileAll compares the current membership of a user to the membership required in order to create prebuilt workspaces. -// If the user in question is not yet a member of an organization that needs prebuilt workspaces, ReconcileAll will create -// the membership required. +// ReconcileAll compares the current organization and group memberships of a user to the memberships required +// in order to create prebuilt workspaces. If the user in question is not yet a member of an organization that +// needs prebuilt workspaces, ReconcileAll will create the membership required. +// +// To facilitate quota management, ReconcileAll will ensure: +// * the existence of a group (defined by PrebuiltWorkspacesGroupName) in each organization that needs prebuilt workspaces +// * that the prebuilds system user belongs to the group in each organization that needs prebuilt workspaces +// * that the group has a quota of 0 by default, which users can adjust based on their needs. // -// This method does not have an opinion on transaction or lock management. These responsibilities are left to the caller. +// ReconcileAll does not have an opinion on transaction or lock management. These responsibilities are left to the caller. func (s StoreMembershipReconciler) ReconcileAll(ctx context.Context, userID uuid.UUID, presets []database.GetTemplatePresetsWithPrebuildsRow) error { organizationMemberships, err := s.store.GetOrganizationsByUserID(ctx, database.GetOrganizationsByUserIDParams{ UserID: userID, @@ -44,37 +54,80 @@ func (s StoreMembershipReconciler) ReconcileAll(ctx context.Context, userID uuid return xerrors.Errorf("determine prebuild organization membership: %w", err) } - systemUserMemberships := make(map[uuid.UUID]struct{}, 0) + orgMemberships := make(map[uuid.UUID]struct{}, 0) defaultOrg, err := s.store.GetDefaultOrganization(ctx) if err != nil { return xerrors.Errorf("get default organization: %w", err) } - systemUserMemberships[defaultOrg.ID] = struct{}{} + orgMemberships[defaultOrg.ID] = struct{}{} for _, o := range organizationMemberships { - systemUserMemberships[o.ID] = struct{}{} + orgMemberships[o.ID] = struct{}{} } var membershipInsertionErrors error for _, preset := range presets { - _, alreadyMember := systemUserMemberships[preset.OrganizationID] - if alreadyMember { - continue + _, alreadyOrgMember := orgMemberships[preset.OrganizationID] + if !alreadyOrgMember { + // Add the organization to our list of memberships regardless of potential failure below + // to avoid a retry that will probably be doomed anyway. + orgMemberships[preset.OrganizationID] = struct{}{} + + // Insert the missing membership + _, err = s.store.InsertOrganizationMember(ctx, database.InsertOrganizationMemberParams{ + OrganizationID: preset.OrganizationID, + UserID: userID, + CreatedAt: s.clock.Now(), + UpdatedAt: s.clock.Now(), + Roles: []string{}, + }) + if err != nil { + membershipInsertionErrors = errors.Join(membershipInsertionErrors, xerrors.Errorf("insert membership for prebuilt workspaces: %w", err)) + continue + } } - // Add the organization to our list of memberships regardless of potential failure below - // to avoid a retry that will probably be doomed anyway. - systemUserMemberships[preset.OrganizationID] = struct{}{} - // Insert the missing membership - _, err = s.store.InsertOrganizationMember(ctx, database.InsertOrganizationMemberParams{ + // determine whether the org already has a prebuilds group + prebuildsGroupExists := true + prebuildsGroup, err := s.store.GetGroupByOrgAndName(ctx, database.GetGroupByOrgAndNameParams{ OrganizationID: preset.OrganizationID, - UserID: userID, - CreatedAt: s.clock.Now(), - UpdatedAt: s.clock.Now(), - Roles: []string{}, + Name: PrebuiltWorkspacesGroupName, + }) + if err != nil { + if !xerrors.Is(err, sql.ErrNoRows) { + membershipInsertionErrors = errors.Join(membershipInsertionErrors, xerrors.Errorf("get prebuilds group: %w", err)) + continue + } + prebuildsGroupExists = false + } + + // if the prebuilds group does not exist, create it + if !prebuildsGroupExists { + // create a "prebuilds" group in the organization and add the system user to it + // this group will have a quota of 0 by default, which users can adjust based on their needs + prebuildsGroup, err = s.store.InsertGroup(ctx, database.InsertGroupParams{ + ID: uuid.New(), + Name: PrebuiltWorkspacesGroupName, + DisplayName: PrebuiltWorkspacesGroupDisplayName, + OrganizationID: preset.OrganizationID, + AvatarURL: "", + QuotaAllowance: 0, // Default quota of 0, users should set this based on their needs + }) + if err != nil { + membershipInsertionErrors = errors.Join(membershipInsertionErrors, xerrors.Errorf("create prebuilds group: %w", err)) + continue + } + } + + // add the system user to the prebuilds group + err = s.store.InsertGroupMember(ctx, database.InsertGroupMemberParams{ + GroupID: prebuildsGroup.ID, + UserID: userID, }) if err != nil { - membershipInsertionErrors = errors.Join(membershipInsertionErrors, xerrors.Errorf("insert membership for prebuilt workspaces: %w", err)) - continue + // ignore unique violation errors as the user might already be in the group + if !database.IsUniqueViolation(err) { + membershipInsertionErrors = errors.Join(membershipInsertionErrors, xerrors.Errorf("add system user to prebuilds group: %w", err)) + } } } return membershipInsertionErrors diff --git a/enterprise/coderd/prebuilds/membership_test.go b/enterprise/coderd/prebuilds/membership_test.go index 82d2abf92a4d8..80e2f907349ae 100644 --- a/enterprise/coderd/prebuilds/membership_test.go +++ b/enterprise/coderd/prebuilds/membership_test.go @@ -1,18 +1,23 @@ package prebuilds_test import ( - "context" + "database/sql" + "errors" "testing" "github.com/google/uuid" "github.com/stretchr/testify/require" + "tailscale.com/types/ptr" "github.com/coder/quartz" + "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbauthz" "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/enterprise/coderd/prebuilds" + "github.com/coder/coder/v2/testutil" ) // TestReconcileAll verifies that StoreMembershipReconciler correctly updates membership @@ -20,7 +25,6 @@ import ( func TestReconcileAll(t *testing.T) { t.Parallel() - ctx := context.Background() clock := quartz.NewMock(t) // Helper to build a minimal Preset row belonging to a given org. @@ -32,87 +36,171 @@ func TestReconcileAll(t *testing.T) { } tests := []struct { - name string - includePreset bool - preExistingMembership bool + name string + includePreset []bool + preExistingOrgMembership []bool + preExistingGroup []bool + preExistingGroupMembership []bool + // Expected outcomes + expectOrgMembershipExists *bool + expectGroupExists *bool + expectUserInGroup *bool }{ - // The StoreMembershipReconciler acts based on the provided agplprebuilds.GlobalSnapshot. - // These test cases must therefore trust any valid snapshot, so the only relevant functional test cases are: - - // No presets to act on and the prebuilds user does not belong to any organizations. - // Reconciliation should be a no-op - {name: "no presets, no memberships", includePreset: false, preExistingMembership: false}, - // If we have a preset that requires prebuilds, but the prebuilds user is not a member of - // that organization, then we should add the membership. - {name: "preset, but no membership", includePreset: true, preExistingMembership: false}, - // If the prebuilds system user is already a member of the organization to which a preset belongs, - // then reconciliation should be a no-op: - {name: "preset, but already a member", includePreset: true, preExistingMembership: true}, - // If the prebuilds system user is a member of an organization that doesn't have need any prebuilds, - // then it must have required prebuilds in the past. The membership is not currently necessary, but - // the reconciler won't remove it, because there's little cost to keeping it and prebuilds might be - // enabled again. - {name: "member, but no presets", includePreset: false, preExistingMembership: true}, + { + name: "if there are no presets, membership reconciliation is a no-op", + includePreset: []bool{false}, + preExistingOrgMembership: []bool{true, false}, + preExistingGroup: []bool{true, false}, + preExistingGroupMembership: []bool{true, false}, + expectOrgMembershipExists: ptr.To(false), + expectGroupExists: ptr.To(false), + }, + { + name: "if there is a preset, then we should enforce org and group membership in all cases", + includePreset: []bool{true}, + preExistingOrgMembership: []bool{true, false}, + preExistingGroup: []bool{true, false}, + preExistingGroupMembership: []bool{true, false}, + expectOrgMembershipExists: ptr.To(true), + expectGroupExists: ptr.To(true), + expectUserInGroup: ptr.To(true), + }, } for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - db, _ := dbtestutil.NewDB(t) - - defaultOrg, err := db.GetDefaultOrganization(ctx) - require.NoError(t, err) - - // introduce an unrelated organization to ensure that the membership reconciler don't interfere with it. - unrelatedOrg := dbgen.Organization(t, db, database.Organization{}) - targetOrg := dbgen.Organization(t, db, database.Organization{}) - - if !dbtestutil.WillUsePostgres() { - // dbmem doesn't ensure membership to the default organization - dbgen.OrganizationMember(t, db, database.OrganizationMember{ - OrganizationID: defaultOrg.ID, - UserID: database.PrebuildsSystemUserID, - }) - } - - dbgen.OrganizationMember(t, db, database.OrganizationMember{OrganizationID: unrelatedOrg.ID, UserID: database.PrebuildsSystemUserID}) - if tc.preExistingMembership { - // System user already a member of both orgs. - dbgen.OrganizationMember(t, db, database.OrganizationMember{OrganizationID: targetOrg.ID, UserID: database.PrebuildsSystemUserID}) + tc := tc + for _, includePreset := range tc.includePreset { + includePreset := includePreset + for _, preExistingOrgMembership := range tc.preExistingOrgMembership { + preExistingOrgMembership := preExistingOrgMembership + for _, preExistingGroup := range tc.preExistingGroup { + preExistingGroup := preExistingGroup + for _, preExistingGroupMembership := range tc.preExistingGroupMembership { + preExistingGroupMembership := preExistingGroupMembership + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + // nolint:gocritic // Reconciliation happens as prebuilds system user, not a human user. + ctx := dbauthz.AsPrebuildsOrchestrator(testutil.Context(t, testutil.WaitLong)) + _, db := coderdtest.NewWithDatabase(t, nil) + + defaultOrg, err := db.GetDefaultOrganization(ctx) + require.NoError(t, err) + + // introduce an unrelated organization to ensure that the membership reconciler doesn't interfere with it. + unrelatedOrg := dbgen.Organization(t, db, database.Organization{}) + targetOrg := dbgen.Organization(t, db, database.Organization{}) + + if !dbtestutil.WillUsePostgres() { + // dbmem doesn't ensure membership to the default organization + dbgen.OrganizationMember(t, db, database.OrganizationMember{ + OrganizationID: defaultOrg.ID, + UserID: database.PrebuildsSystemUserID, + }) + } + + // Ensure membership to unrelated org. + dbgen.OrganizationMember(t, db, database.OrganizationMember{OrganizationID: unrelatedOrg.ID, UserID: database.PrebuildsSystemUserID}) + + if preExistingOrgMembership { + // System user already a member of both orgs. + dbgen.OrganizationMember(t, db, database.OrganizationMember{OrganizationID: targetOrg.ID, UserID: database.PrebuildsSystemUserID}) + } + + // Create pre-existing prebuilds group if required by test case + var prebuildsGroup database.Group + if preExistingGroup { + prebuildsGroup = dbgen.Group(t, db, database.Group{ + Name: prebuilds.PrebuiltWorkspacesGroupName, + DisplayName: prebuilds.PrebuiltWorkspacesGroupDisplayName, + OrganizationID: targetOrg.ID, + QuotaAllowance: 0, + }) + + // Add the system user to the group if preExistingGroupMembership is true + if preExistingGroupMembership { + dbgen.GroupMember(t, db, database.GroupMemberTable{ + GroupID: prebuildsGroup.ID, + UserID: database.PrebuildsSystemUserID, + }) + } + } + + presets := []database.GetTemplatePresetsWithPrebuildsRow{newPresetRow(unrelatedOrg.ID)} + if includePreset { + presets = append(presets, newPresetRow(targetOrg.ID)) + } + + // Verify memberships before reconciliation. + preReconcileMemberships, err := db.GetOrganizationsByUserID(ctx, database.GetOrganizationsByUserIDParams{ + UserID: database.PrebuildsSystemUserID, + }) + require.NoError(t, err) + expectedMembershipsBefore := []uuid.UUID{defaultOrg.ID, unrelatedOrg.ID} + if preExistingOrgMembership { + expectedMembershipsBefore = append(expectedMembershipsBefore, targetOrg.ID) + } + require.ElementsMatch(t, expectedMembershipsBefore, extractOrgIDs(preReconcileMemberships)) + + // Reconcile + reconciler := prebuilds.NewStoreMembershipReconciler(db, clock) + require.NoError(t, reconciler.ReconcileAll(ctx, database.PrebuildsSystemUserID, presets)) + + // Verify memberships after reconciliation. + postReconcileMemberships, err := db.GetOrganizationsByUserID(ctx, database.GetOrganizationsByUserIDParams{ + UserID: database.PrebuildsSystemUserID, + }) + require.NoError(t, err) + expectedMembershipsAfter := expectedMembershipsBefore + if !preExistingOrgMembership && tc.expectOrgMembershipExists != nil && *tc.expectOrgMembershipExists { + expectedMembershipsAfter = append(expectedMembershipsAfter, targetOrg.ID) + } + require.ElementsMatch(t, expectedMembershipsAfter, extractOrgIDs(postReconcileMemberships)) + + // Verify prebuilds group behavior based on expected outcomes + prebuildsGroup, err = db.GetGroupByOrgAndName(ctx, database.GetGroupByOrgAndNameParams{ + OrganizationID: targetOrg.ID, + Name: prebuilds.PrebuiltWorkspacesGroupName, + }) + if tc.expectGroupExists != nil && *tc.expectGroupExists { + require.NoError(t, err) + require.Equal(t, prebuilds.PrebuiltWorkspacesGroupName, prebuildsGroup.Name) + require.Equal(t, prebuilds.PrebuiltWorkspacesGroupDisplayName, prebuildsGroup.DisplayName) + require.Equal(t, int32(0), prebuildsGroup.QuotaAllowance) // Default quota should be 0 + + if tc.expectUserInGroup != nil && *tc.expectUserInGroup { + // Check that the system user is a member of the prebuilds group + groupMembers, err := db.GetGroupMembersByGroupID(ctx, database.GetGroupMembersByGroupIDParams{ + GroupID: prebuildsGroup.ID, + IncludeSystem: true, + }) + require.NoError(t, err) + require.Len(t, groupMembers, 1) + require.Equal(t, database.PrebuildsSystemUserID, groupMembers[0].UserID) + } + + // If no preset exists, then we do not enforce group membership: + if tc.expectUserInGroup != nil && !*tc.expectUserInGroup { + // Check that the system user is NOT a member of the prebuilds group + groupMembers, err := db.GetGroupMembersByGroupID(ctx, database.GetGroupMembersByGroupIDParams{ + GroupID: prebuildsGroup.ID, + IncludeSystem: true, + }) + require.NoError(t, err) + require.Len(t, groupMembers, 0) + } + } + + if !preExistingGroup && tc.expectGroupExists != nil && !*tc.expectGroupExists { + // Verify that no prebuilds group exists + require.Error(t, err) + require.True(t, errors.Is(err, sql.ErrNoRows)) + } + }) + } + } } - - presets := []database.GetTemplatePresetsWithPrebuildsRow{newPresetRow(unrelatedOrg.ID)} - if tc.includePreset { - presets = append(presets, newPresetRow(targetOrg.ID)) - } - - // Verify memberships before reconciliation. - preReconcileMemberships, err := db.GetOrganizationsByUserID(ctx, database.GetOrganizationsByUserIDParams{ - UserID: database.PrebuildsSystemUserID, - }) - require.NoError(t, err) - expectedMembershipsBefore := []uuid.UUID{defaultOrg.ID, unrelatedOrg.ID} - if tc.preExistingMembership { - expectedMembershipsBefore = append(expectedMembershipsBefore, targetOrg.ID) - } - require.ElementsMatch(t, expectedMembershipsBefore, extractOrgIDs(preReconcileMemberships)) - - // Reconcile - reconciler := prebuilds.NewStoreMembershipReconciler(db, clock) - require.NoError(t, reconciler.ReconcileAll(ctx, database.PrebuildsSystemUserID, presets)) - - // Verify memberships after reconciliation. - postReconcileMemberships, err := db.GetOrganizationsByUserID(ctx, database.GetOrganizationsByUserIDParams{ - UserID: database.PrebuildsSystemUserID, - }) - require.NoError(t, err) - expectedMembershipsAfter := expectedMembershipsBefore - if !tc.preExistingMembership && tc.includePreset { - expectedMembershipsAfter = append(expectedMembershipsAfter, targetOrg.ID) - } - require.ElementsMatch(t, expectedMembershipsAfter, extractOrgIDs(postReconcileMemberships)) - }) + } } } diff --git a/enterprise/coderd/prebuilds/reconcile_test.go b/enterprise/coderd/prebuilds/reconcile_test.go index 8d2a81e1ade83..413d61ddbbc6a 100644 --- a/enterprise/coderd/prebuilds/reconcile_test.go +++ b/enterprise/coderd/prebuilds/reconcile_test.go @@ -3,7 +3,6 @@ package prebuilds_test import ( "context" "database/sql" - "fmt" "sort" "sync" "sync/atomic" @@ -46,10 +45,6 @@ func TestNoReconciliationActionsIfNoPresets(t *testing.T) { // Scenario: No reconciliation actions are taken if there are no presets t.Parallel() - if !dbtestutil.WillUsePostgres() { - t.Skip("dbmem times out on nesting transactions, postgres ignores the inner ones") - } - clock := quartz.NewMock(t) ctx := testutil.Context(t, testutil.WaitLong) db, ps := dbtestutil.NewDB(t) @@ -92,10 +87,6 @@ func TestNoReconciliationActionsIfNoPrebuilds(t *testing.T) { // Scenario: No reconciliation actions are taken if there are no prebuilds t.Parallel() - if !dbtestutil.WillUsePostgres() { - t.Skip("dbmem times out on nesting transactions, postgres ignores the inner ones") - } - clock := quartz.NewMock(t) ctx := testutil.Context(t, testutil.WaitLong) db, ps := dbtestutil.NewDB(t) @@ -149,21 +140,7 @@ func TestNoReconciliationActionsIfNoPrebuilds(t *testing.T) { func TestPrebuildReconciliation(t *testing.T) { t.Parallel() - if !dbtestutil.WillUsePostgres() { - t.Skip("This test requires postgres") - } - - type testCase struct { - name string - prebuildLatestTransitions []database.WorkspaceTransition - prebuildJobStatuses []database.ProvisionerJobStatus - templateVersionActive []bool - templateDeleted []bool - shouldCreateNewPrebuild *bool - shouldDeleteOldPrebuild *bool - } - - testCases := []testCase{ + testScenarios := []testScenario{ { name: "never create prebuilds for inactive template versions", prebuildLatestTransitions: allTransitions, @@ -181,8 +158,8 @@ func TestPrebuildReconciliation(t *testing.T) { database.ProvisionerJobStatusSucceeded, }, templateVersionActive: []bool{true}, - shouldCreateNewPrebuild: ptr.To(false), templateDeleted: []bool{false}, + shouldCreateNewPrebuild: ptr.To(false), }, { name: "don't create a new prebuild if one is queued to build or already building", @@ -313,119 +290,173 @@ func TestPrebuildReconciliation(t *testing.T) { templateDeleted: []bool{true}, }, } - for _, tc := range testCases { - for _, templateVersionActive := range tc.templateVersionActive { - for _, prebuildLatestTransition := range tc.prebuildLatestTransitions { - for _, prebuildJobStatus := range tc.prebuildJobStatuses { - for _, templateDeleted := range tc.templateDeleted { - for _, useBrokenPubsub := range []bool{true, false} { - t.Run(fmt.Sprintf("%s - %s - %s - pubsub_broken=%v", tc.name, prebuildLatestTransition, prebuildJobStatus, useBrokenPubsub), func(t *testing.T) { - t.Parallel() - t.Cleanup(func() { - if t.Failed() { - t.Logf("failed to run test: %s", tc.name) - t.Logf("templateVersionActive: %t", templateVersionActive) - t.Logf("prebuildLatestTransition: %s", prebuildLatestTransition) - t.Logf("prebuildJobStatus: %s", prebuildJobStatus) - } - }) - clock := quartz.NewMock(t) - ctx := testutil.Context(t, testutil.WaitShort) - cfg := codersdk.PrebuildsConfig{} - logger := slogtest.Make( - t, &slogtest.Options{IgnoreErrors: true}, - ).Leveled(slog.LevelDebug) - db, pubSub := dbtestutil.NewDB(t) - - ownerID := uuid.New() - dbgen.User(t, db, database.User{ - ID: ownerID, - }) - org, template := setupTestDBTemplate(t, db, ownerID, templateDeleted) - templateVersionID := setupTestDBTemplateVersion( - ctx, - t, - clock, - db, - pubSub, - org.ID, - ownerID, - template.ID, - ) - preset := setupTestDBPreset( - t, - db, - templateVersionID, - 1, - uuid.New().String(), - ) - prebuild, _ := setupTestDBPrebuild( - t, - clock, - db, - pubSub, - prebuildLatestTransition, - prebuildJobStatus, - org.ID, - preset, - template.ID, - templateVersionID, - ) - - setupTestDBPrebuildAntagonists(t, db, pubSub, org) - - if !templateVersionActive { - // Create a new template version and mark it as active - // This marks the template version that we care about as inactive - setupTestDBTemplateVersion(ctx, t, clock, db, pubSub, org.ID, ownerID, template.ID) - } - - if useBrokenPubsub { - pubSub = &brokenPublisher{Pubsub: pubSub} - } - cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{}) - controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr()) - - // Run the reconciliation multiple times to ensure idempotency - // 8 was arbitrary, but large enough to reasonably trust the result - for i := 1; i <= 8; i++ { - require.NoErrorf(t, controller.ReconcileAll(ctx), "failed on iteration %d", i) - - if tc.shouldCreateNewPrebuild != nil { - newPrebuildCount := 0 - workspaces, err := db.GetWorkspacesByTemplateID(ctx, template.ID) - require.NoError(t, err) - for _, workspace := range workspaces { - if workspace.ID != prebuild.ID { - newPrebuildCount++ - } - } - // This test configures a preset that desires one prebuild. - // In cases where new prebuilds should be created, there should be exactly one. - require.Equal(t, *tc.shouldCreateNewPrebuild, newPrebuildCount == 1) - } - - if tc.shouldDeleteOldPrebuild != nil { - builds, err := db.GetWorkspaceBuildsByWorkspaceID(ctx, database.GetWorkspaceBuildsByWorkspaceIDParams{ - WorkspaceID: prebuild.ID, - }) - require.NoError(t, err) - if *tc.shouldDeleteOldPrebuild { - require.Equal(t, 2, len(builds)) - require.Equal(t, database.WorkspaceTransitionDelete, builds[0].Transition) - } else { - require.Equal(t, 1, len(builds)) - require.Equal(t, prebuildLatestTransition, builds[0].Transition) - } - } - } - }) + for _, tc := range testScenarios { + testCases := tc.testCases() + for _, tc := range testCases { + tc.run(t) + } + } +} + +// testScenario is a collection of test cases that illustrate the same business rule. +// A testScenario describes a set of test properties for which the same test expecations +// hold. A testScenario may be decomposed into multiple testCase structs, which can then be run. +type testScenario struct { + name string + prebuildLatestTransitions []database.WorkspaceTransition + prebuildJobStatuses []database.ProvisionerJobStatus + templateVersionActive []bool + templateDeleted []bool + shouldCreateNewPrebuild *bool + shouldDeleteOldPrebuild *bool + expectOrgMembership *bool + expectGroupMembership *bool +} + +func (ts testScenario) testCases() []testCase { + testCases := []testCase{} + for _, templateVersionActive := range ts.templateVersionActive { + for _, prebuildLatestTransition := range ts.prebuildLatestTransitions { + for _, prebuildJobStatus := range ts.prebuildJobStatuses { + for _, templateDeleted := range ts.templateDeleted { + for _, useBrokenPubsub := range []bool{true, false} { + testCase := testCase{ + name: ts.name, + templateVersionActive: templateVersionActive, + prebuildLatestTransition: prebuildLatestTransition, + prebuildJobStatus: prebuildJobStatus, + templateDeleted: templateDeleted, + useBrokenPubsub: useBrokenPubsub, + shouldCreateNewPrebuild: ts.shouldCreateNewPrebuild, + shouldDeleteOldPrebuild: ts.shouldDeleteOldPrebuild, + expectOrgMembership: ts.expectOrgMembership, + expectGroupMembership: ts.expectGroupMembership, } + testCases = append(testCases, testCase) } } } } } + + return testCases +} + +type testCase struct { + name string + prebuildLatestTransition database.WorkspaceTransition + prebuildJobStatus database.ProvisionerJobStatus + templateVersionActive bool + templateDeleted bool + useBrokenPubsub bool + shouldCreateNewPrebuild *bool + shouldDeleteOldPrebuild *bool + expectOrgMembership *bool + expectGroupMembership *bool +} + +func (tc testCase) run(t *testing.T) { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + t.Cleanup(func() { + if t.Failed() { + t.Logf("failed to run test: %s", tc.name) + t.Logf("templateVersionActive: %t", tc.templateVersionActive) + t.Logf("prebuildLatestTransition: %s", tc.prebuildLatestTransition) + t.Logf("prebuildJobStatus: %s", tc.prebuildJobStatus) + } + }) + clock := quartz.NewMock(t) + ctx := testutil.Context(t, testutil.WaitShort) + cfg := codersdk.PrebuildsConfig{} + logger := slogtest.Make( + t, &slogtest.Options{IgnoreErrors: true}, + ).Leveled(slog.LevelDebug) + db, pubSub := dbtestutil.NewDB(t) + + ownerID := uuid.New() + dbgen.User(t, db, database.User{ + ID: ownerID, + }) + org, template := setupTestDBTemplate(t, db, ownerID, tc.templateDeleted) + templateVersionID := setupTestDBTemplateVersion( + ctx, + t, + clock, + db, + pubSub, + org.ID, + ownerID, + template.ID, + ) + preset := setupTestDBPreset( + t, + db, + templateVersionID, + 1, + uuid.New().String(), + ) + prebuild, _ := setupTestDBPrebuild( + t, + clock, + db, + pubSub, + tc.prebuildLatestTransition, + tc.prebuildJobStatus, + org.ID, + preset, + template.ID, + templateVersionID, + ) + + setupTestDBPrebuildAntagonists(t, db, pubSub, org) + + if !tc.templateVersionActive { + // Create a new template version and mark it as active + // This marks the template version that we care about as inactive + setupTestDBTemplateVersion(ctx, t, clock, db, pubSub, org.ID, ownerID, template.ID) + } + + if tc.useBrokenPubsub { + pubSub = &brokenPublisher{Pubsub: pubSub} + } + cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{}) + controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr()) + + // Run the reconciliation multiple times to ensure idempotency + // 8 was arbitrary, but large enough to reasonably trust the result + for i := 1; i <= 8; i++ { + require.NoErrorf(t, controller.ReconcileAll(ctx), "failed on iteration %d", i) + + if tc.shouldCreateNewPrebuild != nil { + newPrebuildCount := 0 + workspaces, err := db.GetWorkspacesByTemplateID(ctx, template.ID) + require.NoError(t, err) + for _, workspace := range workspaces { + if workspace.ID != prebuild.ID { + newPrebuildCount++ + } + } + // This test configures a preset that desires one prebuild. + // In cases where new prebuilds should be created, there should be exactly one. + require.Equal(t, *tc.shouldCreateNewPrebuild, newPrebuildCount == 1) + } + + if tc.shouldDeleteOldPrebuild != nil { + builds, err := db.GetWorkspaceBuildsByWorkspaceID(ctx, database.GetWorkspaceBuildsByWorkspaceIDParams{ + WorkspaceID: prebuild.ID, + }) + require.NoError(t, err) + if *tc.shouldDeleteOldPrebuild { + require.Equal(t, 2, len(builds)) + require.Equal(t, database.WorkspaceTransitionDelete, builds[0].Transition) + } else { + require.Equal(t, 1, len(builds)) + require.Equal(t, tc.prebuildLatestTransition, builds[0].Transition) + } + } + } + }) } // brokenPublisher is used to validate that Publish() calls which always fail do not affect the reconciler's behavior, @@ -446,10 +477,6 @@ func (*brokenPublisher) Publish(event string, _ []byte) error { func TestMultiplePresetsPerTemplateVersion(t *testing.T) { t.Parallel() - if !dbtestutil.WillUsePostgres() { - t.Skip("This test requires postgres") - } - prebuildLatestTransition := database.WorkspaceTransitionStart prebuildJobStatus := database.ProvisionerJobStatusRunning templateDeleted := false @@ -533,10 +560,6 @@ func TestMultiplePresetsPerTemplateVersion(t *testing.T) { func TestPrebuildScheduling(t *testing.T) { t.Parallel() - if !dbtestutil.WillUsePostgres() { - t.Skip("This test requires postgres") - } - templateDeleted := false // The test includes 2 presets, each with 2 schedules. @@ -679,10 +702,6 @@ func TestPrebuildScheduling(t *testing.T) { func TestInvalidPreset(t *testing.T) { t.Parallel() - if !dbtestutil.WillUsePostgres() { - t.Skip("This test requires postgres") - } - templateDeleted := false clock := quartz.NewMock(t) @@ -744,10 +763,6 @@ func TestInvalidPreset(t *testing.T) { func TestDeletionOfPrebuiltWorkspaceWithInvalidPreset(t *testing.T) { t.Parallel() - if !dbtestutil.WillUsePostgres() { - t.Skip("This test requires postgres") - } - templateDeleted := false clock := quartz.NewMock(t) @@ -814,10 +829,6 @@ func TestDeletionOfPrebuiltWorkspaceWithInvalidPreset(t *testing.T) { func TestSkippingHardLimitedPresets(t *testing.T) { t.Parallel() - if !dbtestutil.WillUsePostgres() { - t.Skip("This test requires postgres") - } - // Test cases verify the behavior of prebuild creation depending on configured failure limits. testCases := []struct { name string @@ -955,10 +966,6 @@ func TestSkippingHardLimitedPresets(t *testing.T) { func TestHardLimitedPresetShouldNotBlockDeletion(t *testing.T) { t.Parallel() - if !dbtestutil.WillUsePostgres() { - t.Skip("This test requires postgres") - } - // Test cases verify the behavior of prebuild creation depending on configured failure limits. testCases := []struct { name string @@ -1171,10 +1178,6 @@ func TestHardLimitedPresetShouldNotBlockDeletion(t *testing.T) { func TestRunLoop(t *testing.T) { t.Parallel() - if !dbtestutil.WillUsePostgres() { - t.Skip("This test requires postgres") - } - prebuildLatestTransition := database.WorkspaceTransitionStart prebuildJobStatus := database.ProvisionerJobStatusRunning templateDeleted := false @@ -1305,9 +1308,6 @@ func TestRunLoop(t *testing.T) { func TestFailedBuildBackoff(t *testing.T) { t.Parallel() - if !dbtestutil.WillUsePostgres() { - t.Skip("This test requires postgres") - } ctx := testutil.Context(t, testutil.WaitSuperLong) // Setup. @@ -1426,10 +1426,6 @@ func TestFailedBuildBackoff(t *testing.T) { func TestReconciliationLock(t *testing.T) { t.Parallel() - if !dbtestutil.WillUsePostgres() { - t.Skip("This test requires postgres") - } - ctx := testutil.Context(t, testutil.WaitSuperLong) logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug) db, ps := dbtestutil.NewDB(t) @@ -1470,10 +1466,6 @@ func TestReconciliationLock(t *testing.T) { func TestTrackResourceReplacement(t *testing.T) { t.Parallel() - if !dbtestutil.WillUsePostgres() { - t.Skip("This test requires postgres") - } - ctx := testutil.Context(t, testutil.WaitSuperLong) // Setup. @@ -1559,10 +1551,6 @@ func TestTrackResourceReplacement(t *testing.T) { func TestExpiredPrebuildsMultipleActions(t *testing.T) { t.Parallel() - if !dbtestutil.WillUsePostgres() { - t.Skip("This test requires postgres") - } - testCases := []struct { name string running int @@ -2268,10 +2256,6 @@ func mustParseTime(t *testing.T, layout, value string) time.Time { func TestReconciliationRespectsPauseSetting(t *testing.T) { t.Parallel() - if !dbtestutil.WillUsePostgres() { - t.Skip("This test requires postgres") - } - ctx := testutil.Context(t, testutil.WaitLong) clock := quartz.NewMock(t) db, ps := dbtestutil.NewDB(t) diff --git a/enterprise/coderd/workspacequota_test.go b/enterprise/coderd/workspacequota_test.go index f39b090ca21b1..186af3a787d94 100644 --- a/enterprise/coderd/workspacequota_test.go +++ b/enterprise/coderd/workspacequota_test.go @@ -395,6 +395,265 @@ func TestWorkspaceQuota(t *testing.T) { verifyQuotaUser(ctx, t, client, second.Org.ID.String(), user.ID.String(), consumed, 35) }) + + // ZeroQuota tests that a user with a zero quota allowance can't create a workspace. + // Although relevant for all users, this test ensures that the prebuilds system user + // cannot create workspaces in an organization for which it has exhausted its quota. + t.Run("ZeroQuota", func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + defer cancel() + + // Create a client with no quota allowance + client, _, api, user := coderdenttest.NewWithAPI(t, &coderdenttest.Options{ + UserWorkspaceQuota: 0, // Set user workspace quota to 0 + LicenseOptions: &coderdenttest.LicenseOptions{ + Features: license.Features{ + codersdk.FeatureTemplateRBAC: 1, + }, + }, + }) + coderdtest.NewProvisionerDaemon(t, api.AGPL) + + // Verify initial quota is 0 + verifyQuota(ctx, t, client, user.OrganizationID.String(), 0, 0) + + // Create a template with a workspace that costs 1 credit + authToken := uuid.NewString() + version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{ + Parse: echo.ParseComplete, + ProvisionApply: []*proto.Response{{ + Type: &proto.Response_Apply{ + Apply: &proto.ApplyComplete{ + Resources: []*proto.Resource{{ + Name: "example", + Type: "aws_instance", + DailyCost: 1, + Agents: []*proto.Agent{{ + Id: uuid.NewString(), + Name: "example", + Auth: &proto.Agent_Token{ + Token: authToken, + }, + }}, + }}, + }, + }, + }}, + }) + coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) + template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID) + + // Attempt to create a workspace with zero quota - should fail + workspace := coderdtest.CreateWorkspace(t, client, template.ID) + build := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID) + + // Verify the build failed due to quota + require.Equal(t, codersdk.WorkspaceStatusFailed, build.Status) + require.Contains(t, build.Job.Error, "quota") + + // Verify quota consumption remains at 0 + verifyQuota(ctx, t, client, user.OrganizationID.String(), 0, 0) + + // Test with a template that has zero cost - should pass + versionZeroCost := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{ + Parse: echo.ParseComplete, + ProvisionApply: []*proto.Response{{ + Type: &proto.Response_Apply{ + Apply: &proto.ApplyComplete{ + Resources: []*proto.Resource{{ + Name: "example", + Type: "aws_instance", + DailyCost: 0, // Zero cost workspace + Agents: []*proto.Agent{{ + Id: uuid.NewString(), + Name: "example", + Auth: &proto.Agent_Token{ + Token: uuid.NewString(), + }, + }}, + }}, + }, + }, + }}, + }) + coderdtest.AwaitTemplateVersionJobCompleted(t, client, versionZeroCost.ID) + templateZeroCost := coderdtest.CreateTemplate(t, client, user.OrganizationID, versionZeroCost.ID) + + // Workspace with zero cost should pass + workspaceZeroCost := coderdtest.CreateWorkspace(t, client, templateZeroCost.ID) + buildZeroCost := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspaceZeroCost.LatestBuild.ID) + + require.Equal(t, codersdk.WorkspaceStatusRunning, buildZeroCost.Status) + require.Empty(t, buildZeroCost.Job.Error) + + // Verify quota consumption remains at 0 + verifyQuota(ctx, t, client, user.OrganizationID.String(), 0, 0) + }) + + // MultiOrg tests that a user can create workspaces in multiple organizations + // as long as they have enough quota in each organization. Specifically, + // in exhausted quota in one organization does not affect the ability to + // create workspaces in other organizations. This test is relevant to all users + // but is particularly relevant for the prebuilds system user. + t.Run("MultiOrg", func(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitLong) + + // Create a setup with multiple organizations + owner, _, api, first := coderdenttest.NewWithAPI(t, &coderdenttest.Options{ + LicenseOptions: &coderdenttest.LicenseOptions{ + Features: license.Features{ + codersdk.FeatureTemplateRBAC: 1, + codersdk.FeatureMultipleOrganizations: 1, + codersdk.FeatureExternalProvisionerDaemons: 1, + }, + }, + }) + coderdtest.NewProvisionerDaemon(t, api.AGPL) + + // Create a second organization + second := coderdenttest.CreateOrganization(t, owner, coderdenttest.CreateOrganizationOptions{ + IncludeProvisionerDaemon: true, + }) + + // Create a user that will be a member of both organizations + user, _ := coderdtest.CreateAnotherUser(t, owner, first.OrganizationID, rbac.ScopedRoleOrgMember(second.ID)) + + // Set up quota allowances for both organizations + // First org: 2 credits total + _, err := owner.PatchGroup(ctx, first.OrganizationID, codersdk.PatchGroupRequest{ + QuotaAllowance: ptr.Ref(2), + }) + require.NoError(t, err) + + // Second org: 3 credits total + _, err = owner.PatchGroup(ctx, second.ID, codersdk.PatchGroupRequest{ + QuotaAllowance: ptr.Ref(3), + }) + require.NoError(t, err) + + // Verify initial quotas + verifyQuota(ctx, t, user, first.OrganizationID.String(), 0, 2) + verifyQuota(ctx, t, user, second.ID.String(), 0, 3) + + // Create templates for both organizations + authToken := uuid.NewString() + version1 := coderdtest.CreateTemplateVersion(t, owner, first.OrganizationID, &echo.Responses{ + Parse: echo.ParseComplete, + ProvisionApply: []*proto.Response{{ + Type: &proto.Response_Apply{ + Apply: &proto.ApplyComplete{ + Resources: []*proto.Resource{{ + Name: "example", + Type: "aws_instance", + DailyCost: 1, + Agents: []*proto.Agent{{ + Id: uuid.NewString(), + Name: "example", + Auth: &proto.Agent_Token{ + Token: authToken, + }, + }}, + }}, + }, + }, + }}, + }) + coderdtest.AwaitTemplateVersionJobCompleted(t, owner, version1.ID) + template1 := coderdtest.CreateTemplate(t, owner, first.OrganizationID, version1.ID) + + version2 := coderdtest.CreateTemplateVersion(t, owner, second.ID, &echo.Responses{ + Parse: echo.ParseComplete, + ProvisionApply: []*proto.Response{{ + Type: &proto.Response_Apply{ + Apply: &proto.ApplyComplete{ + Resources: []*proto.Resource{{ + Name: "example", + Type: "aws_instance", + DailyCost: 1, + Agents: []*proto.Agent{{ + Id: uuid.NewString(), + Name: "example", + Auth: &proto.Agent_Token{ + Token: uuid.NewString(), + }, + }}, + }}, + }, + }, + }}, + }) + coderdtest.AwaitTemplateVersionJobCompleted(t, owner, version2.ID) + template2 := coderdtest.CreateTemplate(t, owner, second.ID, version2.ID) + + // Exhaust quota in the first organization by creating 2 workspaces + var workspaces1 []codersdk.Workspace + for i := 0; i < 2; i++ { + workspace := coderdtest.CreateWorkspace(t, user, template1.ID) + build := coderdtest.AwaitWorkspaceBuildJobCompleted(t, user, workspace.LatestBuild.ID) + require.Equal(t, codersdk.WorkspaceStatusRunning, build.Status) + workspaces1 = append(workspaces1, workspace) + } + + // Verify first org quota is exhausted + verifyQuota(ctx, t, user, first.OrganizationID.String(), 2, 2) + + // Try to create another workspace in the first org - should fail + workspace := coderdtest.CreateWorkspace(t, user, template1.ID) + build := coderdtest.AwaitWorkspaceBuildJobCompleted(t, user, workspace.LatestBuild.ID) + require.Equal(t, codersdk.WorkspaceStatusFailed, build.Status) + require.Contains(t, build.Job.Error, "quota") + + // Verify first org quota consumption didn't increase + verifyQuota(ctx, t, user, first.OrganizationID.String(), 2, 2) + + // Verify second org quota is still available + verifyQuota(ctx, t, user, second.ID.String(), 0, 3) + + // Create workspaces in the second organization - should succeed + for i := 0; i < 3; i++ { + workspace := coderdtest.CreateWorkspace(t, user, template2.ID) + build := coderdtest.AwaitWorkspaceBuildJobCompleted(t, user, workspace.LatestBuild.ID) + require.Equal(t, codersdk.WorkspaceStatusRunning, build.Status) + } + + // Verify second org quota is now exhausted + verifyQuota(ctx, t, user, second.ID.String(), 3, 3) + + // Try to create another workspace in the second org - should fail + workspace = coderdtest.CreateWorkspace(t, user, template2.ID) + build = coderdtest.AwaitWorkspaceBuildJobCompleted(t, user, workspace.LatestBuild.ID) + require.Equal(t, codersdk.WorkspaceStatusFailed, build.Status) + require.Contains(t, build.Job.Error, "quota") + + // Verify second org quota consumption didn't increase + verifyQuota(ctx, t, user, second.ID.String(), 3, 3) + + // Verify first org quota is still exhausted + verifyQuota(ctx, t, user, first.OrganizationID.String(), 2, 2) + + // Delete one workspace from the first org to free up quota + build = coderdtest.CreateWorkspaceBuild(t, user, workspaces1[0], database.WorkspaceTransitionDelete) + build = coderdtest.AwaitWorkspaceBuildJobCompleted(t, user, build.ID) + require.Equal(t, codersdk.WorkspaceStatusDeleted, build.Status) + + // Verify first org quota is now available again + verifyQuota(ctx, t, user, first.OrganizationID.String(), 1, 2) + + // Create a workspace in the first org - should succeed + workspace = coderdtest.CreateWorkspace(t, user, template1.ID) + build = coderdtest.AwaitWorkspaceBuildJobCompleted(t, user, workspace.LatestBuild.ID) + require.Equal(t, codersdk.WorkspaceStatusRunning, build.Status) + + // Verify first org quota is exhausted again + verifyQuota(ctx, t, user, first.OrganizationID.String(), 2, 2) + + // Verify second org quota remains exhausted + verifyQuota(ctx, t, user, second.ID.String(), 3, 3) + }) } // nolint:paralleltest,tparallel // Tests must run serially