Skip to content

chore: fix concurrent CommitQuota transactions for unrelated users/orgs #15261

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 19 commits into from
Nov 1, 2024
Merged

chore: fix concurrent CommitQuota transactions for unrelated users/orgs #15261

Changes from 1 commit
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
dce750d
chore: making unit tests to try out various things
Emyrk Oct 23, 2024
4fd9760
wip
Emyrk Oct 23, 2024
6215603
wip
Emyrk Oct 24, 2024
9ccd0b2
wip
Emyrk Oct 28, 2024
784a98f
refactor tests
Emyrk Oct 28, 2024
5f021ad
add extra test
Emyrk Oct 28, 2024
426eeb6
delete unused code
Emyrk Oct 28, 2024
b69dd0f
revert some changes
Emyrk Oct 28, 2024
ec8f799
fix comment
Emyrk Oct 28, 2024
f613f31
prevent seq scan on workspace_builds for GetQuotaAllowanceForUser
Emyrk Oct 28, 2024
746203d
update tests
Emyrk Oct 28, 2024
c0d05db
fixups
Emyrk Oct 29, 2024
245f6df
linting
Emyrk Oct 29, 2024
422e694
fix opts
Emyrk Oct 29, 2024
685b21e
linting
Emyrk Oct 29, 2024
33dbd92
linting
Emyrk Oct 29, 2024
a99882a
text cleanup
Emyrk Oct 31, 2024
6ce8bfe
tests to assert errors, rather then skip the test
Emyrk Oct 31, 2024
3a9270d
chore: add PGLocks query to analyze what locks are held in pg (#15308)
Emyrk Nov 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
wip
  • Loading branch information
@Emyrk
Emyrk committed Oct 28, 2024
commit 9ccd0b2b85decf8f68207390fc0250bf0da54d44
283 changes: 228 additions & 55 deletions enterprise/coderd/workspacequota_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,10 @@ func verifyQuota(ctx context.Context, t *testing.T, client *codersdk.Client, org
func TestWorkspaceQuota(t *testing.T) {
t.Parallel()

if !dbtestutil.WillUsePostgres() {
t.Fatal("We should only run this test with postgres")
}

// This first test verifies the behavior of creating and deleting workspaces.
// It also tests multi-group quota stacking and the everyone group.
t.Run("CreateDelete", func(t *testing.T) {
Expand Down Expand Up @@ -381,13 +385,71 @@ func TestWorkspaceSerialization(t *testing.T) {

// UpdateBuildDeadline bumps a workspace deadline while doing a quota
// commit.
// pq: could not serialize access due to concurrent update
//
// Note: This passes if the interrupt is run before 'GetQuota()'
// Passing orders:
// - BeginTX -> Bump! -> GetQuota -> GetAllowance -> UpdateCost -> EndTx
// - BeginTX -> GetQuota -> GetAllowance -> UpdateCost -> Bump! -> EndTx
t.Run("UpdateBuildDeadline", func(t *testing.T) {
// +------------------------------+------------------+
// | Begin Tx | |
// +------------------------------+------------------+
// | GetQuota(user) | |
// +------------------------------+------------------+
// | | BumpDeadline(w1) |
// +------------------------------+------------------+
// | GetAllowance(user) | |
// +------------------------------+------------------+
// | UpdateWorkspaceBuildCost(w1) | |
// +------------------------------+------------------+
// | CommitTx() | |
// +------------------------------+------------------+
// pq: could not serialize access due to concurrent update
ctx := testutil.Context(t, testutil.WaitLong)
ctx = dbauthz.AsSystemRestricted(ctx)

bumpDeadline := func() {
err = db.InTx(func(db database.Store) error {
err := db.UpdateWorkspaceBuildDeadlineByID(ctx, database.UpdateWorkspaceBuildDeadlineByIDParams{
Deadline: dbtime.Now(),
MaxDeadline: dbtime.Now(),
UpdatedAt: dbtime.Now(),
ID: workspaceResp.Build.ID,
})
return err
}, &sql.TxOptions{
Isolation: sql.LevelSerializable,
})
assert.NoError(t, err)

}

// Start TX
// Run order

quota := newCommitter(t, db, workspace, workspaceResp.Build)
quota.GetQuota(ctx, t) // Step 1
bumpDeadline() // Interrupt
quota.GetAllowance(ctx, t) // Step 2
quota.UpdateWorkspaceBuildCostByID(ctx, t, 10) // Step 3
// End commit
require.NoError(t, quota.Done())
})

t.Run("UpdateOtherBuildDeadline", func(t *testing.T) {
// +------------------------------+------------------+
// | Begin Tx | |
// +------------------------------+------------------+
// | GetQuota(user) | |
// +------------------------------+------------------+
// | | BumpDeadline(w2) |
// +------------------------------+------------------+
// | GetAllowance(user) | |
// +------------------------------+------------------+
// | UpdateWorkspaceBuildCost(w1) | |
// +------------------------------+------------------+
// | CommitTx() | |
// +------------------------------+------------------+
ctx := testutil.Context(t, testutil.WaitLong)
ctx = dbauthz.AsSystemRestricted(ctx)

Expand All @@ -396,7 +458,7 @@ func TestWorkspaceSerialization(t *testing.T) {
Deadline: dbtime.Now(),
MaxDeadline: dbtime.Now(),
UpdatedAt: dbtime.Now(),
ID: workspaceResp.Build.ID,
ID: workspaceTwoResp.Build.ID,
})
require.NoError(t, err)
}
Expand Down Expand Up @@ -464,6 +526,30 @@ func TestWorkspaceSerialization(t *testing.T) {
})

t.Run("DoubleCommit", func(t *testing.T) {
// +---------------------+---------------------+
// | W1 Quota Tx | W2 Quota Tx |
// +---------------------+---------------------+
// | Begin Tx | |
// +---------------------+---------------------+
// | | Begin Tx |
// +---------------------+---------------------+
// | GetQuota(w1) | |
// +---------------------+---------------------+
// | GetAllowance(w1) | |
// +---------------------+---------------------+
// | UpdateBuildCost(w1) | |
// +---------------------+---------------------+
// | | UpdateBuildCost(w2) |
// +---------------------+---------------------+
// | | GetQuota(w2) |
// +---------------------+---------------------+
// | | GetAllowance(w2) |
// +---------------------+---------------------+
// | CommitTx() | |
// +---------------------+---------------------+
// | | CommitTx() |
// +---------------------+---------------------+
// pq: could not serialize access due to read/write dependencies among transactions
ctx := testutil.Context(t, testutil.WaitLong)
ctx = dbauthz.AsSystemRestricted(ctx)

Expand All @@ -476,82 +562,155 @@ func TestWorkspaceSerialization(t *testing.T) {
one.GetAllowance(ctx, t)

one.UpdateWorkspaceBuildCostByID(ctx, t, 10)
two.UpdateWorkspaceBuildCostByID(ctx, t, 10)

two.GetQuota(ctx, t)
two.GetAllowance(ctx, t)
two.UpdateWorkspaceBuildCostByID(ctx, t, 10)

// End commit
require.NoError(t, one.Done())
require.NoError(t, two.Done())
})

t.Run("BumpLastUsedAt", func(t *testing.T) {
// +---------------------+----------------------------------+
// | W1 Quota Tx | |
// +---------------------+----------------------------------+
// | Begin Tx | |
// +---------------------+----------------------------------+
// | GetQuota(w1) | |
// +---------------------+----------------------------------+
// | GetAllowance(w1) | |
// +---------------------+----------------------------------+
// | | UpdateWorkspaceBuildDeadline(w1) |
// +---------------------+----------------------------------+
// | UpdateBuildCost(w1) | |
// +---------------------+----------------------------------+
// | CommitTx() | |
// +---------------------+----------------------------------+
// pq: could not serialize access due to concurrent update
ctx := testutil.Context(t, testutil.WaitShort)
ctx = dbauthz.AsSystemRestricted(ctx)

two := dbtestutil.StartTx(t, db, &sql.TxOptions{
Isolation: sql.LevelSerializable,
ReadOnly: false,
})
one := newCommitter(t, db, workspace, workspaceResp.Build)

//two := newCommitter(t, db, workspaceTwo, workspaceResp.Build)

// Run order
one.GetQuota(ctx, t)
one.GetAllowance(ctx, t)

//two.UpdateWorkspaceBuildCostByID(ctx, t, 10)
err = db.UpdateWorkspaceBuildDeadlineByID(ctx, database.UpdateWorkspaceBuildDeadlineByIDParams{
Deadline: dbtime.Now(),
MaxDeadline: dbtime.Now(),
UpdatedAt: dbtime.Now(),
ID: workspaceResp.Build.ID,
})
assert.NoError(t, err)

//err := two.UpdateWorkspaceBuildCostByID(ctx, database.UpdateWorkspaceBuildCostByIDParams{
// ID: workspaceResp.Build.ID,
// DailyCost: 30,
//})
//require.NoError(t, err)
one.UpdateWorkspaceBuildCostByID(ctx, t, 10)

//q, err := two.GetQuotaConsumedForUser(ctx, database.GetQuotaConsumedForUserParams{
// OwnerID: user.ID,
// OrganizationID: workspace.OrganizationID,
//})
//require.NoError(t, err)
//fmt.Println(q)
// End commit
assert.NoError(t, one.Done())
})

t.Run("ActivityBump", func(t *testing.T) {
// +---------------------+----------------------------------+
// | W1 Quota Tx | |
// +---------------------+----------------------------------+
// | Begin Tx | |
// +---------------------+----------------------------------+
// | GetQuota(w1) | |
// +---------------------+----------------------------------+
// | GetAllowance(w1) | |
// +---------------------+----------------------------------+
// | | ActivityBump(w1) |
// +---------------------+----------------------------------+
// | UpdateBuildCost(w1) | |
// +---------------------+----------------------------------+
// | CommitTx() | |
// +---------------------+----------------------------------+
// pq: could not serialize access due to concurrent update
ctx := testutil.Context(t, testutil.WaitShort)
ctx = dbauthz.AsSystemRestricted(ctx)

one := newCommitter(t, db, workspace, workspaceResp.Build)

// Run order
one.GetQuota(ctx, t)
one.GetAllowance(ctx, t)

err = db.ActivityBumpWorkspace(ctx, database.ActivityBumpWorkspaceParams{
NextAutostart: time.Now(),
WorkspaceID: workspaceResp.Workspace.ID,
})

assert.NoError(t, err)

one.UpdateWorkspaceBuildCostByID(ctx, t, 10)

wg := sync.WaitGroup{}
wg.Add(1)
go func() {
defer wg.Done()
err = two.UpdateWorkspaceBuildDeadlineByID(ctx, database.UpdateWorkspaceBuildDeadlineByIDParams{
Deadline: dbtime.Now(),
MaxDeadline: dbtime.Now(),
UpdatedAt: dbtime.Now(),
ID: workspaceResp.Build.ID,
})
require.NoError(t, err)
}()
time.Sleep(time.Millisecond * 800)
// End commit
assert.NoError(t, one.Done())
})

//err = db.UpdateWorkspaceLastUsedAt(ctx, database.UpdateWorkspaceLastUsedAtParams{
// ID: workspace.ID,
// LastUsedAt: dbtime.Now(),
//})
//require.NoError(t, err)
//
//err = db.UpdateWorkspaceBuildCostByID(ctx, database.UpdateWorkspaceBuildCostByIDParams{
// ID: workspaceResp.Build.ID,
// DailyCost: 20,
//})
//require.NoError(t, err)
t.Run("UserMod", func(t *testing.T) {
// +---------------------+----------------------------------+
// | W1 Quota Tx | |
// +---------------------+----------------------------------+
// | Begin Tx | |
// +---------------------+----------------------------------+
// | GetQuota(w1) | |
// +---------------------+----------------------------------+
// | GetAllowance(w1) | |
// +---------------------+----------------------------------+
// | | UpdateWorkspaceBuildDeadline(w1) |
// +---------------------+----------------------------------+
// | UpdateBuildCost(w1) | |
// +---------------------+----------------------------------+
// | CommitTx() | |
// +---------------------+----------------------------------+
// pq: could not serialize access due to concurrent update
ctx := testutil.Context(t, testutil.WaitShort)
ctx = dbauthz.AsSystemRestricted(ctx)

db.InsertOrganizationMember(ctx, database.InsertOrganizationMemberParams{
OrganizationID: org.ID,
UserID: user.ID,
})

one := newCommitter(t, db, workspace, workspaceResp.Build)

// Run order

err = db.RemoveUserFromAllGroups(ctx, user.ID)
assert.NoError(t, err)

err = db.DeleteOrganizationMember(ctx, database.DeleteOrganizationMemberParams{
OrganizationID: org.ID,
UserID: user.ID,
})
assert.NoError(t, err)

err = db.ActivityBumpWorkspace(ctx, database.ActivityBumpWorkspaceParams{
NextAutostart: time.Now(),
WorkspaceID: workspaceResp.Workspace.ID,
})

one.GetQuota(ctx, t)
one.GetAllowance(ctx, t)

//two.GetQuota(ctx, t)
//two.GetAllowance(ctx, t)
err = db.ActivityBumpWorkspace(ctx, database.ActivityBumpWorkspaceParams{
NextAutostart: time.Now(),
WorkspaceID: workspaceResp.Workspace.ID,
})

one.UpdateWorkspaceBuildCostByID(ctx, t, 10)

err = db.ActivityBumpWorkspace(ctx, database.ActivityBumpWorkspaceParams{
NextAutostart: time.Now(),
WorkspaceID: workspaceResp.Workspace.ID,
})

// End commit
require.NoError(t, one.Done())
wg.Wait()
require.NoError(t, two.Done())
assert.NoError(t, one.Done())
})

// TODO: Try to fail a non-repeatable read only transaction
Expand All @@ -577,13 +736,13 @@ func TestWorkspaceSerialization(t *testing.T) {
two.GetAllowance(ctx, t)

// End commit
require.NoError(t, one.Done())
assert.NoError(t, one.Done())

fmt.Println("2a", two.GetQuota(ctx, t))
two.GetAllowance(ctx, t)

require.NoError(t, two.Done())
require.NoError(t, three.Done())
assert.NoError(t, two.Done())
assert.NoError(t, three.Done())

//allow, err = db.GetQuotaConsumedForUser(ctx, database.GetQuotaConsumedForUserParams{
// OwnerID: user.ID,
Expand Down Expand Up @@ -778,7 +937,12 @@ func newCommitter(t *testing.T, db database.Store, workspace database.WorkspaceT
return &committer{DBTx: quotaTX, w: workspace, b: build}
}

// GetQuota touches:
// - workspace_builds
// - workspaces
func (c *committer) GetQuota(ctx context.Context, t *testing.T) int64 {
t.Helper()

consumed, err := c.DBTx.GetQuotaConsumedForUser(ctx, database.GetQuotaConsumedForUserParams{
OwnerID: c.w.OwnerID,
OrganizationID: c.w.OrganizationID,
Expand All @@ -787,7 +951,14 @@ func (c *committer) GetQuota(ctx context.Context, t *testing.T) int64 {
return consumed
}

// GetAllowance touches:
// - group_members_expanded
// - users
// - groups
// - org_members
func (c *committer) GetAllowance(ctx context.Context, t *testing.T) int64 {
t.Helper()

allowance, err := c.DBTx.GetQuotaAllowanceForUser(ctx, database.GetQuotaAllowanceForUserParams{
UserID: c.w.OwnerID,
OrganizationID: c.w.OrganizationID,
Expand All @@ -796,12 +967,14 @@ func (c *committer) GetAllowance(ctx context.Context, t *testing.T) int64 {
return allowance
}

func (c *committer) UpdateWorkspaceBuildCostByID(ctx context.Context, t *testing.T, cost int32) {
func (c *committer) UpdateWorkspaceBuildCostByID(ctx context.Context, t *testing.T, cost int32) bool {
t.Helper()

err := c.DBTx.UpdateWorkspaceBuildCostByID(ctx, database.UpdateWorkspaceBuildCostByIDParams{
ID: c.b.ID,
DailyCost: cost,
})
require.NoError(t, err)
return assert.NoError(t, err)
}

func (c *committer) Done() error {
Expand Down