Skip to content

fix: make GetWorkspacesEligibleForTransition return even less false positives #15594

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 36 commits into from
Dec 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
78d9bef
fix: begin impl of reducing autostart false-positive rate
DanielleMaywood Nov 18, 2024
dea0c20
chore: update dbmem.go
DanielleMaywood Nov 18, 2024
d0f1851
fix: dbmem.go impl
DanielleMaywood Nov 19, 2024
077439c
test: refactor test slightly
DanielleMaywood Nov 19, 2024
3b41985
fix: update golden files
DanielleMaywood Nov 19, 2024
1a66a14
fix: dbmem.go convertToWorkspaceRowsNoLock
DanielleMaywood Nov 19, 2024
9725ec7
fix: pass currentTick to GetWorkspacesEligibleForTransition
DanielleMaywood Nov 20, 2024
5e9b806
revert: query change
DanielleMaywood Nov 21, 2024
b2a037b
fix: ensure times are converted back to UTC
DanielleMaywood Nov 21, 2024
997c902
Merge branch 'main' into dm-experiment-autostart
DanielleMaywood Nov 21, 2024
dd05558
test: next start at is only a valid value
DanielleMaywood Nov 22, 2024
8bddfdf
fix: appease linter
DanielleMaywood Nov 22, 2024
66a01a3
chore: add an index for next_start_at
DanielleMaywood Nov 22, 2024
b7434c0
fix: run 'make gen'
DanielleMaywood Nov 22, 2024
100f54c
chore: begin impl of tests
DanielleMaywood Nov 25, 2024
d201025
Merge branch 'main' into dm-experiment-autostart
DanielleMaywood Nov 25, 2024
eac63b7
test: fix tests
DanielleMaywood Nov 25, 2024
1b31cbb
fix: use american english
DanielleMaywood Nov 25, 2024
eaf32ee
fix: make NextAllowedAutostart look up to 7 days into the future
DanielleMaywood Nov 26, 2024
1599b2a
fix: TestExecutorAutostartBlocked test
DanielleMaywood Nov 26, 2024
142e335
fix: make enterprise template schedule store use quartz.clock
DanielleMaywood Nov 26, 2024
3f17e46
fix: infinite loop
DanielleMaywood Nov 26, 2024
e993643
refactor: give template schedule store a quartz.clock
DanielleMaywood Nov 26, 2024
d349c56
fix: ensure tests give template schedule store a clock
DanielleMaywood Nov 26, 2024
a48fb99
fix: nullify next_start_at on schedule update
DanielleMaywood Nov 27, 2024
cc93075
fix: tests
DanielleMaywood Nov 27, 2024
45350d1
chore: remove clock from test
DanielleMaywood Nov 27, 2024
c1648ae
Merge branch 'main' into dm-experiment-autostart
DanielleMaywood Nov 27, 2024
03d0b7f
chore: relocate test
DanielleMaywood Nov 27, 2024
23b2ec9
chore: nits
DanielleMaywood Nov 28, 2024
18f3c8a
Merge branch 'main' into dm-experiment-autostart
DanielleMaywood Nov 28, 2024
8b1b6d9
chore: bump migration number
DanielleMaywood Nov 28, 2024
2d239a6
fix: stop query returning dormant workspaces
DanielleMaywood Nov 28, 2024
2e2f13a
Merge branch 'main' into dm-experiment-autostart
DanielleMaywood Nov 28, 2024
df74e6f
Merge branch 'main' into dm-experiment-autostart
DanielleMaywood Dec 2, 2024
dec3d6c
chore: add index to template_id on workspaces
DanielleMaywood Dec 2, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cli/testdata/coder_list_--output_json.golden
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
},
"automatic_updates": "never",
"allow_renames": false,
"favorite": false
"favorite": false,
"next_start_at": "[timestamp]"
}
]
4 changes: 4 additions & 0 deletions coderd/apidoc/docs.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions coderd/apidoc/swagger.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

23 changes: 20 additions & 3 deletions coderd/autobuild/lifecycle_executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ func (e *Executor) runOnce(t time.Time) Stats {
// NOTE: If a workspace build is created with a given TTL and then the user either
// changes or unsets the TTL, the deadline for the workspace build will not
// have changed. This behavior is as expected per #2229.
workspaces, err := e.db.GetWorkspacesEligibleForTransition(e.ctx, t)
workspaces, err := e.db.GetWorkspacesEligibleForTransition(e.ctx, currentTick)
if err != nil {
e.log.Error(e.ctx, "get workspaces for autostart or autostop", slog.Error(err))
return stats
Expand Down Expand Up @@ -205,6 +205,23 @@ func (e *Executor) runOnce(t time.Time) Stats {
return xerrors.Errorf("get template scheduling options: %w", err)
}

// If next start at is not valid we need to re-compute it
if !ws.NextStartAt.Valid && ws.AutostartSchedule.Valid {
next, err := schedule.NextAllowedAutostart(currentTick, ws.AutostartSchedule.String, templateSchedule)
if err == nil {
nextStartAt := sql.NullTime{Valid: true, Time: dbtime.Time(next.UTC())}
if err = tx.UpdateWorkspaceNextStartAt(e.ctx, database.UpdateWorkspaceNextStartAtParams{
ID: wsID,
NextStartAt: nextStartAt,
}); err != nil {
return xerrors.Errorf("update workspace next start at: %w", err)
}

// Save re-fetching the workspace
ws.NextStartAt = nextStartAt
}
}

tmpl, err = tx.GetTemplateByID(e.ctx, ws.TemplateID)
if err != nil {
return xerrors.Errorf("get template by ID: %w", err)
Expand Down Expand Up @@ -463,8 +480,8 @@ func isEligibleForAutostart(user database.User, ws database.Workspace, build dat
return false
}

nextTransition, allowed := schedule.NextAutostart(build.CreatedAt, ws.AutostartSchedule.String, templateSchedule)
if !allowed {
nextTransition, err := schedule.NextAllowedAutostart(build.CreatedAt, ws.AutostartSchedule.String, templateSchedule)
if err != nil {
return false
}

Expand Down
8 changes: 7 additions & 1 deletion coderd/autobuild/lifecycle_executor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1083,6 +1083,10 @@ func TestNotifications(t *testing.T) {
IncludeProvisionerDaemon: true,
NotificationsEnqueuer: &notifyEnq,
TemplateScheduleStore: schedule.MockTemplateScheduleStore{
SetFn: func(ctx context.Context, db database.Store, template database.Template, options schedule.TemplateScheduleOptions) (database.Template, error) {
template.TimeTilDormant = int64(options.TimeTilDormant)
return schedule.NewAGPLTemplateScheduleStore().Set(ctx, db, template, options)
},
GetFn: func(_ context.Context, _ database.Store, _ uuid.UUID) (schedule.TemplateScheduleOptions, error) {
return schedule.TemplateScheduleOptions{
UserAutostartEnabled: false,
Expand All @@ -1099,7 +1103,9 @@ func TestNotifications(t *testing.T) {
)

coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
template := coderdtest.CreateTemplate(t, client, admin.OrganizationID, version.ID)
template := coderdtest.CreateTemplate(t, client, admin.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) {
ctr.TimeTilDormantMillis = ptr.Ref(timeTilDormant.Milliseconds())
})
userClient, _ := coderdtest.CreateAnotherUser(t, client, admin.OrganizationID)
workspace := coderdtest.CreateWorkspace(t, userClient, template.ID)
coderdtest.AwaitWorkspaceBuildJobCompleted(t, userClient, workspace.LatestBuild.ID)
Expand Down
21 changes: 21 additions & 0 deletions coderd/database/dbauthz/dbauthz.go
Original file line number Diff line number Diff line change
Expand Up @@ -1030,6 +1030,13 @@ func (q *querier) BatchUpdateWorkspaceLastUsedAt(ctx context.Context, arg databa
return q.db.BatchUpdateWorkspaceLastUsedAt(ctx, arg)
}

func (q *querier) BatchUpdateWorkspaceNextStartAt(ctx context.Context, arg database.BatchUpdateWorkspaceNextStartAtParams) error {
if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceWorkspace.All()); err != nil {
return err
}
return q.db.BatchUpdateWorkspaceNextStartAt(ctx, arg)
}

func (q *querier) BulkMarkNotificationMessagesFailed(ctx context.Context, arg database.BulkMarkNotificationMessagesFailedParams) (int64, error) {
if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceNotificationMessage); err != nil {
return 0, err
Expand Down Expand Up @@ -2817,6 +2824,13 @@ func (q *querier) GetWorkspacesAndAgentsByOwnerID(ctx context.Context, ownerID u
return q.db.GetAuthorizedWorkspacesAndAgentsByOwnerID(ctx, ownerID, prep)
}

func (q *querier) GetWorkspacesByTemplateID(ctx context.Context, templateID uuid.UUID) ([]database.WorkspaceTable, error) {
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
return nil, err
}
return q.db.GetWorkspacesByTemplateID(ctx, templateID)
}

func (q *querier) GetWorkspacesEligibleForTransition(ctx context.Context, now time.Time) ([]database.GetWorkspacesEligibleForTransitionRow, error) {
return q.db.GetWorkspacesEligibleForTransition(ctx, now)
}
Expand Down Expand Up @@ -4062,6 +4076,13 @@ func (q *querier) UpdateWorkspaceLastUsedAt(ctx context.Context, arg database.Up
return update(q.log, q.auth, fetch, q.db.UpdateWorkspaceLastUsedAt)(ctx, arg)
}

func (q *querier) UpdateWorkspaceNextStartAt(ctx context.Context, arg database.UpdateWorkspaceNextStartAtParams) error {
fetch := func(ctx context.Context, arg database.UpdateWorkspaceNextStartAtParams) (database.Workspace, error) {
return q.db.GetWorkspaceByID(ctx, arg.ID)
}
return update(q.log, q.auth, fetch, q.db.UpdateWorkspaceNextStartAt)(ctx, arg)
}

func (q *querier) UpdateWorkspaceProxy(ctx context.Context, arg database.UpdateWorkspaceProxyParams) (database.WorkspaceProxy, error) {
fetch := func(ctx context.Context, arg database.UpdateWorkspaceProxyParams) (database.WorkspaceProxy, error) {
return q.db.GetWorkspaceProxyByID(ctx, arg.ID)
Expand Down
16 changes: 16 additions & 0 deletions coderd/database/dbauthz/dbauthz_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1908,6 +1908,19 @@ func (s *MethodTestSuite) TestWorkspace() {
ID: ws.ID,
}).Asserts(ws, policy.ActionUpdate).Returns()
}))
s.Run("UpdateWorkspaceNextStartAt", s.Subtest(func(db database.Store, check *expects) {
ws := dbgen.Workspace(s.T(), db, database.WorkspaceTable{})
check.Args(database.UpdateWorkspaceNextStartAtParams{
ID: ws.ID,
NextStartAt: sql.NullTime{Valid: true, Time: dbtime.Now()},
}).Asserts(ws, policy.ActionUpdate)
}))
s.Run("BatchUpdateWorkspaceNextStartAt", s.Subtest(func(db database.Store, check *expects) {
check.Args(database.BatchUpdateWorkspaceNextStartAtParams{
IDs: []uuid.UUID{uuid.New()},
NextStartAts: []time.Time{dbtime.Now()},
}).Asserts(rbac.ResourceWorkspace.All(), policy.ActionUpdate)
}))
s.Run("BatchUpdateWorkspaceLastUsedAt", s.Subtest(func(db database.Store, check *expects) {
ws1 := dbgen.Workspace(s.T(), db, database.WorkspaceTable{})
ws2 := dbgen.Workspace(s.T(), db, database.WorkspaceTable{})
Expand Down Expand Up @@ -2784,6 +2797,9 @@ func (s *MethodTestSuite) TestSystemFunctions() {
s.Run("GetTemplateAverageBuildTime", s.Subtest(func(db database.Store, check *expects) {
check.Args(database.GetTemplateAverageBuildTimeParams{}).Asserts(rbac.ResourceSystem, policy.ActionRead)
}))
s.Run("GetWorkspacesByTemplateID", s.Subtest(func(db database.Store, check *expects) {
check.Args(uuid.Nil).Asserts(rbac.ResourceSystem, policy.ActionRead)
}))
s.Run("GetWorkspacesEligibleForTransition", s.Subtest(func(db database.Store, check *expects) {
check.Args(time.Time{}).Asserts()
}))
Expand Down
1 change: 1 addition & 0 deletions coderd/database/dbgen/dbgen.go
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,7 @@ func Workspace(t testing.TB, db database.Store, orig database.WorkspaceTable) da
AutostartSchedule: orig.AutostartSchedule,
Ttl: orig.Ttl,
AutomaticUpdates: takeFirst(orig.AutomaticUpdates, database.AutomaticUpdatesNever),
NextStartAt: orig.NextStartAt,
})
require.NoError(t, err, "insert workspace")
return workspace
Expand Down
79 changes: 77 additions & 2 deletions coderd/database/dbmem/dbmem.go
Original file line number Diff line number Diff line change
Expand Up @@ -475,6 +475,7 @@ func (q *FakeQuerier) convertToWorkspaceRowsNoLock(ctx context.Context, workspac
DeletingAt: w.DeletingAt,
AutomaticUpdates: w.AutomaticUpdates,
Favorite: w.Favorite,
NextStartAt: w.NextStartAt,

OwnerAvatarUrl: extended.OwnerAvatarUrl,
OwnerUsername: extended.OwnerUsername,
Expand Down Expand Up @@ -1431,6 +1432,35 @@ func (q *FakeQuerier) BatchUpdateWorkspaceLastUsedAt(_ context.Context, arg data
return nil
}

func (q *FakeQuerier) BatchUpdateWorkspaceNextStartAt(_ context.Context, arg database.BatchUpdateWorkspaceNextStartAtParams) error {
err := validateDatabaseType(arg)
if err != nil {
return err
}

q.mutex.Lock()
defer q.mutex.Unlock()

for i, workspace := range q.workspaces {
for j, workspaceID := range arg.IDs {
if workspace.ID != workspaceID {
continue
}

nextStartAt := arg.NextStartAts[j]
if nextStartAt.IsZero() {
q.workspaces[i].NextStartAt = sql.NullTime{}
} else {
q.workspaces[i].NextStartAt = sql.NullTime{Valid: true, Time: nextStartAt}
}

break
}
}

return nil
}

func (*FakeQuerier) BulkMarkNotificationMessagesFailed(_ context.Context, arg database.BulkMarkNotificationMessagesFailedParams) (int64, error) {
err := validateDatabaseType(arg)
if err != nil {
Expand Down Expand Up @@ -6908,6 +6938,20 @@ func (q *FakeQuerier) GetWorkspacesAndAgentsByOwnerID(ctx context.Context, owner
return q.GetAuthorizedWorkspacesAndAgentsByOwnerID(ctx, ownerID, nil)
}

func (q *FakeQuerier) GetWorkspacesByTemplateID(_ context.Context, templateID uuid.UUID) ([]database.WorkspaceTable, error) {
q.mutex.RLock()
defer q.mutex.RUnlock()

workspaces := []database.WorkspaceTable{}
for _, workspace := range q.workspaces {
if workspace.TemplateID == templateID {
workspaces = append(workspaces, workspace)
}
}

return workspaces, nil
}

func (q *FakeQuerier) GetWorkspacesEligibleForTransition(ctx context.Context, now time.Time) ([]database.GetWorkspacesEligibleForTransitionRow, error) {
q.mutex.RLock()
defer q.mutex.RUnlock()
Expand Down Expand Up @@ -6952,7 +6996,13 @@ func (q *FakeQuerier) GetWorkspacesEligibleForTransition(ctx context.Context, no
if user.Status == database.UserStatusActive &&
job.JobStatus != database.ProvisionerJobStatusFailed &&
build.Transition == database.WorkspaceTransitionStop &&
workspace.AutostartSchedule.Valid {
workspace.AutostartSchedule.Valid &&
// We do not know if workspace with a zero next start is eligible
// for autostart, so we accept this false-positive. This can occur
// when a coder version is upgraded and next_start_at has yet to
// be set.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We have a new known case, reset by db trigger. Should we implement the trigger in memory db as well? (I know it's probably going away soon, but still, might cause flakes?)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think it'd cause flakes, but instead cause dbmem tests to fail when postgres tests do not. Happy to add it though

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On second thought, I've checked dbmem.go for how we emulate triggers and I'm not sure there is a good way to do it.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is no good way to do it, just ad hoc code in various functions 😢

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah that's unfortunate.

I think attempting to emulate this trigger isn't worth it then. If someone writes a test that is dependent on the trigger (which they ideally shouldn't) then they'll have to disable it when running under dbmem.go.

(workspace.NextStartAt.Time.IsZero() ||
!now.Before(workspace.NextStartAt.Time)) {
workspaces = append(workspaces, database.GetWorkspacesEligibleForTransitionRow{
ID: workspace.ID,
Name: workspace.Name,
Expand All @@ -6962,7 +7012,7 @@ func (q *FakeQuerier) GetWorkspacesEligibleForTransition(ctx context.Context, no

if !workspace.DormantAt.Valid &&
template.TimeTilDormant > 0 &&
now.Sub(workspace.LastUsedAt) > time.Duration(template.TimeTilDormant) {
now.Sub(workspace.LastUsedAt) >= time.Duration(template.TimeTilDormant) {
workspaces = append(workspaces, database.GetWorkspacesEligibleForTransitionRow{
ID: workspace.ID,
Name: workspace.Name,
Expand Down Expand Up @@ -7927,6 +7977,7 @@ func (q *FakeQuerier) InsertWorkspace(_ context.Context, arg database.InsertWork
Ttl: arg.Ttl,
LastUsedAt: arg.LastUsedAt,
AutomaticUpdates: arg.AutomaticUpdates,
NextStartAt: arg.NextStartAt,
}
q.workspaces = append(q.workspaces, workspace)
return workspace, nil
Expand Down Expand Up @@ -9868,6 +9919,7 @@ func (q *FakeQuerier) UpdateWorkspaceAutostart(_ context.Context, arg database.U
continue
}
workspace.AutostartSchedule = arg.AutostartSchedule
workspace.NextStartAt = arg.NextStartAt
q.workspaces[index] = workspace
return nil
}
Expand Down Expand Up @@ -10017,6 +10069,29 @@ func (q *FakeQuerier) UpdateWorkspaceLastUsedAt(_ context.Context, arg database.
return sql.ErrNoRows
}

func (q *FakeQuerier) UpdateWorkspaceNextStartAt(_ context.Context, arg database.UpdateWorkspaceNextStartAtParams) error {
err := validateDatabaseType(arg)
if err != nil {
return err
}

q.mutex.Lock()
defer q.mutex.Unlock()

for index, workspace := range q.workspaces {
if workspace.ID != arg.ID {
continue
}

workspace.NextStartAt = arg.NextStartAt
q.workspaces[index] = workspace

return nil
}

return sql.ErrNoRows
}

func (q *FakeQuerier) UpdateWorkspaceProxy(_ context.Context, arg database.UpdateWorkspaceProxyParams) (database.WorkspaceProxy, error) {
q.mutex.Lock()
defer q.mutex.Unlock()
Expand Down
21 changes: 21 additions & 0 deletions coderd/database/dbmetrics/querymetrics.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading