Skip to content

Commit 8a39344

Browse files
sreyasreya
committed
feat: add auto-locking/deleting to autobuild
1 parent 749307e commit 8a39344

16 files changed

+527
-33
lines changed

‎coderd/apidoc/docs.go

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/apidoc/swagger.json

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/autobuild/lifecycle_executor.go

Lines changed: 103 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -160,23 +160,65 @@ func (e *Executor) runOnce(t time.Time) Stats {
160160
return nil
161161
}
162162

163-
builder := wsbuilder.New(ws, nextTransition).
164-
SetLastWorkspaceBuildInTx(&latestBuild).
165-
SetLastWorkspaceBuildJobInTx(&latestJob).
166-
Reason(reason)
167-
168-
if _, _, err := builder.Build(e.ctx, tx, nil); err != nil {
169-
log.Error(e.ctx, "workspace build error",
170-
slog.F("transition", nextTransition),
171-
slog.Error(err),
163+
if nextTransition != "" {
164+
builder := wsbuilder.New(ws, nextTransition).
165+
SetLastWorkspaceBuildInTx(&latestBuild).
166+
SetLastWorkspaceBuildJobInTx(&latestJob).
167+
Reason(reason)
168+
169+
if _, _, err := builder.Build(e.ctx, tx, nil); err != nil {
170+
log.Error(e.ctx, "unable to transition workspace",
171+
slog.F("transition", nextTransition),
172+
slog.Error(err),
173+
)
174+
return nil
175+
}
176+
}
177+
178+
// Lock the workspace if it has breached the template's
179+
// threshold for inactivity.
180+
if reason == database.BuildReasonAutolock {
181+
err = tx.UpdateWorkspaceLockedAt(e.ctx, database.UpdateWorkspaceLockedAtParams{
182+
ID: ws.ID,
183+
LockedAt: sql.NullTime{
184+
Time: database.Now(),
185+
Valid: true,
186+
},
187+
})
188+
if err != nil {
189+
log.Error(e.ctx, "unable to lock workspace",
190+
slog.F("transition", nextTransition),
191+
slog.Error(err),
192+
)
193+
return nil
194+
}
195+
196+
log.Info(e.ctx, "locked workspace",
197+
slog.F("last_used_at", ws.LastUsedAt),
198+
slog.F("inactivity_ttl", templateSchedule.InactivityTTL),
199+
slog.F("since_last_used_at", time.Since(ws.LastUsedAt)),
200+
)
201+
}
202+
203+
if reason == database.BuildReasonAutodelete {
204+
log.Info(e.ctx, "deleted workspace",
205+
slog.F("locked_at", ws.LockedAt.Time),
206+
slog.F("locked_ttl", templateSchedule.LockedTTL),
172207
)
208+
}
209+
210+
if nextTransition == "" {
173211
return nil
174212
}
213+
175214
statsMu.Lock()
176215
stats.Transitions[ws.ID] = nextTransition
177216
statsMu.Unlock()
178217

179-
log.Info(e.ctx, "scheduling workspace transition", slog.F("transition", nextTransition))
218+
log.Info(e.ctx, "scheduling workspace transition",
219+
slog.F("transition", nextTransition),
220+
slog.F("reason", reason),
221+
)
180222

181223
return nil
182224

@@ -199,6 +241,12 @@ func (e *Executor) runOnce(t time.Time) Stats {
199241
return stats
200242
}
201243

244+
// getNextTransition returns the next eligible transition for the workspace
245+
// as well as the reason for why it is transitioning. It is possible
246+
// for this function to return a nil error as well as an empty transition.
247+
// In such cases it means no provisioning should occur but the workspace
248+
// may be "transitioning" to a new state (such as a inactive, stopped
249+
// workspace transitioning to the locked state).
202250
func getNextTransition(
203251
ws database.Workspace,
204252
latestBuild database.WorkspaceBuild,
@@ -211,12 +259,23 @@ func getNextTransition(
211259
error,
212260
) {
213261
switch {
214-
case isEligibleForAutostop(latestBuild, latestJob, currentTick):
262+
case isEligibleForAutostop(ws, latestBuild, latestJob, currentTick):
215263
return database.WorkspaceTransitionStop, database.BuildReasonAutostop, nil
216264
case isEligibleForAutostart(ws, latestBuild, latestJob, templateSchedule, currentTick):
217265
return database.WorkspaceTransitionStart, database.BuildReasonAutostart, nil
218266
case isEligibleForFailedStop(latestBuild, latestJob, templateSchedule):
219267
return database.WorkspaceTransitionStop, database.BuildReasonAutostop, nil
268+
case isEligibleForLockedStop(ws, templateSchedule):
269+
// Only stop started workspaces.
270+
if latestBuild.Transition == database.WorkspaceTransitionStart {
271+
return database.WorkspaceTransitionStop, database.BuildReasonAutolock, nil
272+
}
273+
// We shouldn't transition the workspace but we should still
274+
// lock it.
275+
return "", database.BuildReasonAutolock, nil
276+
277+
case isEligibleForDelete(ws, templateSchedule):
278+
return database.WorkspaceTransitionDelete, database.BuildReasonAutodelete, nil
220279
default:
221280
return "", "", xerrors.Errorf("last transition not valid for autostart or autostop")
222281
}
@@ -225,7 +284,12 @@ func getNextTransition(
225284
// isEligibleForAutostart returns true if the workspace should be autostarted.
226285
func isEligibleForAutostart(ws database.Workspace, build database.WorkspaceBuild, job database.ProvisionerJob, templateSchedule schedule.TemplateScheduleOptions, currentTick time.Time) bool {
227286
// Don't attempt to autostart failed workspaces.
228-
if !job.CompletedAt.Valid || job.Error.String != "" {
287+
if db2sdk.ProvisionerJobStatus(job) == codersdk.ProvisionerJobFailed {
288+
return false
289+
}
290+
291+
// If the workspace is locked we should not autostart it.
292+
if ws.LockedAt.Valid {
229293
return false
230294
}
231295

@@ -253,9 +317,13 @@ func isEligibleForAutostart(ws database.Workspace, build database.WorkspaceBuild
253317
}
254318

255319
// isEligibleForAutostart returns true if the workspace should be autostopped.
256-
func isEligibleForAutostop(build database.WorkspaceBuild, job database.ProvisionerJob, currentTick time.Time) bool {
257-
// Don't attempt to autostop failed workspaces.
258-
if !job.CompletedAt.Valid || job.Error.String != "" {
320+
func isEligibleForAutostop(ws database.Workspace, build database.WorkspaceBuild, job database.ProvisionerJob, currentTick time.Time) bool {
321+
if db2sdk.ProvisionerJobStatus(job) == codersdk.ProvisionerJobFailed {
322+
return false
323+
}
324+
325+
// If the workspace is locked we should not autostop it.
326+
if ws.LockedAt.Valid {
259327
return false
260328
}
261329

@@ -266,6 +334,26 @@ func isEligibleForAutostop(build database.WorkspaceBuild, job database.Provision
266334
!currentTick.Before(build.Deadline)
267335
}
268336

337+
// isEligibleForLockedStop returns true if the workspace should be locked
338+
// for breaching the inactivity threshold of the template.
339+
func isEligibleForLockedStop(ws database.Workspace, templateSchedule schedule.TemplateScheduleOptions) bool {
340+
// Only attempt to lock workspaces not already locked.
341+
return !ws.LockedAt.Valid &&
342+
// The template must specify an inactivity TTL.
343+
templateSchedule.InactivityTTL > 0 &&
344+
// The workspace must breach the inactivity TTL.
345+
database.Now().Sub(ws.LastUsedAt) > templateSchedule.InactivityTTL
346+
}
347+
348+
func isEligibleForDelete(ws database.Workspace, templateSchedule schedule.TemplateScheduleOptions) bool {
349+
// Only attempt to delete locked workspaces.
350+
return ws.LockedAt.Valid &&
351+
// Locked workspaces should only be deleted if a locked_ttl is specified.
352+
templateSchedule.LockedTTL > 0 &&
353+
// The workspace must breach the locked_ttl.
354+
database.Now().Sub(ws.LockedAt.Time) > templateSchedule.LockedTTL
355+
}
356+
269357
// isEligibleForFailedStop returns true if the workspace is eligible to be stopped
270358
// due to a failed build.
271359
func isEligibleForFailedStop(build database.WorkspaceBuild, job database.ProvisionerJob, templateSchedule schedule.TemplateScheduleOptions) bool {

‎coderd/autobuild/lifecycle_executor_test.go

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -651,8 +651,9 @@ func TestExecutorAutostartTemplateDisabled(t *testing.T) {
651651
assert.Len(t, stats.Transitions, 0)
652652
}
653653

654-
// TesetExecutorFailedWorkspace tests that failed workspaces that breach
655-
// their template failed_ttl threshold trigger a stop job.
654+
// TestExecutorFailedWorkspace test AGPL functionality which mainly
655+
// ensures that autostop actions as a result of a failed workspace
656+
// build do not trigger.
656657
// For enterprise functionality see enterprise/coderd/workspaces_test.go
657658
func TestExecutorFailedWorkspace(t *testing.T) {
658659
t.Parallel()
@@ -705,6 +706,61 @@ func TestExecutorFailedWorkspace(t *testing.T) {
705706
})
706707
}
707708

709+
// TestExecutorInactiveWorkspace test AGPL functionality which mainly
710+
// ensures that autostop actions as a result of an inactive workspace
711+
// do not trigger.
712+
// For enterprise functionality see enterprise/coderd/workspaces_test.go
713+
func TestExecutorInactiveWorkspace(t *testing.T) {
714+
t.Parallel()
715+
716+
// Test that an AGPL TemplateScheduleStore properly disables
717+
// functionality.
718+
t.Run("OK", func(t *testing.T) {
719+
t.Parallel()
720+
721+
var (
722+
ticker = make(chan time.Time)
723+
statCh = make(chan autobuild.Stats)
724+
logger = slogtest.Make(t, &slogtest.Options{
725+
// We ignore errors here since we expect to fail
726+
// builds.
727+
IgnoreErrors: true,
728+
})
729+
inactiveTTL = time.Millisecond
730+
731+
client = coderdtest.New(t, &coderdtest.Options{
732+
Logger: &logger,
733+
AutobuildTicker: ticker,
734+
IncludeProvisionerDaemon: true,
735+
AutobuildStats: statCh,
736+
TemplateScheduleStore: schedule.NewAGPLTemplateScheduleStore(),
737+
})
738+
)
739+
user := coderdtest.CreateFirstUser(t, client)
740+
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
741+
Parse: echo.ParseComplete,
742+
ProvisionPlan: echo.ProvisionComplete,
743+
ProvisionApply: echo.ProvisionComplete,
744+
})
745+
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) {
746+
ctr.InactivityTTLMillis = ptr.Ref[int64](inactiveTTL.Milliseconds())
747+
})
748+
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
749+
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
750+
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
751+
require.Equal(t, codersdk.WorkspaceStatusRunning, build.Status)
752+
require.Eventually(t,
753+
func() bool {
754+
return database.Now().Sub(ws.LastUsedAt) > inactiveTTL
755+
},
756+
testutil.IntervalMedium, testutil.IntervalFast)
757+
ticker <- time.Now()
758+
stats := <-statCh
759+
// Expect no transitions since we're using AGPL.
760+
require.Len(t, stats.Transitions, 0)
761+
})
762+
}
763+
708764
func mustProvisionWorkspace(t *testing.T, client *codersdk.Client, mut ...func(*codersdk.CreateWorkspaceRequest)) codersdk.Workspace {
709765
t.Helper()
710766
user := coderdtest.CreateFirstUser(t, client)

‎coderd/database/dbfake/dbfake.go

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3482,12 +3482,17 @@ func (q *fakeQuerier) GetWorkspacesEligibleForTransition(ctx context.Context, no
34823482
return nil, err
34833483
}
34843484

3485-
if build.Transition == database.WorkspaceTransitionStart && !build.Deadline.IsZero() && build.Deadline.Before(now) {
3485+
if build.Transition == database.WorkspaceTransitionStart &&
3486+
!build.Deadline.IsZero() &&
3487+
build.Deadline.Before(now) &&
3488+
!workspace.LockedAt.Valid {
34863489
workspaces = append(workspaces, workspace)
34873490
continue
34883491
}
34893492

3490-
if build.Transition == database.WorkspaceTransitionStop && workspace.AutostartSchedule.Valid {
3493+
if build.Transition == database.WorkspaceTransitionStop &&
3494+
workspace.AutostartSchedule.Valid &&
3495+
!workspace.LockedAt.Valid {
34913496
workspaces = append(workspaces, workspace)
34923497
continue
34933498
}
@@ -3500,6 +3505,19 @@ func (q *fakeQuerier) GetWorkspacesEligibleForTransition(ctx context.Context, no
35003505
workspaces = append(workspaces, workspace)
35013506
continue
35023507
}
3508+
3509+
template, err := q.GetTemplateByID(ctx, workspace.TemplateID)
3510+
if err != nil {
3511+
return nil, xerrors.Errorf("get template by ID: %w", err)
3512+
}
3513+
if !workspace.LockedAt.Valid && template.InactivityTTL > 0 {
3514+
workspaces = append(workspaces, workspace)
3515+
continue
3516+
}
3517+
if workspace.LockedAt.Valid && template.LockedTTL > 0 {
3518+
workspaces = append(workspaces, workspace)
3519+
continue
3520+
}
35033521
}
35043522

35053523
return workspaces, nil
@@ -4688,6 +4706,7 @@ func (q *fakeQuerier) UpdateTemplateScheduleByID(_ context.Context, arg database
46884706
tpl.MaxTTL = arg.MaxTTL
46894707
tpl.FailureTTL = arg.FailureTTL
46904708
tpl.InactivityTTL = arg.InactivityTTL
4709+
tpl.LockedTTL = arg.LockedTTL
46914710
q.templates[idx] = tpl
46924711
return tpl.DeepCopy(), nil
46934712
}

‎coderd/database/dump.sql

Lines changed: 4 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/migrations/000132_workspace_build_reason.down.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
-- It's not possible to delete enum values.

‎coderd/database/migrations/000132_workspace_build_reason.up.sql

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
BEGIN;
2+
ALTER TYPE build_reason ADD VALUE IF NOT EXISTS 'autolock';
3+
ALTER TYPE build_reason ADD VALUE IF NOT EXISTS 'failedstop';
4+
ALTER TYPE build_reason ADD VALUE IF NOT EXISTS 'autodelete';
5+
COMMIT;

‎coderd/database/models.go

Lines changed: 13 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/queries.sql.go

Lines changed: 16 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)