From 10a877bb070c8a9e07fa5a6e1ab9d35ab17c6776 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Mon, 19 Sep 2022 19:04:33 +0000 Subject: [PATCH 1/9] feat: bump workspace deadline on user activity Resolves #2995 --- coderd/coderd.go | 2 + coderd/httpmw/workspacebump.go | 75 ++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 coderd/httpmw/workspacebump.go diff --git a/coderd/coderd.go b/coderd/coderd.go index 607f15ff7931b..d9c55715e2d9c 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -201,6 +201,7 @@ func New(options *Options) *API { httpmw.ExtractUserParam(api.Database), // Extracts the from the url httpmw.ExtractWorkspaceAndAgentParam(api.Database), + httpmw.BumpWorkspaceAutoStop(api.Logger, api.Database), ) r.HandleFunc("/*", api.workspaceAppsProxyPath) } @@ -421,6 +422,7 @@ func New(options *Options) *API { apiKeyMiddleware, httpmw.ExtractWorkspaceAgentParam(options.Database), httpmw.ExtractWorkspaceParam(options.Database), + httpmw.BumpWorkspaceAutoStop(api.Logger, options.Database), ) r.Get("/", api.workspaceAgent) r.Get("/pty", api.workspaceAgentPTY) diff --git a/coderd/httpmw/workspacebump.go b/coderd/httpmw/workspacebump.go new file mode 100644 index 0000000000000..fa8043c28b87e --- /dev/null +++ b/coderd/httpmw/workspacebump.go @@ -0,0 +1,75 @@ +package httpmw + +import ( + "database/sql" + "errors" + "net/http" + "time" + + "golang.org/x/xerrors" + + "cdr.dev/slog" + "github.com/coder/coder/coderd/database" +) + +// BumpWorkspaceAutoStop automatically bumps the workspace's auto-off timer +// if it is set to expire soon. +// It must be ran after ExtractWorkspace. +func BumpWorkspaceAutoStop(log slog.Logger, db database.Store) func(h http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + workspace := WorkspaceParam(r) + + err := db.InTx(func(s database.Store) error { + build, err := s.GetLatestWorkspaceBuildByWorkspaceID(r.Context(), workspace.ID) + if errors.Is(err, sql.ErrNoRows) { + return nil + } else if err != nil { + return xerrors.Errorf("get latest workspace build: %w", err) + } + + job, err := s.GetProvisionerJobByID(r.Context(), build.JobID) + if err != nil { + return xerrors.Errorf("get provisioner job: %w", err) + } + + if build.Transition != database.WorkspaceTransitionStart || !job.CompletedAt.Valid { + return nil + } + + if build.Deadline.IsZero() { + // Workspace shutdown is manual + return nil + } + + // We sent bumpThreshold slightly under bumpAmount to minimize DB writes. + const ( + bumpAmount = time.Hour + bumpThreshold = time.Hour - time.Minute*10 + ) + + if !build.Deadline.Before(time.Now().Add(bumpThreshold)) { + return nil + } + + newDeadline := time.Now().Add(bumpAmount) + + if err := s.UpdateWorkspaceBuildByID(r.Context(), database.UpdateWorkspaceBuildByIDParams{ + ID: build.ID, + UpdatedAt: build.UpdatedAt, + ProvisionerState: build.ProvisionerState, + Deadline: newDeadline, + }); err != nil { + return xerrors.Errorf("update workspace build: %w", err) + } + return nil + }) + + if err != nil { + log.Error(r.Context(), "auto-bump", slog.Error(err)) + } + + next.ServeHTTP(w, r) + }) + } +} From c43c2ffc840ba7a74ab5fa82464c7ae116397db5 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 20 Sep 2022 02:30:23 +0000 Subject: [PATCH 2/9] Add backend --- coderd/coderd.go | 4 +- coderd/httpmw/workspaceactivitybump.go | 89 ++++++++++++++++++++++++++ coderd/httpmw/workspacebump.go | 75 ---------------------- coderd/workspaceapps_test.go | 19 +++++- coderd/workspaces_test.go | 84 ++++++++++++++++++++++++ 5 files changed, 192 insertions(+), 79 deletions(-) create mode 100644 coderd/httpmw/workspaceactivitybump.go delete mode 100644 coderd/httpmw/workspacebump.go diff --git a/coderd/coderd.go b/coderd/coderd.go index d9c55715e2d9c..bbe67b19f446a 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -201,7 +201,7 @@ func New(options *Options) *API { httpmw.ExtractUserParam(api.Database), // Extracts the from the url httpmw.ExtractWorkspaceAndAgentParam(api.Database), - httpmw.BumpWorkspaceAutoStop(api.Logger, api.Database), + httpmw.ActivityBumpWorkspace(api.Logger, api.Database), ) r.HandleFunc("/*", api.workspaceAppsProxyPath) } @@ -422,7 +422,7 @@ func New(options *Options) *API { apiKeyMiddleware, httpmw.ExtractWorkspaceAgentParam(options.Database), httpmw.ExtractWorkspaceParam(options.Database), - httpmw.BumpWorkspaceAutoStop(api.Logger, options.Database), + httpmw.ActivityBumpWorkspace(api.Logger, options.Database), ) r.Get("/", api.workspaceAgent) r.Get("/pty", api.workspaceAgentPTY) diff --git a/coderd/httpmw/workspaceactivitybump.go b/coderd/httpmw/workspaceactivitybump.go new file mode 100644 index 0000000000000..76d3127e2fe10 --- /dev/null +++ b/coderd/httpmw/workspaceactivitybump.go @@ -0,0 +1,89 @@ +package httpmw + +import ( + "context" + "database/sql" + "errors" + "net/http" + "time" + + "golang.org/x/xerrors" + + "cdr.dev/slog" + "github.com/coder/coder/coderd/database" +) + +// ActivityBumpWorkspace automatically bumps the workspace's auto-off timer +// if it is set to expire soon. +// It must be ran after ExtractWorkspace. +func ActivityBumpWorkspace(log slog.Logger, db database.Store) func(h http.Handler) http.Handler { + log = log.Named("activity_bump") + + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + workspace := WorkspaceParam(r) + log.Debug(r.Context(), "middleware called") + // We run the bump logic asynchronously since the result doesn't + // affect the response. + go func() { + // We cannot use the Request context since the goroutine + // may be around after the request terminates. + // We set a short timeout so if the app is under load, these + // low priority operations fail first. + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + err := db.InTx(func(s database.Store) error { + build, err := s.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspace.ID) + log.Debug(ctx, "build", slog.F("build", build)) + if errors.Is(err, sql.ErrNoRows) { + return nil + } else if err != nil { + return xerrors.Errorf("get latest workspace build: %w", err) + } + + job, err := s.GetProvisionerJobByID(ctx, build.JobID) + if err != nil { + return xerrors.Errorf("get provisioner job: %w", err) + } + + if build.Transition != database.WorkspaceTransitionStart || !job.CompletedAt.Valid { + return nil + } + + if build.Deadline.IsZero() { + // Workspace shutdown is manual + return nil + } + + // We sent bumpThreshold slightly under bumpAmount to minimize DB writes. + const ( + bumpAmount = time.Hour + bumpThreshold = time.Hour - (time.Minute * 10) + ) + + if !build.Deadline.Before(time.Now().Add(bumpThreshold)) { + return nil + } + + newDeadline := time.Now().Add(bumpAmount) + + if err := s.UpdateWorkspaceBuildByID(ctx, database.UpdateWorkspaceBuildByIDParams{ + ID: build.ID, + UpdatedAt: build.UpdatedAt, + ProvisionerState: build.ProvisionerState, + Deadline: newDeadline, + }); err != nil { + return xerrors.Errorf("update workspace build: %w", err) + } + return nil + }) + + if err != nil { + log.Error(ctx, "bump failed", slog.Error(err)) + } + }() + next.ServeHTTP(w, r) + }) + } +} diff --git a/coderd/httpmw/workspacebump.go b/coderd/httpmw/workspacebump.go deleted file mode 100644 index fa8043c28b87e..0000000000000 --- a/coderd/httpmw/workspacebump.go +++ /dev/null @@ -1,75 +0,0 @@ -package httpmw - -import ( - "database/sql" - "errors" - "net/http" - "time" - - "golang.org/x/xerrors" - - "cdr.dev/slog" - "github.com/coder/coder/coderd/database" -) - -// BumpWorkspaceAutoStop automatically bumps the workspace's auto-off timer -// if it is set to expire soon. -// It must be ran after ExtractWorkspace. -func BumpWorkspaceAutoStop(log slog.Logger, db database.Store) func(h http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - workspace := WorkspaceParam(r) - - err := db.InTx(func(s database.Store) error { - build, err := s.GetLatestWorkspaceBuildByWorkspaceID(r.Context(), workspace.ID) - if errors.Is(err, sql.ErrNoRows) { - return nil - } else if err != nil { - return xerrors.Errorf("get latest workspace build: %w", err) - } - - job, err := s.GetProvisionerJobByID(r.Context(), build.JobID) - if err != nil { - return xerrors.Errorf("get provisioner job: %w", err) - } - - if build.Transition != database.WorkspaceTransitionStart || !job.CompletedAt.Valid { - return nil - } - - if build.Deadline.IsZero() { - // Workspace shutdown is manual - return nil - } - - // We sent bumpThreshold slightly under bumpAmount to minimize DB writes. - const ( - bumpAmount = time.Hour - bumpThreshold = time.Hour - time.Minute*10 - ) - - if !build.Deadline.Before(time.Now().Add(bumpThreshold)) { - return nil - } - - newDeadline := time.Now().Add(bumpAmount) - - if err := s.UpdateWorkspaceBuildByID(r.Context(), database.UpdateWorkspaceBuildByIDParams{ - ID: build.ID, - UpdatedAt: build.UpdatedAt, - ProvisionerState: build.ProvisionerState, - Deadline: newDeadline, - }); err != nil { - return xerrors.Errorf("update workspace build: %w", err) - } - return nil - }) - - if err != nil { - log.Error(r.Context(), "auto-bump", slog.Error(err)) - } - - next.ServeHTTP(w, r) - }) - } -} diff --git a/coderd/workspaceapps_test.go b/coderd/workspaceapps_test.go index 831b3761693df..2337095e185d8 100644 --- a/coderd/workspaceapps_test.go +++ b/coderd/workspaceapps_test.go @@ -36,7 +36,7 @@ const ( // setupProxyTest creates a workspace with an agent and some apps. It returns a // codersdk client, the workspace, and the port number the test listener is // running on. -func setupProxyTest(t *testing.T) (*codersdk.Client, uuid.UUID, codersdk.Workspace, uint16) { +func setupProxyTest(t *testing.T, workspaceMutators ...func(*codersdk.CreateWorkspaceRequest)) (*codersdk.Client, uuid.UUID, codersdk.Workspace, uint16) { // #nosec ln, err := net.Listen("tcp", ":0") require.NoError(t, err) @@ -95,7 +95,7 @@ func setupProxyTest(t *testing.T) (*codersdk.Client, uuid.UUID, codersdk.Workspa }) template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID) coderdtest.AwaitTemplateVersionJob(t, client, version.ID) - workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID) + workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID, workspaceMutators...) coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID) agentClient := codersdk.New(client.URL) @@ -208,6 +208,21 @@ func TestWorkspaceAppsProxyPath(t *testing.T) { require.Equal(t, http.StatusOK, resp.StatusCode) }) + t.Run("Proxies", func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + defer cancel() + + resp, err := client.Request(ctx, http.MethodGet, "/@me/"+workspace.Name+"/apps/example/?"+proxyTestAppQuery, nil) + require.NoError(t, err) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, proxyTestAppBody, string(body)) + require.Equal(t, http.StatusOK, resp.StatusCode) + }) + t.Run("ProxyError", func(t *testing.T) { t.Parallel() diff --git a/coderd/workspaces_test.go b/coderd/workspaces_test.go index 11e23ea341c60..6d921050482c1 100644 --- a/coderd/workspaces_test.go +++ b/coderd/workspaces_test.go @@ -1029,6 +1029,90 @@ func TestWorkspaceUpdateAutostart(t *testing.T) { require.Contains(t, coderSDKErr.Message, "Resource not found", "unexpected response code") }) } +func TestWorkspaceActivityBump(t *testing.T) { + t.Parallel() + + ctx := context.Background() + setupActivityTest := func(t *testing.T) (client *codersdk.Client, workspace codersdk.Workspace, assertBumped func(want bool)) { + var ttlMillis int64 = 60 * 1000 + + client, _, workspace, _ = setupProxyTest(t, func(cwr *codersdk.CreateWorkspaceRequest) { + cwr.TTLMillis = &ttlMillis + }) + + // Sanity-check that deadline is near. + workspace, err := client.Workspace(ctx, workspace.ID) + require.NoError(t, err) + require.WithinDuration(t, + time.Now().Add(time.Duration(ttlMillis)*time.Millisecond), + workspace.LatestBuild.Deadline.Time, testutil.WaitShort, + ) + firstDeadline := workspace.LatestBuild.Deadline.Time + + return client, workspace, func(want bool) { + if !want { + time.Sleep(testutil.IntervalMedium) + workspace, err = client.Workspace(ctx, workspace.ID) + require.NoError(t, err) + require.Equal(t, workspace.LatestBuild.Deadline.Time, firstDeadline) + return + } + + // The Deadline bump occurs asynchronously. + require.Eventuallyf(t, + func() bool { + workspace, err = client.Workspace(ctx, workspace.ID) + require.NoError(t, err) + return workspace.LatestBuild.Deadline.Time != firstDeadline + }, + testutil.WaitShort, testutil.IntervalFast, + "deadline %v never updated", firstDeadline, + ) + + require.WithinDuration(t, time.Now().Add(time.Hour), workspace.LatestBuild.Deadline.Time, time.Second) + } + } + + t.Run("Apps", func(t *testing.T) { + t.Parallel() + + client, workspace, assertBumped := setupActivityTest(t) + + // A request to the /apps/ endpoint extends the deadline an hour. + // The particular app doesn't matter. The deadline is extended + // regardless of error state. + resp, err := client.Request(ctx, http.MethodGet, "/@me/"+workspace.Name+"/apps/example", nil) + require.NoError(t, err) + resp.Body.Close() + assertBumped(true) + }) + + t.Run("Dial", func(t *testing.T) { + t.Parallel() + + client, workspace, assertBumped := setupActivityTest(t) + + resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID) + conn, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil) + require.NoError(t, err) + _ = conn.Close() + + assertBumped(true) + }) + + t.Run("NoBump", func(t *testing.T) { + t.Parallel() + + client, workspace, assertBumped := setupActivityTest(t) + + // Doing some inactive operation like retrieving resources must not + // bump the deadline. + _, err := client.WorkspaceResourcesByBuild(ctx, workspace.LatestBuild.ID) + require.NoError(t, err) + + assertBumped(false) + }) +} func TestWorkspaceUpdateTTL(t *testing.T) { t.Parallel() From 52f68e668f65d3118b304276ffe45be7700bda8e Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 20 Sep 2022 02:33:49 +0000 Subject: [PATCH 3/9] The point at which I realized I'm doing this wrong --- coderd/httpmw/workspaceactivitybump.go | 103 ++++++++++++++----------- 1 file changed, 57 insertions(+), 46 deletions(-) diff --git a/coderd/httpmw/workspaceactivitybump.go b/coderd/httpmw/workspaceactivitybump.go index 76d3127e2fe10..dc6280ff9296b 100644 --- a/coderd/httpmw/workspaceactivitybump.go +++ b/coderd/httpmw/workspaceactivitybump.go @@ -26,62 +26,73 @@ func ActivityBumpWorkspace(log slog.Logger, db database.Store) func(h http.Handl // We run the bump logic asynchronously since the result doesn't // affect the response. go func() { - // We cannot use the Request context since the goroutine - // may be around after the request terminates. - // We set a short timeout so if the app is under load, these - // low priority operations fail first. - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() + bump := func() { + // We cannot use the Request context since the goroutine + // may be around after the request terminates. + // We set a short timeout so if the app is under load, these + // low priority operations fail first. + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() - err := db.InTx(func(s database.Store) error { - build, err := s.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspace.ID) - log.Debug(ctx, "build", slog.F("build", build)) - if errors.Is(err, sql.ErrNoRows) { - return nil - } else if err != nil { - return xerrors.Errorf("get latest workspace build: %w", err) - } + err := db.InTx(func(s database.Store) error { + build, err := s.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspace.ID) + log.Debug(ctx, "build", slog.F("build", build)) + if errors.Is(err, sql.ErrNoRows) { + return nil + } else if err != nil { + return xerrors.Errorf("get latest workspace build: %w", err) + } - job, err := s.GetProvisionerJobByID(ctx, build.JobID) - if err != nil { - return xerrors.Errorf("get provisioner job: %w", err) - } + job, err := s.GetProvisionerJobByID(ctx, build.JobID) + if err != nil { + return xerrors.Errorf("get provisioner job: %w", err) + } - if build.Transition != database.WorkspaceTransitionStart || !job.CompletedAt.Valid { - return nil - } + if build.Transition != database.WorkspaceTransitionStart || !job.CompletedAt.Valid { + return nil + } - if build.Deadline.IsZero() { - // Workspace shutdown is manual - return nil - } + if build.Deadline.IsZero() { + // Workspace shutdown is manual + return nil + } - // We sent bumpThreshold slightly under bumpAmount to minimize DB writes. - const ( - bumpAmount = time.Hour - bumpThreshold = time.Hour - (time.Minute * 10) - ) + // We sent bumpThreshold slightly under bumpAmount to minimize DB writes. + const ( + bumpAmount = time.Hour + bumpThreshold = time.Hour - (time.Minute * 10) + ) - if !build.Deadline.Before(time.Now().Add(bumpThreshold)) { - return nil - } + if !build.Deadline.Before(time.Now().Add(bumpThreshold)) { + return nil + } - newDeadline := time.Now().Add(bumpAmount) + newDeadline := time.Now().Add(bumpAmount) - if err := s.UpdateWorkspaceBuildByID(ctx, database.UpdateWorkspaceBuildByIDParams{ - ID: build.ID, - UpdatedAt: build.UpdatedAt, - ProvisionerState: build.ProvisionerState, - Deadline: newDeadline, - }); err != nil { - return xerrors.Errorf("update workspace build: %w", err) - } - return nil - }) + if err := s.UpdateWorkspaceBuildByID(ctx, database.UpdateWorkspaceBuildByIDParams{ + ID: build.ID, + UpdatedAt: build.UpdatedAt, + ProvisionerState: build.ProvisionerState, + Deadline: newDeadline, + }); err != nil { + return xerrors.Errorf("update workspace build: %w", err) + } + return nil + }) - if err != nil { - log.Error(ctx, "bump failed", slog.Error(err)) + if err != nil { + log.Error(ctx, "bump failed", slog.Error(err)) + } else { + log.Debug( + ctx, "bumped deadline from activity", + slog.F("workspace_id", workspace.ID), + ) + } } + // For long running connections (e.g. web terminal), we need + // to bump periodically + // ticker := time.NewTicker(time.Minute) + bump() }() next.ServeHTTP(w, r) }) From 24e42e17422588e7ad36e88ee00846e9f01d23de Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 20 Sep 2022 03:24:10 +0000 Subject: [PATCH 4/9] Move logic to agent + frontend --- coderd/activitybump.go | 79 ++++++++++++++ coderd/activitybump_test.go | 93 ++++++++++++++++ coderd/coderd.go | 3 +- coderd/httpmw/workspaceactivitybump.go | 100 ------------------ coderd/workspaceagents.go | 2 + coderd/workspaceapps_test.go | 5 +- coderd/workspaces_test.go | 84 --------------- codersdk/workspaceagents.go | 2 +- .../WorkspaceScheduleForm.tsx | 3 +- 9 files changed, 182 insertions(+), 189 deletions(-) create mode 100644 coderd/activitybump.go create mode 100644 coderd/activitybump_test.go delete mode 100644 coderd/httpmw/workspaceactivitybump.go diff --git a/coderd/activitybump.go b/coderd/activitybump.go new file mode 100644 index 0000000000000..35f4b725c94d9 --- /dev/null +++ b/coderd/activitybump.go @@ -0,0 +1,79 @@ +package coderd + +import ( + "context" + "database/sql" + "errors" + "time" + + "golang.org/x/xerrors" + + "cdr.dev/slog" + "github.com/coder/coder/coderd/database" +) + +// activityBumpWorkspace automatically bumps the workspace's auto-off timer +// if it is set to expire soon. +func activityBumpWorkspace(log slog.Logger, db database.Store, workspace database.Workspace) { + // We cannot use the Request context since the goroutine + // may be around after the request terminates. + // We set a short timeout so if the app is under load, these + // low priority operations fail first. + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + err := db.InTx(func(s database.Store) error { + build, err := s.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspace.ID) + log.Debug(ctx, "build", slog.F("build", build)) + if errors.Is(err, sql.ErrNoRows) { + return nil + } else if err != nil { + return xerrors.Errorf("get latest workspace build: %w", err) + } + + job, err := s.GetProvisionerJobByID(ctx, build.JobID) + if err != nil { + return xerrors.Errorf("get provisioner job: %w", err) + } + + if build.Transition != database.WorkspaceTransitionStart || !job.CompletedAt.Valid { + return nil + } + + if build.Deadline.IsZero() { + // Workspace shutdown is manual + return nil + } + + // We sent bumpThreshold slightly under bumpAmount to minimize DB writes. + const ( + bumpAmount = time.Hour + bumpThreshold = time.Hour - (time.Minute * 10) + ) + + if !build.Deadline.Before(time.Now().Add(bumpThreshold)) { + return nil + } + + newDeadline := time.Now().Add(bumpAmount) + + if err := s.UpdateWorkspaceBuildByID(ctx, database.UpdateWorkspaceBuildByIDParams{ + ID: build.ID, + UpdatedAt: build.UpdatedAt, + ProvisionerState: build.ProvisionerState, + Deadline: newDeadline, + }); err != nil { + return xerrors.Errorf("update workspace build: %w", err) + } + return nil + }) + + if err != nil { + log.Error(ctx, "bump failed", slog.Error(err)) + } else { + log.Debug( + ctx, "bumped deadline from activity", + slog.F("workspace_id", workspace.ID), + ) + } +} diff --git a/coderd/activitybump_test.go b/coderd/activitybump_test.go new file mode 100644 index 0000000000000..2246e3dc1bf15 --- /dev/null +++ b/coderd/activitybump_test.go @@ -0,0 +1,93 @@ +package coderd_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "cdr.dev/slog/sloggers/slogtest" + + "github.com/coder/coder/coderd/coderdtest" + "github.com/coder/coder/codersdk" + "github.com/coder/coder/testutil" +) + +func TestWorkspaceActivityBump(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + setupActivityTest := func(t *testing.T) (client *codersdk.Client, workspace codersdk.Workspace, assertBumped func(want bool)) { + var ttlMillis int64 = 60 * 1000 + + client, _, workspace, _ = setupProxyTest(t, func(cwr *codersdk.CreateWorkspaceRequest) { + cwr.TTLMillis = &ttlMillis + }) + + // Sanity-check that deadline is near. + workspace, err := client.Workspace(ctx, workspace.ID) + require.NoError(t, err) + require.WithinDuration(t, + time.Now().Add(time.Duration(ttlMillis)*time.Millisecond), + workspace.LatestBuild.Deadline.Time, testutil.WaitShort, + ) + firstDeadline := workspace.LatestBuild.Deadline.Time + + _ = coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID) + + return client, workspace, func(want bool) { + if !want { + time.Sleep(testutil.IntervalMedium) + workspace, err = client.Workspace(ctx, workspace.ID) + require.NoError(t, err) + require.Equal(t, workspace.LatestBuild.Deadline.Time, firstDeadline) + return + } + + // The Deadline bump occurs asynchronously. + require.Eventuallyf(t, + func() bool { + workspace, err = client.Workspace(ctx, workspace.ID) + require.NoError(t, err) + return workspace.LatestBuild.Deadline.Time != firstDeadline + }, + testutil.WaitShort, testutil.IntervalFast, + "deadline %v never updated", firstDeadline, + ) + + require.WithinDuration(t, time.Now().Add(time.Hour), workspace.LatestBuild.Deadline.Time, time.Second) + } + } + + t.Run("Dial", func(t *testing.T) { + t.Parallel() + + client, workspace, assertBumped := setupActivityTest(t) + + resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID) + conn, err := client.DialWorkspaceAgentTailnet(ctx, slogtest.Make(t, nil), resources[0].Agents[0].ID) + require.NoError(t, err) + defer conn.Close() + + sshConn, err := conn.SSHClient() + require.NoError(t, err) + _ = sshConn.Close() + + assertBumped(true) + }) + + t.Run("NoBump", func(t *testing.T) { + t.Parallel() + + client, workspace, assertBumped := setupActivityTest(t) + + // Doing some inactive operation like retrieving resources must not + // bump the deadline. + _, err := client.WorkspaceResourcesByBuild(ctx, workspace.LatestBuild.ID) + require.NoError(t, err) + + assertBumped(false) + }) +} diff --git a/coderd/coderd.go b/coderd/coderd.go index bbe67b19f446a..b49fdaf6eaba6 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -201,7 +201,7 @@ func New(options *Options) *API { httpmw.ExtractUserParam(api.Database), // Extracts the from the url httpmw.ExtractWorkspaceAndAgentParam(api.Database), - httpmw.ActivityBumpWorkspace(api.Logger, api.Database), + // httpmw.ActivityBumpWorkspace(api.Logger, api.Database), ) r.HandleFunc("/*", api.workspaceAppsProxyPath) } @@ -422,7 +422,6 @@ func New(options *Options) *API { apiKeyMiddleware, httpmw.ExtractWorkspaceAgentParam(options.Database), httpmw.ExtractWorkspaceParam(options.Database), - httpmw.ActivityBumpWorkspace(api.Logger, options.Database), ) r.Get("/", api.workspaceAgent) r.Get("/pty", api.workspaceAgentPTY) diff --git a/coderd/httpmw/workspaceactivitybump.go b/coderd/httpmw/workspaceactivitybump.go deleted file mode 100644 index dc6280ff9296b..0000000000000 --- a/coderd/httpmw/workspaceactivitybump.go +++ /dev/null @@ -1,100 +0,0 @@ -package httpmw - -import ( - "context" - "database/sql" - "errors" - "net/http" - "time" - - "golang.org/x/xerrors" - - "cdr.dev/slog" - "github.com/coder/coder/coderd/database" -) - -// ActivityBumpWorkspace automatically bumps the workspace's auto-off timer -// if it is set to expire soon. -// It must be ran after ExtractWorkspace. -func ActivityBumpWorkspace(log slog.Logger, db database.Store) func(h http.Handler) http.Handler { - log = log.Named("activity_bump") - - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - workspace := WorkspaceParam(r) - log.Debug(r.Context(), "middleware called") - // We run the bump logic asynchronously since the result doesn't - // affect the response. - go func() { - bump := func() { - // We cannot use the Request context since the goroutine - // may be around after the request terminates. - // We set a short timeout so if the app is under load, these - // low priority operations fail first. - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - - err := db.InTx(func(s database.Store) error { - build, err := s.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspace.ID) - log.Debug(ctx, "build", slog.F("build", build)) - if errors.Is(err, sql.ErrNoRows) { - return nil - } else if err != nil { - return xerrors.Errorf("get latest workspace build: %w", err) - } - - job, err := s.GetProvisionerJobByID(ctx, build.JobID) - if err != nil { - return xerrors.Errorf("get provisioner job: %w", err) - } - - if build.Transition != database.WorkspaceTransitionStart || !job.CompletedAt.Valid { - return nil - } - - if build.Deadline.IsZero() { - // Workspace shutdown is manual - return nil - } - - // We sent bumpThreshold slightly under bumpAmount to minimize DB writes. - const ( - bumpAmount = time.Hour - bumpThreshold = time.Hour - (time.Minute * 10) - ) - - if !build.Deadline.Before(time.Now().Add(bumpThreshold)) { - return nil - } - - newDeadline := time.Now().Add(bumpAmount) - - if err := s.UpdateWorkspaceBuildByID(ctx, database.UpdateWorkspaceBuildByIDParams{ - ID: build.ID, - UpdatedAt: build.UpdatedAt, - ProvisionerState: build.ProvisionerState, - Deadline: newDeadline, - }); err != nil { - return xerrors.Errorf("update workspace build: %w", err) - } - return nil - }) - - if err != nil { - log.Error(ctx, "bump failed", slog.Error(err)) - } else { - log.Debug( - ctx, "bumped deadline from activity", - slog.F("workspace_id", workspace.ID), - ) - } - } - // For long running connections (e.g. web terminal), we need - // to bump periodically - // ticker := time.NewTicker(time.Minute) - bump() - }() - next.ServeHTTP(w, r) - }) - } -} diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 57c90c5b84479..e04046f2762bd 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -616,6 +616,8 @@ func (api *API) workspaceAgentReportStats(rw http.ResponseWriter, r *http.Reques ) if updateDB { + go activityBumpWorkspace(api.Logger.Named("activty_bump"), api.Database, workspace) + lastReport = rep _, err = api.Database.InsertAgentStat(ctx, database.InsertAgentStatParams{ diff --git a/coderd/workspaceapps_test.go b/coderd/workspaceapps_test.go index 2337095e185d8..c20c9c67b4330 100644 --- a/coderd/workspaceapps_test.go +++ b/coderd/workspaceapps_test.go @@ -58,7 +58,9 @@ func setupProxyTest(t *testing.T, workspaceMutators ...func(*codersdk.CreateWork require.True(t, ok) client := coderdtest.New(t, &coderdtest.Options{ - IncludeProvisionerDaemon: true, + IncludeProvisionerDaemon: true, + AgentStatsRefreshInterval: time.Millisecond * 100, + MetricsCacheRefreshInterval: time.Millisecond * 100, }) user := coderdtest.CreateFirstUser(t, client) authToken := uuid.NewString() @@ -104,6 +106,7 @@ func setupProxyTest(t *testing.T, workspaceMutators ...func(*codersdk.CreateWork FetchMetadata: agentClient.WorkspaceAgentMetadata, CoordinatorDialer: agentClient.ListenWorkspaceAgentTailnet, Logger: slogtest.Make(t, nil).Named("agent"), + StatsReporter: agentClient.AgentReportStats, }) t.Cleanup(func() { _ = agentCloser.Close() diff --git a/coderd/workspaces_test.go b/coderd/workspaces_test.go index 6d921050482c1..11e23ea341c60 100644 --- a/coderd/workspaces_test.go +++ b/coderd/workspaces_test.go @@ -1029,90 +1029,6 @@ func TestWorkspaceUpdateAutostart(t *testing.T) { require.Contains(t, coderSDKErr.Message, "Resource not found", "unexpected response code") }) } -func TestWorkspaceActivityBump(t *testing.T) { - t.Parallel() - - ctx := context.Background() - setupActivityTest := func(t *testing.T) (client *codersdk.Client, workspace codersdk.Workspace, assertBumped func(want bool)) { - var ttlMillis int64 = 60 * 1000 - - client, _, workspace, _ = setupProxyTest(t, func(cwr *codersdk.CreateWorkspaceRequest) { - cwr.TTLMillis = &ttlMillis - }) - - // Sanity-check that deadline is near. - workspace, err := client.Workspace(ctx, workspace.ID) - require.NoError(t, err) - require.WithinDuration(t, - time.Now().Add(time.Duration(ttlMillis)*time.Millisecond), - workspace.LatestBuild.Deadline.Time, testutil.WaitShort, - ) - firstDeadline := workspace.LatestBuild.Deadline.Time - - return client, workspace, func(want bool) { - if !want { - time.Sleep(testutil.IntervalMedium) - workspace, err = client.Workspace(ctx, workspace.ID) - require.NoError(t, err) - require.Equal(t, workspace.LatestBuild.Deadline.Time, firstDeadline) - return - } - - // The Deadline bump occurs asynchronously. - require.Eventuallyf(t, - func() bool { - workspace, err = client.Workspace(ctx, workspace.ID) - require.NoError(t, err) - return workspace.LatestBuild.Deadline.Time != firstDeadline - }, - testutil.WaitShort, testutil.IntervalFast, - "deadline %v never updated", firstDeadline, - ) - - require.WithinDuration(t, time.Now().Add(time.Hour), workspace.LatestBuild.Deadline.Time, time.Second) - } - } - - t.Run("Apps", func(t *testing.T) { - t.Parallel() - - client, workspace, assertBumped := setupActivityTest(t) - - // A request to the /apps/ endpoint extends the deadline an hour. - // The particular app doesn't matter. The deadline is extended - // regardless of error state. - resp, err := client.Request(ctx, http.MethodGet, "/@me/"+workspace.Name+"/apps/example", nil) - require.NoError(t, err) - resp.Body.Close() - assertBumped(true) - }) - - t.Run("Dial", func(t *testing.T) { - t.Parallel() - - client, workspace, assertBumped := setupActivityTest(t) - - resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID) - conn, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil) - require.NoError(t, err) - _ = conn.Close() - - assertBumped(true) - }) - - t.Run("NoBump", func(t *testing.T) { - t.Parallel() - - client, workspace, assertBumped := setupActivityTest(t) - - // Doing some inactive operation like retrieving resources must not - // bump the deadline. - _, err := client.WorkspaceResourcesByBuild(ctx, workspace.LatestBuild.ID) - require.NoError(t, err) - - assertBumped(false) - }) -} func TestWorkspaceUpdateTTL(t *testing.T) { t.Parallel() diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index 2117de03c6ce3..e46eec1cbdf8c 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -285,8 +285,8 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg return } if err != nil { + // WARN: closing here may lead to nhooyr websocket panicking. logger.Debug(ctx, "failed to dial", slog.Error(err)) - _ = ws.Close(websocket.StatusAbnormalClosure, "") continue } sendNode, errChan := tailnet.ServeCoordinator(websocket.NetConn(ctx, ws, websocket.MessageBinary), func(node []*tailnet.Node) error { diff --git a/site/src/components/WorkspaceScheduleForm/WorkspaceScheduleForm.tsx b/site/src/components/WorkspaceScheduleForm/WorkspaceScheduleForm.tsx index 1b53583ed141c..6acdbf3ab22a8 100644 --- a/site/src/components/WorkspaceScheduleForm/WorkspaceScheduleForm.tsx +++ b/site/src/components/WorkspaceScheduleForm/WorkspaceScheduleForm.tsx @@ -55,7 +55,8 @@ export const Language = { timezoneLabel: "Timezone", ttlLabel: "Time until shutdown (hours)", ttlCausesShutdownHelperText: "Your workspace will shut down", - ttlCausesShutdownAfterStart: "after its next start", + ttlCausesShutdownAfterStart: `after its next start. We delay shutdown by an +hour whenever we detect activity`, ttlCausesNoShutdownHelperText: "Your workspace will not automatically shut down.", formTitle: "Workspace schedule", startSection: "Start", From bd152432f8007a9581293562d8dade2caa04a1de Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 20 Sep 2022 03:27:34 +0000 Subject: [PATCH 5/9] fixup! Move logic to agent + frontend --- coderd/activitybump.go | 2 -- coderd/activitybump_test.go | 2 +- coderd/coderd.go | 1 - coderd/workspaceagents.go | 2 +- coderd/workspaceapps_test.go | 15 --------------- 5 files changed, 2 insertions(+), 20 deletions(-) diff --git a/coderd/activitybump.go b/coderd/activitybump.go index 35f4b725c94d9..1f22175f20952 100644 --- a/coderd/activitybump.go +++ b/coderd/activitybump.go @@ -15,8 +15,6 @@ import ( // activityBumpWorkspace automatically bumps the workspace's auto-off timer // if it is set to expire soon. func activityBumpWorkspace(log slog.Logger, db database.Store, workspace database.Workspace) { - // We cannot use the Request context since the goroutine - // may be around after the request terminates. // We set a short timeout so if the app is under load, these // low priority operations fail first. ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) diff --git a/coderd/activitybump_test.go b/coderd/activitybump_test.go index 2246e3dc1bf15..705b8312a99bb 100644 --- a/coderd/activitybump_test.go +++ b/coderd/activitybump_test.go @@ -83,7 +83,7 @@ func TestWorkspaceActivityBump(t *testing.T) { client, workspace, assertBumped := setupActivityTest(t) - // Doing some inactive operation like retrieving resources must not + // Benign operations like retrieving resources must not // bump the deadline. _, err := client.WorkspaceResourcesByBuild(ctx, workspace.LatestBuild.ID) require.NoError(t, err) diff --git a/coderd/coderd.go b/coderd/coderd.go index b49fdaf6eaba6..607f15ff7931b 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -201,7 +201,6 @@ func New(options *Options) *API { httpmw.ExtractUserParam(api.Database), // Extracts the from the url httpmw.ExtractWorkspaceAndAgentParam(api.Database), - // httpmw.ActivityBumpWorkspace(api.Logger, api.Database), ) r.HandleFunc("/*", api.workspaceAppsProxyPath) } diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index e04046f2762bd..219fbb78b7d00 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -616,7 +616,7 @@ func (api *API) workspaceAgentReportStats(rw http.ResponseWriter, r *http.Reques ) if updateDB { - go activityBumpWorkspace(api.Logger.Named("activty_bump"), api.Database, workspace) + go activityBumpWorkspace(api.Logger.Named("activity_bump"), api.Database, workspace) lastReport = rep diff --git a/coderd/workspaceapps_test.go b/coderd/workspaceapps_test.go index c20c9c67b4330..164ceb2b15de6 100644 --- a/coderd/workspaceapps_test.go +++ b/coderd/workspaceapps_test.go @@ -211,21 +211,6 @@ func TestWorkspaceAppsProxyPath(t *testing.T) { require.Equal(t, http.StatusOK, resp.StatusCode) }) - t.Run("Proxies", func(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) - defer cancel() - - resp, err := client.Request(ctx, http.MethodGet, "/@me/"+workspace.Name+"/apps/example/?"+proxyTestAppQuery, nil) - require.NoError(t, err) - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - require.NoError(t, err) - require.Equal(t, proxyTestAppBody, string(body)) - require.Equal(t, http.StatusOK, resp.StatusCode) - }) - t.Run("ProxyError", func(t *testing.T) { t.Parallel() From ef8e29db63933e85be6802f479312f3b9e7f7f9c Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 20 Sep 2022 16:52:02 +0000 Subject: [PATCH 6/9] Address review comments --- coderd/activitybump.go | 7 +++---- codersdk/workspaceagents.go | 2 -- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/coderd/activitybump.go b/coderd/activitybump.go index 1f22175f20952..91a5ede3bf6f1 100644 --- a/coderd/activitybump.go +++ b/coderd/activitybump.go @@ -17,7 +17,7 @@ import ( func activityBumpWorkspace(log slog.Logger, db database.Store, workspace database.Workspace) { // We set a short timeout so if the app is under load, these // low priority operations fail first. - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer cancel() err := db.InTx(func(s database.Store) error { @@ -53,11 +53,11 @@ func activityBumpWorkspace(log slog.Logger, db database.Store, workspace databas return nil } - newDeadline := time.Now().Add(bumpAmount) + newDeadline := database.Now().Add(bumpAmount) if err := s.UpdateWorkspaceBuildByID(ctx, database.UpdateWorkspaceBuildByIDParams{ ID: build.ID, - UpdatedAt: build.UpdatedAt, + UpdatedAt: database.Now(), ProvisionerState: build.ProvisionerState, Deadline: newDeadline, }); err != nil { @@ -65,7 +65,6 @@ func activityBumpWorkspace(log slog.Logger, db database.Store, workspace databas } return nil }) - if err != nil { log.Error(ctx, "bump failed", slog.Error(err)) } else { diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index e46eec1cbdf8c..bf1f58eceafac 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -281,11 +281,9 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg CompressionMode: websocket.CompressionDisabled, }) if errors.Is(err, context.Canceled) { - _ = ws.Close(websocket.StatusAbnormalClosure, "") return } if err != nil { - // WARN: closing here may lead to nhooyr websocket panicking. logger.Debug(ctx, "failed to dial", slog.Error(err)) continue } From f6c555cbdc661e21fec38633351335361141746d Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 20 Sep 2022 16:53:19 +0000 Subject: [PATCH 7/9] fixup! Address review comments --- coderd/activitybump.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/coderd/activitybump.go b/coderd/activitybump.go index 91a5ede3bf6f1..7277affe0971c 100644 --- a/coderd/activitybump.go +++ b/coderd/activitybump.go @@ -66,7 +66,11 @@ func activityBumpWorkspace(log slog.Logger, db database.Store, workspace databas return nil }) if err != nil { - log.Error(ctx, "bump failed", slog.Error(err)) + log.Error( + ctx, "bump failed", + slog.Error(err), + slog.F("workspace_id", workspace.ID), + ) } else { log.Debug( ctx, "bumped deadline from activity", From 60a271571507f5fe5da7c96b1b91210294a81dcf Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 20 Sep 2022 18:22:51 +0000 Subject: [PATCH 8/9] Address PR comments (round 2) --- coderd/activitybump.go | 1 - coderd/activitybump_test.go | 3 ++- .../WorkspaceScheduleForm/WorkspaceScheduleForm.tsx | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/coderd/activitybump.go b/coderd/activitybump.go index 7277affe0971c..fa90f7d275daf 100644 --- a/coderd/activitybump.go +++ b/coderd/activitybump.go @@ -22,7 +22,6 @@ func activityBumpWorkspace(log slog.Logger, db database.Store, workspace databas err := db.InTx(func(s database.Store) error { build, err := s.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspace.ID) - log.Debug(ctx, "build", slog.F("build", build)) if errors.Is(err, sql.ErrNoRows) { return nil } else if err != nil { diff --git a/coderd/activitybump_test.go b/coderd/activitybump_test.go index 705b8312a99bb..167c85eb11287 100644 --- a/coderd/activitybump_test.go +++ b/coderd/activitybump_test.go @@ -10,6 +10,7 @@ import ( "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/coderd/coderdtest" + "github.com/coder/coder/coderd/database" "github.com/coder/coder/codersdk" "github.com/coder/coder/testutil" ) @@ -57,7 +58,7 @@ func TestWorkspaceActivityBump(t *testing.T) { "deadline %v never updated", firstDeadline, ) - require.WithinDuration(t, time.Now().Add(time.Hour), workspace.LatestBuild.Deadline.Time, time.Second) + require.WithinDuration(t, database.Now().Add(time.Hour), workspace.LatestBuild.Deadline.Time, time.Second) } } diff --git a/site/src/components/WorkspaceScheduleForm/WorkspaceScheduleForm.tsx b/site/src/components/WorkspaceScheduleForm/WorkspaceScheduleForm.tsx index 6acdbf3ab22a8..49f9aec931975 100644 --- a/site/src/components/WorkspaceScheduleForm/WorkspaceScheduleForm.tsx +++ b/site/src/components/WorkspaceScheduleForm/WorkspaceScheduleForm.tsx @@ -55,8 +55,8 @@ export const Language = { timezoneLabel: "Timezone", ttlLabel: "Time until shutdown (hours)", ttlCausesShutdownHelperText: "Your workspace will shut down", - ttlCausesShutdownAfterStart: `after its next start. We delay shutdown by an -hour whenever we detect activity`, + ttlCausesShutdownAfterStart: + "after its next start. We delay shutdown by an hour whenever we detect activity", ttlCausesNoShutdownHelperText: "Your workspace will not automatically shut down.", formTitle: "Workspace schedule", startSection: "Start", From 0c48a6719106b2d9b2e1ede0e131a4e5b83c2fb8 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 20 Sep 2022 18:28:25 +0000 Subject: [PATCH 9/9] fixup! Address PR comments (round 2) --- coderd/activitybump_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/coderd/activitybump_test.go b/coderd/activitybump_test.go index 167c85eb11287..39c2f5fc6c5c8 100644 --- a/coderd/activitybump_test.go +++ b/coderd/activitybump_test.go @@ -40,6 +40,12 @@ func TestWorkspaceActivityBump(t *testing.T) { return client, workspace, func(want bool) { if !want { + // It is difficult to test the absence of a call in a non-racey + // way. In general, it is difficult for the API to generate + // false positive activity since Agent networking event + // is required. The Activity Bump behavior is also coupled with + // Last Used, so it would be obvious to the user if we + // are falsely recognizing activity. time.Sleep(testutil.IntervalMedium) workspace, err = client.Workspace(ctx, workspace.ID) require.NoError(t, err)