Skip to content

fix: wsbuilder: complete job and mark workspace deleted if no provisioners available #18465

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
+430 −86
Open

fix: wsbuilder: complete job and mark workspace deleted if no provisioners available #18465

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9d3ea6f
fix: wsbuilder: complete job and mark workspace deleted if no provisi…
johnstcn Jun 19, 2025
d46f156
check daemon last seen at
johnstcn Jun 20, 2025
dba9fce
allow orphaning if last build was cancelled
johnstcn Jun 20, 2025
f312292
re-fetch provisioner job before returning
johnstcn Jun 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 17 additions & 9 deletions cli/delete_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"io"
"net/http"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -51,28 +52,35 @@ func TestDelete(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
owner := coderdtest.CreateFirstUser(t, client)
version := coderdtest.CreateTemplateVersion(t, client, owner.OrganizationID, nil)
coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
template := coderdtest.CreateTemplate(t, client, owner.OrganizationID, version.ID)
workspace := coderdtest.CreateWorkspace(t, client, template.ID)
coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
templateAdmin, _ := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID, rbac.RoleTemplateAdmin())
version := coderdtest.CreateTemplateVersion(t, templateAdmin, owner.OrganizationID, nil)
coderdtest.AwaitTemplateVersionJobCompleted(t, templateAdmin, version.ID)
template := coderdtest.CreateTemplate(t, templateAdmin, owner.OrganizationID, version.ID)
workspace := coderdtest.CreateWorkspace(t, templateAdmin, template.ID)
coderdtest.AwaitWorkspaceBuildJobCompleted(t, templateAdmin, workspace.LatestBuild.ID)

ctx := testutil.Context(t, testutil.WaitShort)
inv, root := clitest.New(t, "delete", workspace.Name, "-y", "--orphan")
clitest.SetupConfig(t, templateAdmin, root)

//nolint:gocritic // Deleting orphaned workspaces requires an admin.
clitest.SetupConfig(t, client, root)
doneChan := make(chan struct{})
pty := ptytest.New(t).Attach(inv)
inv.Stderr = pty.Output()
go func() {
defer close(doneChan)
err := inv.Run()
err := inv.WithContext(ctx).Run()
// When running with the race detector on, we sometimes get an EOF.
if err != nil {
assert.ErrorIs(t, err, io.EOF)
}
}()
pty.ExpectMatch("has been deleted")
<-doneChan
testutil.TryReceive(ctx, t, doneChan)

_, err := client.Workspace(ctx, workspace.ID)
require.Error(t, err)
cerr := coderdtest.SDKError(t, err)
require.Equal(t, http.StatusGone, cerr.StatusCode())
})

// Super orphaned, as the workspace doesn't even have a user.
Expand Down
2 changes: 1 addition & 1 deletion coderd/database/dbmem/dbmem.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4464,7 +4464,7 @@ func (q *FakeQuerier) GetProvisionerDaemons(_ context.Context) ([]database.Provi
defer q.mutex.RUnlock()

if len(q.provisionerDaemons) == 0 {
return nil, sql.ErrNoRows
return []database.ProvisionerDaemon{}, nil
Copy link
Preview

Copilot AI Jun 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nitpick] Update or add a comment above this line to note that GetProvisionerDaemons now returns an empty slice instead of sql.ErrNoRows to maintain consistency with other methods.

Copilot uses AI. Check for mistakes.

}
// copy the data so that the caller can't manipulate any data inside dbmem
// after returning
Expand Down
8 changes: 4 additions & 4 deletions coderd/workspacebuilds.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,10 @@ func (api *API) postWorkspaceBuilds(rw http.ResponseWriter, r *http.Request) {
return
}

var qpr database.GetProvisionerJobsByIDsWithQueuePositionRow
if provisionerJob != nil {
qpr.ProvisionerJob = *provisionerJob
qpr.QueuePosition = 0
Comment on lines +426 to +429
Copy link
Preview

Copilot AI Jun 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nitpick] Consider renaming qpr to a more descriptive name like provJobRow to improve code readability.

Suggested change
var qpr database.GetProvisionerJobsByIDsWithQueuePositionRow
if provisionerJob != nil {
qpr.ProvisionerJob = *provisionerJob
qpr.QueuePosition = 0
var provJobRow database.GetProvisionerJobsByIDsWithQueuePositionRow
if provisionerJob != nil {
provJobRow.ProvisionerJob = *provisionerJob
provJobRow.QueuePosition = 0

Copilot uses AI. Check for mistakes.

if err := provisionerjobs.PostJob(api.Pubsub, *provisionerJob); err != nil {
// Client probably doesn't care about this error, so just log it.
api.Logger.Error(ctx, "failed to post provisioner job to pubsub", slog.Error(err))
Expand All @@ -433,10 +436,7 @@ func (api *API) postWorkspaceBuilds(rw http.ResponseWriter, r *http.Request) {
apiBuild, err := api.convertWorkspaceBuild(
*workspaceBuild,
workspace,
database.GetProvisionerJobsByIDsWithQueuePositionRow{
ProvisionerJob: *provisionerJob,
QueuePosition: 0,
},
qpr,
[]database.WorkspaceResource{},
[]database.WorkspaceResourceMetadatum{},
[]database.WorkspaceAgent{},
Expand Down
159 changes: 130 additions & 29 deletions coderd/workspacebuilds_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/coder/coder/v2/coderd/coderdtest/oidctest"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbauthz"
"github.com/coder/coder/v2/coderd/database/dbfake"
"github.com/coder/coder/v2/coderd/database/dbgen"
"github.com/coder/coder/v2/coderd/database/dbtestutil"
"github.com/coder/coder/v2/coderd/database/dbtime"
Expand Down Expand Up @@ -371,42 +372,142 @@ func TestWorkspaceBuildsProvisionerState(t *testing.T) {

t.Run("Orphan", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
first := coderdtest.CreateFirstUser(t, client)

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
Comment on lines -374 to -378
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

review: refactored to use dbfake instead


version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, nil)
template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID)
coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
t.Run("WithoutDelete", func(t *testing.T) {
t.Parallel()
client, store := coderdtest.NewWithDatabase(t, nil)
first := coderdtest.CreateFirstUser(t, client)
templateAdmin, templateAdminUser := coderdtest.CreateAnotherUser(t, client, first.OrganizationID, rbac.RoleTemplateAdmin())

r := dbfake.WorkspaceBuild(t, store, database.WorkspaceTable{
OwnerID: templateAdminUser.ID,
OrganizationID: first.OrganizationID,
}).Do()

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()

// Trying to orphan without delete transition fails.
_, err := templateAdmin.CreateWorkspaceBuild(ctx, r.Workspace.ID, codersdk.CreateWorkspaceBuildRequest{
TemplateVersionID: r.TemplateVersion.ID,
Transition: codersdk.WorkspaceTransitionStart,
Orphan: true,
})
require.Error(t, err, "Orphan is only permitted when deleting a workspace.")
cerr := coderdtest.SDKError(t, err)
require.Equal(t, http.StatusBadRequest, cerr.StatusCode())
})

workspace := coderdtest.CreateWorkspace(t, client, template.ID)
coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
t.Run("WithState", func(t *testing.T) {
t.Parallel()
client, store := coderdtest.NewWithDatabase(t, nil)
first := coderdtest.CreateFirstUser(t, client)
templateAdmin, templateAdminUser := coderdtest.CreateAnotherUser(t, client, first.OrganizationID, rbac.RoleTemplateAdmin())

r := dbfake.WorkspaceBuild(t, store, database.WorkspaceTable{
OwnerID: templateAdminUser.ID,
OrganizationID: first.OrganizationID,
}).Do()

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()

// Providing both state and orphan fails.
_, err := templateAdmin.CreateWorkspaceBuild(ctx, r.Workspace.ID, codersdk.CreateWorkspaceBuildRequest{
TemplateVersionID: r.TemplateVersion.ID,
Transition: codersdk.WorkspaceTransitionDelete,
ProvisionerState: []byte(" "),
Orphan: true,
})
require.Error(t, err)
cerr := coderdtest.SDKError(t, err)
require.Equal(t, http.StatusBadRequest, cerr.StatusCode())
})

// Providing both state and orphan fails.
_, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{
TemplateVersionID: workspace.LatestBuild.TemplateVersionID,
Transition: codersdk.WorkspaceTransitionDelete,
ProvisionerState: []byte(" "),
Orphan: true,
t.Run("NoPermission", func(t *testing.T) {
t.Parallel()
client, store := coderdtest.NewWithDatabase(t, nil)
first := coderdtest.CreateFirstUser(t, client)
member, memberUser := coderdtest.CreateAnotherUser(t, client, first.OrganizationID)

r := dbfake.WorkspaceBuild(t, store, database.WorkspaceTable{
OwnerID: memberUser.ID,
OrganizationID: first.OrganizationID,
}).Do()

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()

// Trying to orphan without being a template admin fails.
_, err := member.CreateWorkspaceBuild(ctx, r.Workspace.ID, codersdk.CreateWorkspaceBuildRequest{
TemplateVersionID: r.TemplateVersion.ID,
Transition: codersdk.WorkspaceTransitionDelete,
Orphan: true,
})
require.Error(t, err)
cerr := coderdtest.SDKError(t, err)
require.Equal(t, http.StatusForbidden, cerr.StatusCode())
})
require.Error(t, err)
cerr := coderdtest.SDKError(t, err)
require.Equal(t, http.StatusBadRequest, cerr.StatusCode())

// Regular orphan operation succeeds.
build, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{
TemplateVersionID: workspace.LatestBuild.TemplateVersionID,
Transition: codersdk.WorkspaceTransitionDelete,
Orphan: true,
t.Run("OK", func(t *testing.T) {
// Include a provisioner so that we can test that provisionerdserver
// performs deletion.
client, store := coderdtest.NewWithDatabase(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
first := coderdtest.CreateFirstUser(t, client)
templateAdmin, templateAdminUser := coderdtest.CreateAnotherUser(t, client, first.OrganizationID, rbac.RoleTemplateAdmin())

r := dbfake.WorkspaceBuild(t, store, database.WorkspaceTable{
OwnerID: templateAdminUser.ID,
OrganizationID: first.OrganizationID,
}).Do()

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()

// Regular orphan operation succeeds.
build, err := templateAdmin.CreateWorkspaceBuild(ctx, r.Workspace.ID, codersdk.CreateWorkspaceBuildRequest{
TemplateVersionID: r.TemplateVersion.ID,
Transition: codersdk.WorkspaceTransitionDelete,
Orphan: true,
})
require.NoError(t, err)
require.Equal(t, codersdk.WorkspaceTransitionDelete, build.Transition)
require.Equal(t, codersdk.ProvisionerJobPending, build.Job.Status)
})
require.NoError(t, err)
coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, build.ID)

_, err = client.Workspace(ctx, workspace.ID)
require.Error(t, err)
require.Equal(t, http.StatusGone, coderdtest.SDKError(t, err).StatusCode())
Comment on lines -407 to -409
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

review: moved this check to the relevant CLI test

t.Run("NoProvisioners", func(t *testing.T) {
t.Parallel()
client, store := coderdtest.NewWithDatabase(t, nil)
first := coderdtest.CreateFirstUser(t, client)
templateAdmin, templateAdminUser := coderdtest.CreateAnotherUser(t, client, first.OrganizationID, rbac.RoleTemplateAdmin())

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
r := dbfake.WorkspaceBuild(t, store, database.WorkspaceTable{
OwnerID: templateAdminUser.ID,
OrganizationID: first.OrganizationID,
}).Do()

// nolint:gocritic // For testing
daemons, err := store.GetProvisionerDaemons(dbauthz.AsSystemReadProvisionerDaemons(ctx))
require.NoError(t, err)
require.Empty(t, daemons, "Provisioner daemons should be empty for this test")

// Orphan deletion still succeeds despite no provisioners being available.
build, err := templateAdmin.CreateWorkspaceBuild(ctx, r.Workspace.ID, codersdk.CreateWorkspaceBuildRequest{
TemplateVersionID: r.TemplateVersion.ID,
Transition: codersdk.WorkspaceTransitionDelete,
Orphan: true,
})
require.NoError(t, err)
require.Equal(t, codersdk.WorkspaceTransitionDelete, build.Transition)
require.Equal(t, codersdk.ProvisionerJobFailed, build.Job.Status)
require.Contains(t, build.Job.Error, "No provisioners were available to handle the request")

ws, err := client.Workspace(ctx, r.Workspace.ID)
require.Empty(t, ws)
require.Equal(t, http.StatusGone, coderdtest.SDKError(t, err).StatusCode())
})
})
}

Expand Down
44 changes: 44 additions & 0 deletions coderd/wsbuilder/wsbuilder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -464,6 +464,50 @@ func (b *Builder) buildTx(authFunc func(action policy.Action, object rbac.Object
return BuildError{http.StatusInternalServerError, "get workspace build", err}
}

// If the requestor is trying to orphan-delete a workspace and there are no
// provisioners available, we should complete the build and mark the
// workspace as deleted ourselves.
// Orphan-deleting a workspace sends an empty state to Terraform, which means
// it won't actually delete anything.
// There are cases where tagged provisioner daemons have been decommissioned
// without deleting the relevant workspaces, and without any provisioners
// available these workspaces cannot be deleted.
hasActiveEligibleProvisioner := false
for _, pd := range provisionerDaemons {
age := now.Sub(pd.ProvisionerDaemon.LastSeenAt.Time)
if age <= provisionerdserver.StaleInterval {
hasActiveEligibleProvisioner = true
break
}
}
if b.state.orphan && !hasActiveEligibleProvisioner {
// nolint: gocritic // At this moment, we are pretending to be provisionerd.
if err := store.UpdateProvisionerJobWithCompleteWithStartedAtByID(dbauthz.AsProvisionerd(b.ctx), database.UpdateProvisionerJobWithCompleteWithStartedAtByIDParams{
CompletedAt: sql.NullTime{Valid: true, Time: now},
Error: sql.NullString{Valid: true, String: "No provisioners were available to handle the request. The workspace has been deleted. No resources were destroyed."},
ErrorCode: sql.NullString{Valid: false},
ID: provisionerJob.ID,
StartedAt: sql.NullTime{Valid: true, Time: now},
UpdatedAt: now,
}); err != nil {
return BuildError{http.StatusInternalServerError, "mark orphan-delete provisioner job as completed", err}
}

// TODO: audit baggage?

// Re-fetch the completed provisioner job.
if pj, err := store.GetProvisionerJobByID(b.ctx, provisionerJob.ID); err == nil {
provisionerJob = pj
}

if err := store.UpdateWorkspaceDeletedByID(b.ctx, database.UpdateWorkspaceDeletedByIDParams{
ID: b.workspace.ID,
Deleted: true,
}); err != nil {
return BuildError{http.StatusInternalServerError, "mark workspace as deleted", err}
}
}

return nil
}, nil)
if err != nil {
Expand Down
Loading
Loading