Skip to content

feat: implement claiming of prebuilt workspaces #17458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 38 commits into from
Apr 24, 2025
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
8b9e30d
feat: implement claiming of prebuilt workspaces
evgeniy-scherbina Apr 17, 2025
d9497df
Merge branch 'main' of github.com:/coder/coder into yevhenii/512-clai…
dannykopping Apr 22, 2025
217e46f
Reverting unncessary changes
dannykopping Apr 21, 2025
c459533
Refactoring
dannykopping Apr 22, 2025
f905219
Removing unnecessary field
dannykopping Apr 22, 2025
8266338
make fmt
dannykopping Apr 22, 2025
4098ed7
CR's fixes
evgeniy-scherbina Apr 22, 2025
50d23e4
CR's fixes
evgeniy-scherbina Apr 22, 2025
546b7ae
Update coderd/workspaces.go
evgeniy-scherbina Apr 22, 2025
85f7926
CR's fixes
evgeniy-scherbina Apr 22, 2025
ee9908c
CR's fixes
evgeniy-scherbina Apr 22, 2025
70bf179
CR's fixes
evgeniy-scherbina Apr 22, 2025
7a72d03
CR's fixes
evgeniy-scherbina Apr 22, 2025
b246589
fix tests by updating noop.go
evgeniy-scherbina Apr 22, 2025
ff8d3de
Merge branch 'main' of github.com:/coder/coder into yevhenii/512-clai…
dannykopping Apr 23, 2025
fcdbba8
test: add failure testcase scenario
evgeniy-scherbina Apr 23, 2025
e7b7f80
refactor: revert interface changes
evgeniy-scherbina Apr 23, 2025
e7455db
refactor: simplify signature of claim method
evgeniy-scherbina Apr 23, 2025
077d7f0
refactor: CR's fixes
evgeniy-scherbina Apr 23, 2025
0d426c7
Merge remote-tracking branch 'origin/main' into yevhenii/512-claim-pr…
evgeniy-scherbina Apr 23, 2025
663a8c0
refactor: CR's fixes
evgeniy-scherbina Apr 23, 2025
bc31fac
refactor: make lint
evgeniy-scherbina Apr 23, 2025
087bd20
refactor: fix linter
evgeniy-scherbina Apr 23, 2025
9f66169
refactor: remove PrebuildsOrchestrator context from claim tests
evgeniy-scherbina Apr 23, 2025
7f25f24
refactor: don't fail test in a goroutine
evgeniy-scherbina Apr 23, 2025
ff9eeb4
docs: fix ssh coder example in testing-templates doc (#17550)
EdwardAngert Apr 23, 2025
31b873f
fix: add websocket close handling (#17548)
jaaydenh Apr 23, 2025
4fe770d
ci: move go install tools to separate action (#17552)
ethanndickson Apr 24, 2025
8d8fc99
chore(dogfood): allow provider minor version updates (#17554)
matifali Apr 24, 2025
af32325
fix(examples/templates/docker-devcontainer): update folder path and p…
Aericio Apr 24, 2025
0826e8a
feat: enable masking password inputs instead of blocking echo (#17469)
ibetitsmike Apr 24, 2025
cbb6c12
fix(examples/templates/kubernetes-devcontainer): update coder provide…
matifali Apr 24, 2025
e466844
docs: add automatic release calendar updates in docs (#17531)
matifali Apr 24, 2025
f24557b
chore(dogfood): add windsurf module (#17558)
matifali Apr 24, 2025
fa65564
Merge branch 'main' of github.com:/coder/coder into yevhenii/512-clai…
dannykopping Apr 24, 2025
53b38d7
fix: use userCtx instead of prebuildCtx for claiming prebuild
evgeniy-scherbina Apr 24, 2025
6c41e03
refactor: CR's fixes
evgeniy-scherbina Apr 24, 2025
9173e9a
CR's fixes
evgeniy-scherbina Apr 24, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Reverting unncessary changes
Signed-off-by: Danny Kopping <dannykopping@gmail.com>
  • Loading branch information
dannykopping committed Apr 22, 2025
commit 217e46fc35220f673fe4ccab5e9f1227b92f24d7
3 changes: 3 additions & 0 deletions coderd/apidoc/docs.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions coderd/apidoc/swagger.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 1 addition & 8 deletions coderd/provisionerdserver/provisionerdserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -2473,14 +2473,7 @@ type WorkspaceProvisionJob struct {
DryRun bool `json:"dry_run"`
IsPrebuild bool `json:"is_prebuild,omitempty"`
PrebuildClaimedByUser uuid.UUID `json:"prebuild_claimed_by,omitempty"`
// RunningWorkspaceAgentID is *only* used for prebuilds. We pass it down when we want to rebuild a prebuilt workspace
// but not generate a new agent token. The provisionerdserver will retrieve this token and push it down to
// the provisioner (and ultimately to the `coder_agent` resource in the Terraform provider) where it will be
// reused. Context: the agent token is often used in immutable attributes of workspace resource (e.g. VM/container)
// to initialize the agent, so if that value changes it will necessitate a replacement of that resource, thus
// obviating the whole point of the prebuild.
RunningWorkspaceAgentID uuid.UUID `json:"running_workspace_agent_id"`
LogLevel string `json:"log_level,omitempty"`
LogLevel string `json:"log_level,omitempty"`
}

// TemplateVersionDryRunJob is the payload for the "template_version_dry_run" job type.
Expand Down
19 changes: 4 additions & 15 deletions coderd/workspaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,21 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/coder/coder/v2/coderd/prebuilds"
"net/http"
"slices"
"strconv"
"time"

"github.com/coder/coder/v2/coderd/prebuilds"

"github.com/dustin/go-humanize"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"golang.org/x/sync/errgroup"
"golang.org/x/xerrors"

"cdr.dev/slog"

"github.com/coder/coder/v2/agent/proto"
"github.com/coder/coder/v2/coderd/audit"
"github.com/coder/coder/v2/coderd/database"
Expand Down Expand Up @@ -636,8 +638,6 @@ func createWorkspace(
provisionerJob *database.ProvisionerJob
workspaceBuild *database.WorkspaceBuild
provisionerDaemons []database.GetEligibleProvisionerDaemonsByProvisionerJobIDsRow

runningWorkspaceAgentID uuid.UUID
)

prebuilds := (*api.PrebuildsClaimer.Load()).(prebuilds.Claimer)
Expand Down Expand Up @@ -685,16 +685,6 @@ func createWorkspace(
// Prebuild found!
workspaceID = claimedWorkspace.ID
initiatorID = prebuilds.Initiator()
agents, err := db.GetWorkspaceAgentsInLatestBuildByWorkspaceID(ctx, claimedWorkspace.ID)
if err != nil {
// TODO: comment about best-effort, workspace can be restarted if this fails...
api.Logger.Error(ctx, "failed to retrieve running agents of claimed prebuilt workspace",
slog.F("workspace_id", claimedWorkspace.ID), slog.Error(err))
}
if len(agents) >= 1 {
// TODO: handle multiple agents
runningWorkspaceAgentID = agents[0].ID
}
}

// We have to refetch the workspace for the joined in fields.
Expand All @@ -710,8 +700,7 @@ func createWorkspace(
Initiator(initiatorID).
ActiveVersion().
RichParameterValues(req.RichParameterValues).
TemplateVersionPresetID(req.TemplateVersionPresetID).
RunningWorkspaceAgentID(runningWorkspaceAgentID)
TemplateVersionPresetID(req.TemplateVersionPresetID)
if req.TemplateVersionID != uuid.Nil {
builder = builder.VersionID(req.TemplateVersionID)
}
Expand Down
9 changes: 0 additions & 9 deletions coderd/wsbuilder/wsbuilder.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@ type Builder struct {

prebuild bool
prebuildClaimedBy uuid.UUID
runningWorkspaceAgentID uuid.UUID

verifyNoLegacyParametersOnce bool
}
Expand Down Expand Up @@ -186,13 +185,6 @@ func (b Builder) MarkPrebuildClaimedBy(userID uuid.UUID) Builder {
return b
}

// RunningWorkspaceAgentID is only used for prebuilds; see the associated field in `provisionerdserver.WorkspaceProvisionJob`.
func (b Builder) RunningWorkspaceAgentID(id uuid.UUID) Builder {
// nolint: revive
b.runningWorkspaceAgentID = id
return b
}

// SetLastWorkspaceBuildInTx prepopulates the Builder's cache with the last workspace build. This allows us
// to avoid a repeated database query when the Builder's caller also needs the workspace build, e.g. auto-start &
// auto-stop.
Expand Down Expand Up @@ -328,7 +320,6 @@ func (b *Builder) buildTx(authFunc func(action policy.Action, object rbac.Object
LogLevel: b.logLevel,
IsPrebuild: b.prebuild,
PrebuildClaimedByUser: b.prebuildClaimedBy,
RunningWorkspaceAgentID: b.runningWorkspaceAgentID,
})
if err != nil {
return nil, nil, nil, BuildError{
Expand Down
49 changes: 1 addition & 48 deletions codersdk/deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,6 @@ const (
FeatureControlSharedPorts FeatureName = "control_shared_ports"
FeatureCustomRoles FeatureName = "custom_roles"
FeatureMultipleOrganizations FeatureName = "multiple_organizations"
FeatureWorkspacePrebuilds FeatureName = "workspace_prebuilds"
)

// FeatureNames must be kept in-sync with the Feature enum above.
Expand All @@ -104,7 +103,6 @@ var FeatureNames = []FeatureName{
FeatureControlSharedPorts,
FeatureCustomRoles,
FeatureMultipleOrganizations,
FeatureWorkspacePrebuilds,
}

// Humanize returns the feature name in a human-readable format.
Expand Down Expand Up @@ -134,7 +132,6 @@ func (n FeatureName) AlwaysEnable() bool {
FeatureHighAvailability: true,
FeatureCustomRoles: true,
FeatureMultipleOrganizations: true,
FeatureWorkspacePrebuilds: true,
}[n]
}

Expand Down Expand Up @@ -396,7 +393,6 @@ type DeploymentValues struct {
TermsOfServiceURL serpent.String `json:"terms_of_service_url,omitempty" typescript:",notnull"`
Notifications NotificationsConfig `json:"notifications,omitempty" typescript:",notnull"`
AdditionalCSPPolicy serpent.StringArray `json:"additional_csp_policy,omitempty" typescript:",notnull"`
Prebuilds PrebuildsConfig `json:"workspace_prebuilds,omitempty" typescript:",notnull"`
WorkspaceHostnameSuffix serpent.String `json:"workspace_hostname_suffix,omitempty" typescript:",notnull"`

Config serpent.YAMLConfigPath `json:"config,omitempty" typescript:",notnull"`
Expand Down Expand Up @@ -1038,11 +1034,6 @@ func (c *DeploymentValues) Options() serpent.OptionSet {
Parent: &deploymentGroupNotifications,
YAML: "webhook",
}
deploymentGroupPrebuilds = serpent.Group{
Name: "Workspace Prebuilds",
YAML: "workspace_prebuilds",
Description: "Configure how workspace prebuilds behave.",
}
deploymentGroupInbox = serpent.Group{
Name: "Inbox",
Parent: &deploymentGroupNotifications,
Expand Down Expand Up @@ -3038,41 +3029,6 @@ Write out the current server config as YAML to stdout.`,
Annotations: serpent.Annotations{}.Mark(annotationFormatDuration, "true"),
Hidden: true, // Hidden because most operators should not need to modify this.
},
{
Name: "Reconciliation Interval",
Description: "How often to reconcile workspace prebuilds state.",
Flag: "workspace-prebuilds-reconciliation-interval",
Env: "CODER_WORKSPACE_PREBUILDS_RECONCILIATION_INTERVAL",
Value: &c.Prebuilds.ReconciliationInterval,
Default: (time.Second * 15).String(),
Group: &deploymentGroupPrebuilds,
YAML: "reconciliation_interval",
Annotations: serpent.Annotations{}.Mark(annotationFormatDuration, "true"),
},
{
Name: "Reconciliation Backoff Interval",
Description: "Interval to increase reconciliation backoff by when unrecoverable errors occur.",
Flag: "workspace-prebuilds-reconciliation-backoff-interval",
Env: "CODER_WORKSPACE_PREBUILDS_RECONCILIATION_BACKOFF_INTERVAL",
Value: &c.Prebuilds.ReconciliationBackoffInterval,
Default: (time.Second * 15).String(),
Group: &deploymentGroupPrebuilds,
YAML: "reconciliation_backoff_interval",
Annotations: serpent.Annotations{}.Mark(annotationFormatDuration, "true"),
Hidden: true,
},
{
Name: "Reconciliation Backoff Lookback Period",
Description: "Interval to look back to determine number of failed builds, which influences backoff.",
Flag: "workspace-prebuilds-reconciliation-backoff-lookback-period",
Env: "CODER_WORKSPACE_PREBUILDS_RECONCILIATION_BACKOFF_LOOKBACK_PERIOD",
Value: &c.Prebuilds.ReconciliationBackoffLookback,
Default: (time.Hour).String(), // TODO: use https://pkg.go.dev/github.com/jackc/pgtype@v1.12.0#Interval
Group: &deploymentGroupPrebuilds,
YAML: "reconciliation_backoff_lookback_period",
Annotations: serpent.Annotations{}.Mark(annotationFormatDuration, "true"),
Hidden: true,
},
// Push notifications.
}

Expand Down Expand Up @@ -3298,7 +3254,6 @@ const (
ExperimentAutoFillParameters Experiment = "auto-fill-parameters" // This should not be taken out of experiments until we have redesigned the feature.
ExperimentNotifications Experiment = "notifications" // Sends notifications via SMTP and webhooks following certain events.
ExperimentWorkspaceUsage Experiment = "workspace-usage" // Enables the new workspace usage tracking.
ExperimentWorkspacePrebuilds Experiment = "workspace-prebuilds" // Enables the new workspace prebuilds feature.
ExperimentWebPush Experiment = "web-push" // Enables web push notifications through the browser.
ExperimentDynamicParameters Experiment = "dynamic-parameters" // Enables dynamic parameters when creating a workspace.
)
Expand All @@ -3307,9 +3262,7 @@ const (
// users to opt-in to via --experimental='*'.
// Experiments that are not ready for consumption by all users should
// not be included here and will be essentially hidden.
var ExperimentsAll = Experiments{
ExperimentWorkspacePrebuilds,
}
var ExperimentsAll = Experiments{}

// Experiments is a list of experiments.
// Multiple experiments may be enabled at the same time.
Expand Down
22 changes: 12 additions & 10 deletions docs/reference/api/schemas.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions docs/reference/api/workspaces.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

43 changes: 0 additions & 43 deletions enterprise/coderd/coderd.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,6 @@ import (
"context"
"crypto/ed25519"
"fmt"
agplprebuilds "github.com/coder/coder/v2/coderd/prebuilds"
"github.com/coder/coder/v2/enterprise/coderd/prebuilds"
"github.com/coder/quartz"
"math"
"net/http"
"net/url"
Expand Down Expand Up @@ -631,8 +628,6 @@ type API struct {

licenseMetricsCollector *license.MetricsCollector
tailnetService *tailnet.ClientService

PrebuildsReconciler agplprebuilds.ReconciliationOrchestrator
}

// writeEntitlementWarningsHeader writes the entitlement warnings to the response header
Expand Down Expand Up @@ -663,14 +658,6 @@ func (api *API) Close() error {
if api.Options.CheckInactiveUsersCancelFunc != nil {
api.Options.CheckInactiveUsersCancelFunc()
}

if api.PrebuildsReconciler != nil {
ctx, giveUp := context.WithTimeoutCause(context.Background(), time.Second*30, xerrors.New("gave up waiting for reconciler to stop"))
defer giveUp()
// TODO: determine root cause (requires changes up the stack, though).
api.PrebuildsReconciler.Stop(ctx, xerrors.New("api closed"))
}

return api.AGPL.Close()
}

Expand Down Expand Up @@ -873,20 +860,6 @@ func (api *API) updateEntitlements(ctx context.Context) error {
api.AGPL.PortSharer.Store(&ps)
}

if initial, changed, enabled := featureChanged(codersdk.FeatureWorkspacePrebuilds); shouldUpdate(initial, changed, enabled) || api.PrebuildsReconciler == nil {
reconciler, claimer := api.setupPrebuilds(enabled)
if api.PrebuildsReconciler != nil {
stopCtx, giveUp := context.WithTimeoutCause(context.Background(), time.Second*30, xerrors.New("gave up waiting for reconciler to stop"))
defer giveUp()
api.PrebuildsReconciler.Stop(stopCtx, xerrors.New("entitlements change"))
}

api.PrebuildsReconciler = reconciler
go reconciler.RunLoop(context.Background())

api.AGPL.PrebuildsClaimer.Store(&claimer)
}

// External token encryption is soft-enforced
featureExternalTokenEncryption := reloadedEntitlements.Features[codersdk.FeatureExternalTokenEncryption]
featureExternalTokenEncryption.Enabled = len(api.ExternalTokenEncryption) > 0
Expand Down Expand Up @@ -1155,19 +1128,3 @@ func (api *API) runEntitlementsLoop(ctx context.Context) {
func (api *API) Authorize(r *http.Request, action policy.Action, object rbac.Objecter) bool {
return api.AGPL.HTTPAuth.Authorize(r, action, object)
}

func (api *API) setupPrebuilds(entitled bool) (agplprebuilds.ReconciliationOrchestrator, agplprebuilds.Claimer) {
enabled := api.AGPL.Experiments.Enabled(codersdk.ExperimentWorkspacePrebuilds)
if !enabled || !entitled {
api.Logger.Debug(context.Background(), "prebuilds not enabled",
slog.F("experiment_enabled", enabled), slog.F("entitled", entitled))

return agplprebuilds.NewNoopReconciler(), agplprebuilds.DefaultClaimer
}

reconciler := prebuilds.NewStoreReconciler(api.Database, api.Pubsub, api.DeploymentValues.Prebuilds,
api.Logger.Named("prebuilds"), quartz.NewReal())

return reconciler,
prebuilds.EnterpriseClaimer{}
}
Loading