Skip to content

feat(coderd): add matched provisioner daemons information to more places #15688

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Dec 2, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions coderd/apidoc/docs.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions coderd/apidoc/swagger.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion coderd/autobuild/lifecycle_executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ func (e *Executor) runOnce(t time.Time) Stats {
}
}

nextBuild, job, err = builder.Build(e.ctx, tx, nil, audit.WorkspaceBuildBaggage{IP: "127.0.0.1"})
nextBuild, job, _, err = builder.Build(e.ctx, tx, nil, audit.WorkspaceBuildBaggage{IP: "127.0.0.1"})
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

self-review: we may need to notify or log about this, but deferring for later.

if err != nil {
return xerrors.Errorf("build workspace with transition %q: %w", nextTransition, err)
}
Expand Down
20 changes: 20 additions & 0 deletions coderd/database/db2sdk/db2sdk.go
Original file line number Diff line number Diff line change
Expand Up @@ -673,3 +673,23 @@ func CryptoKey(key database.CryptoKey) codersdk.CryptoKey {
Secret: key.Secret.String,
}
}

func MatchedProvisioners(provisionerDaemons []database.ProvisionerDaemon, now time.Time, staleInterval time.Duration) codersdk.MatchedProvisioners {
minLastSeenAt := now.Add(-staleInterval)
mostRecentlySeen := codersdk.NullTime{}
var matched codersdk.MatchedProvisioners
for _, provisioner := range provisionerDaemons {
if !provisioner.LastSeenAt.Valid {
continue
}
matched.Count++
if provisioner.LastSeenAt.Time.After(minLastSeenAt) {
matched.Available++
}
if provisioner.LastSeenAt.Time.After(mostRecentlySeen.Time) {
matched.MostRecentlySeen.Valid = true
matched.MostRecentlySeen.Time = provisioner.LastSeenAt.Time
}
}
return matched
}
25 changes: 24 additions & 1 deletion coderd/database/dbauthz/dbauthz.go
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,7 @@ var (
rbac.ResourceSystem.Type: {policy.WildcardSymbol},
rbac.ResourceOrganization.Type: {policy.ActionCreate, policy.ActionRead},
rbac.ResourceOrganizationMember.Type: {policy.ActionCreate, policy.ActionDelete, policy.ActionRead},
rbac.ResourceProvisionerDaemon.Type: {policy.ActionCreate, policy.ActionUpdate},
rbac.ResourceProvisionerDaemon.Type: {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate},
rbac.ResourceProvisionerKeys.Type: {policy.ActionCreate, policy.ActionRead, policy.ActionDelete},
rbac.ResourceUser.Type: rbac.ResourceUser.AvailableActions(),
rbac.ResourceWorkspaceDormant.Type: {policy.ActionUpdate, policy.ActionDelete, policy.ActionWorkspaceStop},
Expand All @@ -317,6 +317,23 @@ var (
}),
Scope: rbac.ScopeAll,
}.WithCachedASTValue()

subjectSystemReadProvisionerDaemons = rbac.Subject{
FriendlyName: "System",
ID: uuid.Nil.String(),
Roles: rbac.Roles([]rbac.Role{
{
Identifier: rbac.RoleIdentifier{Name: "system-read-provisioner-daemons"},
DisplayName: "Coder",
Site: rbac.Permissions(map[string][]policy.Action{
rbac.ResourceProvisionerDaemon.Type: {policy.ActionRead},
}),
Org: map[string][]rbac.Permission{},
User: []rbac.Permission{},
},
}),
Scope: rbac.ScopeAll,
}.WithCachedASTValue()
)

// AsProvisionerd returns a context with an actor that has permissions required
Expand Down Expand Up @@ -359,6 +376,12 @@ func AsSystemRestricted(ctx context.Context) context.Context {
return context.WithValue(ctx, authContextKey{}, subjectSystemRestricted)
}

// AsSystemReadProvisionerDaemons returns a context with an actor that has permissions
// to read provisioner daemons.
func AsSystemReadProvisionerDaemons(ctx context.Context) context.Context {
return context.WithValue(ctx, authContextKey{}, subjectSystemReadProvisionerDaemons)
}

Comment on lines +379 to +384
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

self-review: I wanted to avoid sprinkling dbauthz.AsSystemRestricted everywhere, so I made a separate RBAC role for when we just wish to read provisioner daemons. I can remove this and switch back to SystemRestricted if folks prefer.

var AsRemoveActor = rbac.Subject{
ID: "remove-actor",
}
Expand Down
175 changes: 117 additions & 58 deletions coderd/templateversions.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"fmt"
"net/http"
"os"
"time"

"github.com/go-chi/chi/v5"
"github.com/google/uuid"
Expand All @@ -22,6 +21,8 @@ import (

"github.com/coder/coder/v2/coderd/audit"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/db2sdk"
"github.com/coder/coder/v2/coderd/database/dbauthz"
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/coderd/database/provisionerjobs"
"github.com/coder/coder/v2/coderd/externalauth"
Expand All @@ -32,6 +33,7 @@ import (
"github.com/coder/coder/v2/coderd/rbac/policy"
"github.com/coder/coder/v2/coderd/render"
"github.com/coder/coder/v2/coderd/tracing"
"github.com/coder/coder/v2/coderd/util/ptr"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/examples"
"github.com/coder/coder/v2/provisioner/terraform/tfparse"
Expand Down Expand Up @@ -60,6 +62,22 @@ func (api *API) templateVersion(rw http.ResponseWriter, r *http.Request) {
return
}

var matchedProvisioners *codersdk.MatchedProvisioners
if jobs[0].ProvisionerJob.JobStatus == database.ProvisionerJobStatusPending {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we only ever expect a single job to be returned by GetProvisionerJobsByIDsWithQueuePosition?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In this instance, we should get either 0 or 1:

jobs, err := api.Database.GetProvisionerJobsByIDsWithQueuePosition(ctx, []uuid.UUID{templateVersion.JobID})

Copy link
Contributor

@dannykopping dannykopping Dec 2, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My point here is really that if we're only ever expecting a single job, we should consider changing the semantics of the GetProvisionerJobsByIDsWithQueuePosition to be a :one not a :many.
non-blocking suggestion, of course.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess it's possible though that a single template version could have multiple provisioner jobs associated if something went wrong.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we use the :many version of this query in a few places, and not just the :one.

// nolint: gocritic // The user hitting this endpoint may not have
// permission to read provisioner daemons, but we want to show them
// information about the provisioner daemons that are available.
provisioners, err := api.Database.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(ctx), database.GetProvisionerDaemonsByOrganizationParams{
OrganizationID: jobs[0].ProvisionerJob.OrganizationID,
WantTags: jobs[0].ProvisionerJob.Tags,
})
if err != nil {
api.Logger.Error(ctx, "failed to fetch provisioners for job id", slog.F("job_id", jobs[0].ProvisionerJob.ID), slog.Error(err))
} else {
matchedProvisioners = ptr.Ref(db2sdk.MatchedProvisioners(provisioners, dbtime.Now(), provisionerdserver.StaleInterval))
}
}

schemas, err := api.Database.GetParameterSchemasByJobID(ctx, jobs[0].ProvisionerJob.ID)
if errors.Is(err, sql.ErrNoRows) {
err = nil
Expand All @@ -77,7 +95,7 @@ func (api *API) templateVersion(rw http.ResponseWriter, r *http.Request) {
warnings = append(warnings, codersdk.TemplateVersionWarningUnsupportedWorkspaces)
}

httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), nil, warnings))
httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), matchedProvisioners, warnings))
}

// @Summary Patch template version by ID
Expand Down Expand Up @@ -173,7 +191,23 @@ func (api *API) patchTemplateVersion(rw http.ResponseWriter, r *http.Request) {
return
}

httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(updatedTemplateVersion, convertProvisionerJob(jobs[0]), nil, nil))
var matchedProvisioners *codersdk.MatchedProvisioners
if jobs[0].ProvisionerJob.JobStatus == database.ProvisionerJobStatusPending {
// nolint: gocritic // The user hitting this endpoint may not have
// permission to read provisioner daemons, but we want to show them
// information about the provisioner daemons that are available.
provisioners, err := api.Database.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(ctx), database.GetProvisionerDaemonsByOrganizationParams{
OrganizationID: jobs[0].ProvisionerJob.OrganizationID,
WantTags: jobs[0].ProvisionerJob.Tags,
})
if err != nil {
api.Logger.Error(ctx, "failed to fetch provisioners for job id", slog.F("job_id", jobs[0].ProvisionerJob.ID), slog.Error(err))
} else {
matchedProvisioners = ptr.Ref(db2sdk.MatchedProvisioners(provisioners, dbtime.Now(), provisionerdserver.StaleInterval))
}
}

httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(updatedTemplateVersion, convertProvisionerJob(jobs[0]), matchedProvisioners, nil))
}

// @Summary Cancel template version by ID
Expand Down Expand Up @@ -868,8 +902,23 @@ func (api *API) templateVersionByName(rw http.ResponseWriter, r *http.Request) {
})
return
}
var matchedProvisioners *codersdk.MatchedProvisioners
if jobs[0].ProvisionerJob.JobStatus == database.ProvisionerJobStatusPending {
// nolint: gocritic // The user hitting this endpoint may not have
// permission to read provisioner daemons, but we want to show them
// information about the provisioner daemons that are available.
provisioners, err := api.Database.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(ctx), database.GetProvisionerDaemonsByOrganizationParams{
OrganizationID: jobs[0].ProvisionerJob.OrganizationID,
WantTags: jobs[0].ProvisionerJob.Tags,
})
if err != nil {
api.Logger.Error(ctx, "failed to fetch provisioners for job id", slog.F("job_id", jobs[0].ProvisionerJob.ID), slog.Error(err))
} else {
matchedProvisioners = ptr.Ref(db2sdk.MatchedProvisioners(provisioners, dbtime.Now(), provisionerdserver.StaleInterval))
}
}

httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), nil, nil))
httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), matchedProvisioners, nil))
}

// @Summary Get template version by organization, template, and name
Expand Down Expand Up @@ -934,7 +983,23 @@ func (api *API) templateVersionByOrganizationTemplateAndName(rw http.ResponseWri
return
}

httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), nil, nil))
var matchedProvisioners *codersdk.MatchedProvisioners
if jobs[0].ProvisionerJob.JobStatus == database.ProvisionerJobStatusPending {
// nolint: gocritic // The user hitting this endpoint may not have
// permission to read provisioner daemons, but we want to show them
// information about the provisioner daemons that are available.
provisioners, err := api.Database.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(ctx), database.GetProvisionerDaemonsByOrganizationParams{
OrganizationID: jobs[0].ProvisionerJob.OrganizationID,
WantTags: jobs[0].ProvisionerJob.Tags,
})
if err != nil {
api.Logger.Error(ctx, "failed to fetch provisioners for job id", slog.F("job_id", jobs[0].ProvisionerJob.ID), slog.Error(err))
} else {
matchedProvisioners = ptr.Ref(db2sdk.MatchedProvisioners(provisioners, dbtime.Now(), provisionerdserver.StaleInterval))
}
}

httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), matchedProvisioners, nil))
Comment on lines +1023 to +1039
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

self-review: This addresses an issue I noticed in the frontend where it quickly "flashes" with the tag warning message but then gets overwritten when the FE re-requests the template version. Adding it to other template-related endpoints for posterity.

}

// @Summary Get previous template version by organization, template, and name
Expand Down Expand Up @@ -1020,7 +1085,23 @@ func (api *API) previousTemplateVersionByOrganizationTemplateAndName(rw http.Res
return
}

httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(previousTemplateVersion, convertProvisionerJob(jobs[0]), nil, nil))
var matchedProvisioners *codersdk.MatchedProvisioners
if jobs[0].ProvisionerJob.JobStatus == database.ProvisionerJobStatusPending {
// nolint: gocritic // The user hitting this endpoint may not have
// permission to read provisioner daemons, but we want to show them
// information about the provisioner daemons that are available.
provisioners, err := api.Database.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(ctx), database.GetProvisionerDaemonsByOrganizationParams{
OrganizationID: jobs[0].ProvisionerJob.OrganizationID,
WantTags: jobs[0].ProvisionerJob.Tags,
})
if err != nil {
api.Logger.Error(ctx, "failed to fetch provisioners for job id", slog.F("job_id", jobs[0].ProvisionerJob.ID), slog.Error(err))
} else {
matchedProvisioners = ptr.Ref(db2sdk.MatchedProvisioners(provisioners, dbtime.Now(), provisionerdserver.StaleInterval))
}
}

httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(previousTemplateVersion, convertProvisionerJob(jobs[0]), matchedProvisioners, nil))
}

// @Summary Archive template unused versions by template id
Expand Down Expand Up @@ -1513,27 +1594,6 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht
return err
}

// Check for eligible provisioners. This allows us to log a message warning deployment administrators
// of users submitting jobs for which no provisioners are available.
matchedProvisioners, err = checkProvisioners(ctx, tx, organization.ID, tags)
if err != nil {
api.Logger.Error(ctx, "failed to check eligible provisioner daemons for job", slog.Error(err))
} else if matchedProvisioners.Count == 0 {
api.Logger.Warn(ctx, "no matching provisioners found for job",
slog.F("user_id", apiKey.UserID),
slog.F("job_id", jobID),
slog.F("job_type", database.ProvisionerJobTypeTemplateVersionImport),
slog.F("tags", tags),
)
} else if matchedProvisioners.Available == 0 {
api.Logger.Warn(ctx, "no active provisioners found for job",
slog.F("user_id", apiKey.UserID),
slog.F("job_id", jobID),
slog.F("job_type", database.ProvisionerJobTypeTemplateVersionImport),
slog.F("tags", tags),
)
}

provisionerJob, err = tx.InsertProvisionerJob(ctx, database.InsertProvisionerJobParams{
ID: jobID,
CreatedAt: dbtime.Now(),
Expand All @@ -1559,6 +1619,36 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht
return err
}

// Check for eligible provisioners. This allows us to return a warning to the user if they
// submit a job for which no provisioner is available.
// nolint: gocritic // The user hitting this endpoint may not have
// permission to read provisioner daemons, but we want to show them
// information about the provisioner daemons that are available.
eligibleProvisioners, err := tx.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(ctx), database.GetProvisionerDaemonsByOrganizationParams{
OrganizationID: organization.ID,
WantTags: provisionerJob.Tags,
})
if err != nil {
// Log the error but do not return any warnings. This is purely advisory and we should not block.
api.Logger.Error(ctx, "failed to check eligible provisioner daemons for job", slog.Error(err))
}
matchedProvisioners = db2sdk.MatchedProvisioners(eligibleProvisioners, provisionerJob.CreatedAt, provisionerdserver.StaleInterval)
if matchedProvisioners.Count == 0 {
api.Logger.Warn(ctx, "no matching provisioners found for job",
slog.F("user_id", apiKey.UserID),
slog.F("job_id", jobID),
slog.F("job_type", database.ProvisionerJobTypeTemplateVersionImport),
slog.F("tags", tags),
)
} else if matchedProvisioners.Available == 0 {
api.Logger.Warn(ctx, "no active provisioners found for job",
slog.F("user_id", apiKey.UserID),
slog.F("job_id", jobID),
slog.F("job_type", database.ProvisionerJobTypeTemplateVersionImport),
slog.F("tags", tags),
)
}

var templateID uuid.NullUUID
if req.TemplateID != uuid.Nil {
templateID = uuid.NullUUID{
Expand Down Expand Up @@ -1822,34 +1912,3 @@ func (api *API) publishTemplateUpdate(ctx context.Context, templateID uuid.UUID)
slog.F("template_id", templateID), slog.Error(err))
}
}

func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUID, wantTags map[string]string) (codersdk.MatchedProvisioners, error) {
// Check for eligible provisioners. This allows us to return a warning to the user if they
// submit a job for which no provisioner is available.
eligibleProvisioners, err := store.GetProvisionerDaemonsByOrganization(ctx, database.GetProvisionerDaemonsByOrganizationParams{
OrganizationID: orgID,
WantTags: wantTags,
})
if err != nil {
// Log the error but do not return any warnings. This is purely advisory and we should not block.
return codersdk.MatchedProvisioners{}, xerrors.Errorf("provisioner daemons by organization: %w", err)
}

staleInterval := time.Now().Add(-provisionerdserver.StaleInterval)
mostRecentlySeen := codersdk.NullTime{}
var matched codersdk.MatchedProvisioners
for _, provisioner := range eligibleProvisioners {
if !provisioner.LastSeenAt.Valid {
continue
}
matched.Count++
if provisioner.LastSeenAt.Time.After(staleInterval) {
matched.Available++
}
if provisioner.LastSeenAt.Time.After(mostRecentlySeen.Time) {
matched.MostRecentlySeen.Valid = true
matched.MostRecentlySeen.Time = provisioner.LastSeenAt.Time
}
}
return matched, nil
}
Comment on lines -1826 to -1855
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

self-review: extracted to db2sdk.

Loading
Loading