From 7afd3fbf3351135581695c6c485f0902600af396 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 7 Feb 2022 19:53:42 +0000 Subject: [PATCH 1/4] refactor: Generalize log ownership to allow for scratch jobs Importing may fail when creating a project. We don't want to lose this output, but we don't want to allow users to create a failing project. This generalizes logs to soon enable one-off situations where a user can upload their archive, create a project, and watch the output parse to completion. --- coderd/coderd.go | 13 +- coderd/projectversion.go | 5 - coderd/provisionerdaemons.go | 130 ++---- ...cehistorylogs.go => provisionerjoblogs.go} | 75 ++-- ...ogs_test.go => provisionerjoblogs_test.go} | 13 +- coderd/provisioners.go | 7 + coderd/workspacehistory.go | 1 - codersdk/provisioners.go | 69 ++++ codersdk/provisioners_test.go | 79 ++++ codersdk/workspaces.go | 74 ---- codersdk/workspaces_test.go | 77 ---- database/databasefake/databasefake.go | 183 ++++----- database/dump.sql | 56 ++- database/migrations/000002_projects.up.sql | 29 +- database/migrations/000003_workspaces.up.sql | 9 - database/migrations/000004_jobs.up.sql | 23 +- database/models.go | 34 +- database/querier.go | 8 +- database/query.sql | 72 ++-- database/query.sql.go | 377 +++++++----------- httpmw/provisionerjobparam.go | 64 +++ httpmw/provisionerjobparam_test.go | 109 +++++ provisionerd/proto/provisionerd.pb.go | 259 +++++------- provisionerd/proto/provisionerd.proto | 16 +- provisionerd/provisionerd.go | 11 +- provisionerd/provisionerd_test.go | 4 +- 26 files changed, 870 insertions(+), 927 deletions(-) rename coderd/{workspacehistorylogs.go => provisionerjoblogs.go} (59%) rename coderd/{workspacehistorylogs_test.go => provisionerjoblogs_test.go} (90%) create mode 100644 httpmw/provisionerjobparam.go create mode 100644 httpmw/provisionerjobparam_test.go diff --git a/coderd/coderd.go b/coderd/coderd.go index abce77ce2a10e..076ea4721af30 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -96,16 +96,21 @@ func New(options *Options) http.Handler { r.Route("/{workspacehistory}", func(r chi.Router) { r.Use(httpmw.ExtractWorkspaceHistoryParam(options.Database)) r.Get("/", api.workspaceHistoryByName) - r.Get("/logs", api.workspaceHistoryLogsByName) }) }) }) }) }) - r.Route("/provisioners/daemons", func(r chi.Router) { - r.Get("/", api.provisionerDaemons) - r.Get("/serve", api.provisionerDaemonsServe) + r.Route("/provisioners", func(r chi.Router) { + r.Route("/daemons", func(r chi.Router) { + r.Get("/", api.provisionerDaemons) + r.Get("/serve", api.provisionerDaemonsServe) + }) + r.Route("/jobs/{provisionerjob}", func(r chi.Router) { + r.Use(httpmw.ExtractProvisionerJobParam(options.Database)) + r.Get("/logs", api.provisionerJobLogsByID) + }) }) }) r.NotFound(site.Handler().ServeHTTP) diff --git a/coderd/projectversion.go b/coderd/projectversion.go index b27b4959af0b0..bd60d4c9bd288 100644 --- a/coderd/projectversion.go +++ b/coderd/projectversion.go @@ -153,7 +153,6 @@ func (api *api) postProjectVersionByOrganization(rw http.ResponseWriter, r *http InitiatorID: apiKey.UserID, Provisioner: project.Provisioner, Type: database.ProvisionerJobTypeProjectImport, - ProjectID: project.ID, Input: input, }) if err != nil { @@ -249,7 +248,3 @@ func convertProjectParameter(parameter database.ProjectParameter) ProjectParamet ValidationValueType: parameter.ValidationValueType, } } - -func projectVersionLogsChannel(projectVersionID uuid.UUID) string { - return fmt.Sprintf("project-version-logs:%s", projectVersionID) -} diff --git a/coderd/provisionerdaemons.go b/coderd/provisionerdaemons.go index 87611a8ba6734..d3c1e7590b80d 100644 --- a/coderd/provisionerdaemons.go +++ b/coderd/provisionerdaemons.go @@ -165,26 +165,16 @@ func (server *provisionerdServer) AcquireJob(ctx context.Context, _ *proto.Empty return xerrors.Errorf("request job was invalidated: %s", errorMessage) } - project, err := server.Database.GetProjectByID(ctx, job.ProjectID) - if err != nil { - return nil, failJob(fmt.Sprintf("get project: %s", err)) - } - organization, err := server.Database.GetOrganizationByID(ctx, project.OrganizationID) - if err != nil { - return nil, failJob(fmt.Sprintf("get organization: %s", err)) - } user, err := server.Database.GetUserByID(ctx, job.InitiatorID) if err != nil { return nil, failJob(fmt.Sprintf("get user: %s", err)) } protoJob := &proto.AcquiredJob{ - JobId: job.ID.String(), - CreatedAt: job.CreatedAt.UnixMilli(), - Provisioner: string(job.Provisioner), - OrganizationName: organization.Name, - ProjectName: project.Name, - UserName: user.Username, + JobId: job.ID.String(), + CreatedAt: job.CreatedAt.UnixMilli(), + Provisioner: string(job.Provisioner), + UserName: user.Username, } var projectVersion database.ProjectVersion switch job.Type { @@ -206,6 +196,14 @@ func (server *provisionerdServer) AcquireJob(ctx context.Context, _ *proto.Empty if err != nil { return nil, failJob(fmt.Sprintf("get project version: %s", err)) } + project, err := server.Database.GetProjectByID(ctx, projectVersion.ProjectID) + if err != nil { + return nil, failJob(fmt.Sprintf("get project: %s", err)) + } + organization, err := server.Database.GetOrganizationByID(ctx, project.OrganizationID) + if err != nil { + return nil, failJob(fmt.Sprintf("get organization: %s", err)) + } // Compute parameters for the workspace to consume. parameters, err := projectparameter.Compute(ctx, server.Database, projectparameter.Scope{ @@ -246,8 +244,8 @@ func (server *provisionerdServer) AcquireJob(ctx context.Context, _ *proto.Empty protoJob.Type = &proto.AcquiredJob_ProjectImport_{ ProjectImport: &proto.AcquiredJob_ProjectImport{ - ProjectVersionId: projectVersion.ID.String(), - ProjectVersionName: projectVersion.Name, + // This will be replaced once the project import has been refactored. + ProjectName: "placeholder", }, } } @@ -289,85 +287,35 @@ func (server *provisionerdServer) UpdateJob(stream proto.DRPCProvisionerDaemon_U if err != nil { return xerrors.Errorf("update job: %w", err) } - switch job.Type { - case database.ProvisionerJobTypeProjectImport: - if len(update.ProjectImportLogs) == 0 { - continue - } - var input projectImportJob - err = json.Unmarshal(job.Input, &input) - if err != nil { - return xerrors.Errorf("unmarshal job input %q: %s", job.Input, err) - } - insertParams := database.InsertProjectVersionLogsParams{ - ProjectVersionID: input.ProjectVersionID, - } - for _, log := range update.ProjectImportLogs { - logLevel, err := convertLogLevel(log.Level) - if err != nil { - return xerrors.Errorf("convert log level: %w", err) - } - logSource, err := convertLogSource(log.Source) - if err != nil { - return xerrors.Errorf("convert log source: %w", err) - } - insertParams.ID = append(insertParams.ID, uuid.New()) - insertParams.CreatedAt = append(insertParams.CreatedAt, time.UnixMilli(log.CreatedAt)) - insertParams.Level = append(insertParams.Level, logLevel) - insertParams.Source = append(insertParams.Source, logSource) - insertParams.Output = append(insertParams.Output, log.Output) - } - logs, err := server.Database.InsertProjectVersionLogs(stream.Context(), insertParams) - if err != nil { - return xerrors.Errorf("insert project logs: %w", err) - } - data, err := json.Marshal(logs) - if err != nil { - return xerrors.Errorf("marshal project log: %w", err) - } - err = server.Pubsub.Publish(projectVersionLogsChannel(input.ProjectVersionID), data) - if err != nil { - return xerrors.Errorf("publish history log: %w", err) - } - case database.ProvisionerJobTypeWorkspaceProvision: - if len(update.WorkspaceProvisionLogs) == 0 { - continue - } - var input workspaceProvisionJob - err = json.Unmarshal(job.Input, &input) - if err != nil { - return xerrors.Errorf("unmarshal job input %q: %s", job.Input, err) - } - insertParams := database.InsertWorkspaceHistoryLogsParams{ - WorkspaceHistoryID: input.WorkspaceHistoryID, - } - for _, log := range update.WorkspaceProvisionLogs { - logLevel, err := convertLogLevel(log.Level) - if err != nil { - return xerrors.Errorf("convert log level: %w", err) - } - logSource, err := convertLogSource(log.Source) - if err != nil { - return xerrors.Errorf("convert log source: %w", err) - } - insertParams.ID = append(insertParams.ID, uuid.New()) - insertParams.CreatedAt = append(insertParams.CreatedAt, time.UnixMilli(log.CreatedAt)) - insertParams.Level = append(insertParams.Level, logLevel) - insertParams.Source = append(insertParams.Source, logSource) - insertParams.Output = append(insertParams.Output, log.Output) - } - logs, err := server.Database.InsertWorkspaceHistoryLogs(stream.Context(), insertParams) - if err != nil { - return xerrors.Errorf("insert workspace logs: %w", err) - } - data, err := json.Marshal(logs) + insertParams := database.InsertProvisionerJobLogsParams{ + JobID: parsedID, + } + for _, log := range update.Logs { + logLevel, err := convertLogLevel(log.Level) if err != nil { - return xerrors.Errorf("marshal project log: %w", err) + return xerrors.Errorf("convert log level: %w", err) } - err = server.Pubsub.Publish(workspaceHistoryLogsChannel(input.WorkspaceHistoryID), data) + logSource, err := convertLogSource(log.Source) if err != nil { - return xerrors.Errorf("publish history log: %w", err) + return xerrors.Errorf("convert log source: %w", err) } + insertParams.ID = append(insertParams.ID, uuid.New()) + insertParams.CreatedAt = append(insertParams.CreatedAt, time.UnixMilli(log.CreatedAt)) + insertParams.Level = append(insertParams.Level, logLevel) + insertParams.Source = append(insertParams.Source, logSource) + insertParams.Output = append(insertParams.Output, log.Output) + } + logs, err := server.Database.InsertProvisionerJobLogs(stream.Context(), insertParams) + if err != nil { + return xerrors.Errorf("insert job logs: %w", err) + } + data, err := json.Marshal(logs) + if err != nil { + return xerrors.Errorf("marshal job log: %w", err) + } + err = server.Pubsub.Publish(provisionerJobLogsChannel(parsedID), data) + if err != nil { + return xerrors.Errorf("publish job log: %w", err) } } } diff --git a/coderd/workspacehistorylogs.go b/coderd/provisionerjoblogs.go similarity index 59% rename from coderd/workspacehistorylogs.go rename to coderd/provisionerjoblogs.go index 029167c0a7572..f47c7825a300f 100644 --- a/coderd/workspacehistorylogs.go +++ b/coderd/provisionerjoblogs.go @@ -14,14 +14,13 @@ import ( "github.com/google/uuid" "cdr.dev/slog" - "github.com/coder/coder/database" "github.com/coder/coder/httpapi" "github.com/coder/coder/httpmw" ) -// WorkspaceHistoryLog represents a single log from workspace history. -type WorkspaceHistoryLog struct { +// ProvisionerJobLog represents a single log from a provisioner job. +type ProvisionerJobLog struct { ID uuid.UUID CreatedAt time.Time `json:"created_at"` Source database.LogSource `json:"log_source"` @@ -29,14 +28,14 @@ type WorkspaceHistoryLog struct { Output string `json:"output"` } -// Returns workspace history logs based on query parameters. +// Returns provisioner logs based on query parameters. // The intended usage for a client to stream all logs (with JS API): // const timestamp = new Date().getTime(); // 1. GET /logs?before= // 2. GET /logs?after=&follow // The combination of these responses should provide all current logs // to the consumer, and future logs are streamed in the follow request. -func (api *api) workspaceHistoryLogsByName(rw http.ResponseWriter, r *http.Request) { +func (api *api) provisionerJobLogsByID(rw http.ResponseWriter, r *http.Request) { follow := r.URL.Query().Has("follow") afterRaw := r.URL.Query().Get("after") beforeRaw := r.URL.Query().Get("before") @@ -78,36 +77,36 @@ func (api *api) workspaceHistoryLogsByName(rw http.ResponseWriter, r *http.Reque before = database.Now() } - history := httpmw.WorkspaceHistoryParam(r) + job := httpmw.ProvisionerJobParam(r) if !follow { - logs, err := api.Database.GetWorkspaceHistoryLogsByIDBetween(r.Context(), database.GetWorkspaceHistoryLogsByIDBetweenParams{ - WorkspaceHistoryID: history.ID, - CreatedAfter: after, - CreatedBefore: before, + logs, err := api.Database.GetProvisionerLogsByIDBetween(r.Context(), database.GetProvisionerLogsByIDBetweenParams{ + JobID: job.ID, + CreatedAfter: after, + CreatedBefore: before, }) if errors.Is(err, sql.ErrNoRows) { err = nil } if err != nil { httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ - Message: fmt.Sprintf("get workspace history: %s", err), + Message: fmt.Sprintf("get provisioner logs: %s", err), }) return } if logs == nil { - logs = []database.WorkspaceHistoryLog{} + logs = []database.ProvisionerJobLog{} } render.Status(r, http.StatusOK) render.JSON(rw, r, logs) return } - bufferedLogs := make(chan database.WorkspaceHistoryLog, 128) - closeSubscribe, err := api.Pubsub.Subscribe(workspaceHistoryLogsChannel(history.ID), func(ctx context.Context, message []byte) { - var logs []database.WorkspaceHistoryLog + bufferedLogs := make(chan database.ProvisionerJobLog, 128) + closeSubscribe, err := api.Pubsub.Subscribe(provisionerJobLogsChannel(job.ID), func(ctx context.Context, message []byte) { + var logs []database.ProvisionerJobLog err := json.Unmarshal(message, &logs) if err != nil { - api.Logger.Warn(r.Context(), fmt.Sprintf("invalid workspace log on channel %q: %s", workspaceHistoryLogsChannel(history.ID), err.Error())) + api.Logger.Warn(r.Context(), fmt.Sprintf("invalid provisioner job log on channel %q: %s", provisionerJobLogsChannel(job.ID), err.Error())) return } @@ -117,30 +116,30 @@ func (api *api) workspaceHistoryLogsByName(rw http.ResponseWriter, r *http.Reque default: // If this overflows users could miss logs streaming. This can happen // if a database request takes a long amount of time, and we get a lot of logs. - api.Logger.Warn(r.Context(), "workspace history log overflowing channel") + api.Logger.Warn(r.Context(), "provisioner job log overflowing channel") } } }) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ - Message: fmt.Sprintf("subscribe to workspace history logs: %s", err), + Message: fmt.Sprintf("subscribe to provisioner job logs: %s", err), }) return } defer closeSubscribe() - workspaceHistoryLogs, err := api.Database.GetWorkspaceHistoryLogsByIDBetween(r.Context(), database.GetWorkspaceHistoryLogsByIDBetweenParams{ - WorkspaceHistoryID: history.ID, - CreatedAfter: after, - CreatedBefore: before, + provisionerJobLogs, err := api.Database.GetProvisionerLogsByIDBetween(r.Context(), database.GetProvisionerLogsByIDBetweenParams{ + JobID: job.ID, + CreatedAfter: after, + CreatedBefore: before, }) if errors.Is(err, sql.ErrNoRows) { err = nil - workspaceHistoryLogs = []database.WorkspaceHistoryLog{} + provisionerJobLogs = []database.ProvisionerJobLog{} } if err != nil { httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ - Message: fmt.Sprint("get workspace history logs: %w", err), + Message: fmt.Sprint("get provisioner job logs: %w", err), }) return } @@ -154,8 +153,8 @@ func (api *api) workspaceHistoryLogsByName(rw http.ResponseWriter, r *http.Reque // The Go stdlib JSON encoder appends a newline character after message write. encoder := json.NewEncoder(rw) - for _, workspaceHistoryLog := range workspaceHistoryLogs { - err = encoder.Encode(convertWorkspaceHistoryLog(workspaceHistoryLog)) + for _, provisionerJobLog := range provisionerJobLogs { + err = encoder.Encode(convertProvisionerJobLog(provisionerJobLog)) if err != nil { return } @@ -168,15 +167,15 @@ func (api *api) workspaceHistoryLogsByName(rw http.ResponseWriter, r *http.Reque case <-r.Context().Done(): return case log := <-bufferedLogs: - err = encoder.Encode(convertWorkspaceHistoryLog(log)) + err = encoder.Encode(convertProvisionerJobLog(log)) if err != nil { return } rw.(http.Flusher).Flush() case <-ticker.C: - job, err := api.Database.GetProvisionerJobByID(r.Context(), history.ProvisionJobID) + job, err := api.Database.GetProvisionerJobByID(r.Context(), job.ID) if err != nil { - api.Logger.Warn(r.Context(), "streaming workspace logs; checking if job completed", slog.Error(err), slog.F("job_id", history.ProvisionJobID)) + api.Logger.Warn(r.Context(), "streaming job logs; checking if completed", slog.Error(err), slog.F("job_id", job.ID.String())) continue } if convertProvisionerJob(job).Status.Completed() { @@ -186,16 +185,12 @@ func (api *api) workspaceHistoryLogsByName(rw http.ResponseWriter, r *http.Reque } } -func convertWorkspaceHistoryLog(workspaceHistoryLog database.WorkspaceHistoryLog) WorkspaceHistoryLog { - return WorkspaceHistoryLog{ - ID: workspaceHistoryLog.ID, - CreatedAt: workspaceHistoryLog.CreatedAt, - Source: workspaceHistoryLog.Source, - Level: workspaceHistoryLog.Level, - Output: workspaceHistoryLog.Output, +func convertProvisionerJobLog(provisionerJobLog database.ProvisionerJobLog) ProvisionerJobLog { + return ProvisionerJobLog{ + ID: provisionerJobLog.ID, + CreatedAt: provisionerJobLog.CreatedAt, + Source: provisionerJobLog.Source, + Level: provisionerJobLog.Level, + Output: provisionerJobLog.Output, } } - -func workspaceHistoryLogsChannel(workspaceHistoryID uuid.UUID) string { - return fmt.Sprintf("workspace-history-logs:%s", workspaceHistoryID) -} diff --git a/coderd/workspacehistorylogs_test.go b/coderd/provisionerjoblogs_test.go similarity index 90% rename from coderd/workspacehistorylogs_test.go rename to coderd/provisionerjoblogs_test.go index 1e95a0f506cc1..454b3bb85caec 100644 --- a/coderd/workspacehistorylogs_test.go +++ b/coderd/provisionerjoblogs_test.go @@ -46,7 +46,7 @@ func TestWorkspaceHistoryLogsByName(t *testing.T) { require.NoError(t, err) // Successfully return empty logs before the job starts! - logs, err := client.WorkspaceHistoryLogs(context.Background(), "", workspace.Name, history.Name) + logs, err := client.ProvisionerJobLogs(context.Background(), history.Provision.ID) require.NoError(t, err) require.NotNil(t, logs) require.Len(t, logs, 0) @@ -54,7 +54,7 @@ func TestWorkspaceHistoryLogsByName(t *testing.T) { coderdtest.AwaitWorkspaceHistoryProvisioned(t, client, "", workspace.Name, history.Name) // Return the log after completion! - logs, err = client.WorkspaceHistoryLogs(context.Background(), "", workspace.Name, history.Name) + logs, err = client.ProvisionerJobLogs(context.Background(), history.Provision.ID) require.NoError(t, err) require.NotNil(t, logs) require.Len(t, logs, 1) @@ -91,12 +91,13 @@ func TestWorkspaceHistoryLogsByName(t *testing.T) { require.NoError(t, err) coderdtest.AwaitWorkspaceHistoryProvisioned(t, client, "", workspace.Name, history.Name) - logs, err := client.FollowWorkspaceHistoryLogsAfter(context.Background(), "", workspace.Name, history.Name, before) + logs, err := client.FollowProvisionerJobLogsAfter(context.Background(), history.Provision.ID, before) require.NoError(t, err) - log := <-logs + log, ok := <-logs + require.True(t, ok) require.Equal(t, "log-output", log.Output) // Make sure the channel automatically closes! - _, ok := <-logs + _, ok = <-logs require.False(t, ok) }) @@ -129,7 +130,7 @@ func TestWorkspaceHistoryLogsByName(t *testing.T) { }) require.NoError(t, err) - logs, err := client.FollowWorkspaceHistoryLogsAfter(context.Background(), "", workspace.Name, history.Name, time.Time{}) + logs, err := client.FollowProvisionerJobLogsAfter(context.Background(), history.Provision.ID, time.Time{}) require.NoError(t, err) log := <-logs require.Equal(t, "log-output", log.Output) diff --git a/coderd/provisioners.go b/coderd/provisioners.go index f2cd46eb4b763..4e998afd9ede8 100644 --- a/coderd/provisioners.go +++ b/coderd/provisioners.go @@ -1,6 +1,7 @@ package coderd import ( + "fmt" "time" "github.com/google/uuid" @@ -24,6 +25,7 @@ const ( ) type ProvisionerJob struct { + ID uuid.UUID `json:"id"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` StartedAt *time.Time `json:"started_at,omitempty"` @@ -37,6 +39,7 @@ type ProvisionerJob struct { func convertProvisionerJob(provisionerJob database.ProvisionerJob) ProvisionerJob { job := ProvisionerJob{ + ID: provisionerJob.ID, CreatedAt: provisionerJob.CreatedAt, UpdatedAt: provisionerJob.UpdatedAt, Error: provisionerJob.Error.String, @@ -76,3 +79,7 @@ func convertProvisionerJob(provisionerJob database.ProvisionerJob) ProvisionerJo return job } + +func provisionerJobLogsChannel(jobID uuid.UUID) string { + return fmt.Sprintf("provisioner-log-logs:%s", jobID) +} diff --git a/coderd/workspacehistory.go b/coderd/workspacehistory.go index 5d7cfac8ef0fe..7f3afd7670901 100644 --- a/coderd/workspacehistory.go +++ b/coderd/workspacehistory.go @@ -157,7 +157,6 @@ func (api *api) postWorkspaceHistoryByUser(rw http.ResponseWriter, r *http.Reque InitiatorID: user.ID, Provisioner: project.Provisioner, Type: database.ProvisionerJobTypeWorkspaceProvision, - ProjectID: project.ID, Input: input, }) if err != nil { diff --git a/codersdk/provisioners.go b/codersdk/provisioners.go index 5f96271a4218d..f2685ccb939ad 100644 --- a/codersdk/provisioners.go +++ b/codersdk/provisioners.go @@ -3,9 +3,14 @@ package codersdk import ( "context" "encoding/json" + "fmt" "io" "net/http" + "net/url" + "strconv" + "time" + "github.com/google/uuid" "github.com/hashicorp/yamux" "golang.org/x/xerrors" "nhooyr.io/websocket" @@ -53,3 +58,67 @@ func (c *Client) ProvisionerDaemonClient(ctx context.Context) (proto.DRPCProvisi } return proto.NewDRPCProvisionerDaemonClient(provisionersdk.Conn(session)), nil } + +// ProvisionerJobLogs returns all logs for workspace history. +// To stream logs, use the FollowProvisionerJobLogs function. +func (c *Client) ProvisionerJobLogs(ctx context.Context, jobID uuid.UUID) ([]coderd.ProvisionerJobLog, error) { + return c.ProvisionerJobLogsBetween(ctx, jobID, time.Time{}, time.Time{}) +} + +// ProvisionerJobLogsBetween returns logs between a specific time. +func (c *Client) ProvisionerJobLogsBetween(ctx context.Context, jobID uuid.UUID, after, before time.Time) ([]coderd.ProvisionerJobLog, error) { + values := url.Values{} + if !after.IsZero() { + values["after"] = []string{strconv.FormatInt(after.UTC().UnixMilli(), 10)} + } + if !before.IsZero() { + values["before"] = []string{strconv.FormatInt(before.UTC().UnixMilli(), 10)} + } + res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/provisioners/jobs/%s/logs?%s", jobID, values.Encode()), nil) + if err != nil { + return nil, err + } + if res.StatusCode != http.StatusOK { + defer res.Body.Close() + return nil, readBodyAsError(res) + } + + var logs []coderd.ProvisionerJobLog + return logs, json.NewDecoder(res.Body).Decode(&logs) +} + +// FollowProvisionerJobLogsAfter returns a stream of workspace history logs. +// The channel will close when the workspace history job is no longer active. +func (c *Client) FollowProvisionerJobLogsAfter(ctx context.Context, jobID uuid.UUID, after time.Time) (<-chan coderd.ProvisionerJobLog, error) { + afterQuery := "" + if !after.IsZero() { + afterQuery = fmt.Sprintf("&after=%d", after.UTC().UnixMilli()) + } + res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/provisioners/jobs/%s/logs?follow%s", jobID, afterQuery), nil) + if err != nil { + return nil, err + } + if res.StatusCode != http.StatusOK { + defer res.Body.Close() + return nil, readBodyAsError(res) + } + + logs := make(chan coderd.ProvisionerJobLog) + decoder := json.NewDecoder(res.Body) + go func() { + defer close(logs) + var log coderd.ProvisionerJobLog + for { + err = decoder.Decode(&log) + if err != nil { + return + } + select { + case <-ctx.Done(): + return + case logs <- log: + } + } + }() + return logs, nil +} diff --git a/codersdk/provisioners_test.go b/codersdk/provisioners_test.go index 9bb4528ebec1e..7b6b4194b2645 100644 --- a/codersdk/provisioners_test.go +++ b/codersdk/provisioners_test.go @@ -3,11 +3,17 @@ package codersdk_test import ( "context" "testing" + "time" + "github.com/google/uuid" "github.com/stretchr/testify/require" + "github.com/coder/coder/coderd" "github.com/coder/coder/coderd/coderdtest" + "github.com/coder/coder/database" + "github.com/coder/coder/provisioner/echo" "github.com/coder/coder/provisionerd/proto" + sdkproto "github.com/coder/coder/provisionersdk/proto" ) func TestProvisionerDaemons(t *testing.T) { @@ -44,3 +50,76 @@ func TestProvisionerDaemonClient(t *testing.T) { require.NoError(t, err) }) } +func TestProvisionerJobLogs(t *testing.T) { + t.Parallel() + t.Run("Error", func(t *testing.T) { + t.Parallel() + client := coderdtest.New(t) + _, err := client.ProvisionerJobLogs(context.Background(), uuid.New()) + require.Error(t, err) + }) + + t.Run("List", func(t *testing.T) { + t.Parallel() + client := coderdtest.New(t) + user := coderdtest.CreateInitialUser(t, client) + _ = coderdtest.NewProvisionerDaemon(t, client) + project := coderdtest.CreateProject(t, client, user.Organization) + version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, nil) + coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name) + workspace := coderdtest.CreateWorkspace(t, client, "", project.ID) + history, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{ + ProjectVersionID: version.ID, + Transition: database.WorkspaceTransitionCreate, + }) + require.NoError(t, err) + _, err = client.ProvisionerJobLogs(context.Background(), history.Provision.ID) + require.NoError(t, err) + }) +} + +func TestFollowProvisionerJobLogsAfter(t *testing.T) { + t.Parallel() + t.Run("Error", func(t *testing.T) { + t.Parallel() + client := coderdtest.New(t) + _, err := client.FollowProvisionerJobLogsAfter(context.Background(), uuid.New(), time.Time{}) + require.Error(t, err) + }) + + t.Run("Stream", func(t *testing.T) { + t.Parallel() + client := coderdtest.New(t) + user := coderdtest.CreateInitialUser(t, client) + _ = coderdtest.NewProvisionerDaemon(t, client) + project := coderdtest.CreateProject(t, client, user.Organization) + version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, &echo.Responses{ + Parse: echo.ParseComplete, + Provision: []*sdkproto.Provision_Response{{ + Type: &sdkproto.Provision_Response_Log{ + Log: &sdkproto.Log{ + Output: "hello", + }, + }, + }, { + Type: &sdkproto.Provision_Response_Complete{ + Complete: &sdkproto.Provision_Complete{}, + }, + }}, + }) + coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name) + workspace := coderdtest.CreateWorkspace(t, client, "", project.ID) + after := database.Now() + history, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{ + ProjectVersionID: version.ID, + Transition: database.WorkspaceTransitionCreate, + }) + require.NoError(t, err) + logs, err := client.FollowProvisionerJobLogsAfter(context.Background(), history.Provision.ID, after) + require.NoError(t, err) + _, ok := <-logs + require.True(t, ok) + _, ok = <-logs + require.False(t, ok) + }) +} diff --git a/codersdk/workspaces.go b/codersdk/workspaces.go index 82c2ce2853c0f..b84efeff628d1 100644 --- a/codersdk/workspaces.go +++ b/codersdk/workspaces.go @@ -5,9 +5,6 @@ import ( "encoding/json" "fmt" "net/http" - "net/url" - "strconv" - "time" "github.com/coder/coder/coderd" ) @@ -134,74 +131,3 @@ func (c *Client) CreateWorkspaceHistory(ctx context.Context, owner, workspace st var workspaceHistory coderd.WorkspaceHistory return workspaceHistory, json.NewDecoder(res.Body).Decode(&workspaceHistory) } - -// WorkspaceHistoryLogs returns all logs for workspace history. -// To stream logs, use the FollowWorkspaceHistoryLogs function. -func (c *Client) WorkspaceHistoryLogs(ctx context.Context, owner, workspace, history string) ([]coderd.WorkspaceHistoryLog, error) { - return c.WorkspaceHistoryLogsBetween(ctx, owner, workspace, history, time.Time{}, time.Time{}) -} - -// WorkspaceHistoryLogsBetween returns logs between a specific time. -func (c *Client) WorkspaceHistoryLogsBetween(ctx context.Context, owner, workspace, history string, after, before time.Time) ([]coderd.WorkspaceHistoryLog, error) { - if owner == "" { - owner = "me" - } - values := url.Values{} - if !after.IsZero() { - values["after"] = []string{strconv.FormatInt(after.UTC().UnixMilli(), 10)} - } - if !before.IsZero() { - values["before"] = []string{strconv.FormatInt(before.UTC().UnixMilli(), 10)} - } - res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/workspaces/%s/%s/version/%s/logs?%s", owner, workspace, history, values.Encode()), nil) - if err != nil { - return nil, err - } - if res.StatusCode != http.StatusOK { - defer res.Body.Close() - return nil, readBodyAsError(res) - } - - var logs []coderd.WorkspaceHistoryLog - return logs, json.NewDecoder(res.Body).Decode(&logs) -} - -// FollowWorkspaceHistoryLogsAfter returns a stream of workspace history logs. -// The channel will close when the workspace history job is no longer active. -func (c *Client) FollowWorkspaceHistoryLogsAfter(ctx context.Context, owner, workspace, history string, after time.Time) (<-chan coderd.WorkspaceHistoryLog, error) { - afterQuery := "" - if !after.IsZero() { - afterQuery = fmt.Sprintf("&after=%d", after.UTC().UnixMilli()) - } - if owner == "" { - owner = "me" - } - - res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/workspaces/%s/%s/version/%s/logs?follow%s", owner, workspace, history, afterQuery), nil) - if err != nil { - return nil, err - } - if res.StatusCode != http.StatusOK { - defer res.Body.Close() - return nil, readBodyAsError(res) - } - - logs := make(chan coderd.WorkspaceHistoryLog) - decoder := json.NewDecoder(res.Body) - go func() { - defer close(logs) - var log coderd.WorkspaceHistoryLog - for { - err = decoder.Decode(&log) - if err != nil { - return - } - select { - case <-ctx.Done(): - return - case logs <- log: - } - } - }() - return logs, nil -} diff --git a/codersdk/workspaces_test.go b/codersdk/workspaces_test.go index e5d170c170311..e3d303f9b97a2 100644 --- a/codersdk/workspaces_test.go +++ b/codersdk/workspaces_test.go @@ -3,15 +3,12 @@ package codersdk_test import ( "context" "testing" - "time" "github.com/stretchr/testify/require" "github.com/coder/coder/coderd" "github.com/coder/coder/coderd/coderdtest" "github.com/coder/coder/database" - "github.com/coder/coder/provisioner/echo" - "github.com/coder/coder/provisionersdk/proto" ) func TestWorkspaces(t *testing.T) { @@ -160,77 +157,3 @@ func TestCreateWorkspaceHistory(t *testing.T) { require.NoError(t, err) }) } - -func TestWorkspaceHistoryLogs(t *testing.T) { - t.Parallel() - t.Run("Error", func(t *testing.T) { - t.Parallel() - client := coderdtest.New(t) - _, err := client.WorkspaceHistoryLogs(context.Background(), "", "", "") - require.Error(t, err) - }) - - t.Run("List", func(t *testing.T) { - t.Parallel() - client := coderdtest.New(t) - user := coderdtest.CreateInitialUser(t, client) - _ = coderdtest.NewProvisionerDaemon(t, client) - project := coderdtest.CreateProject(t, client, user.Organization) - version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, nil) - coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name) - workspace := coderdtest.CreateWorkspace(t, client, "", project.ID) - history, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{ - ProjectVersionID: version.ID, - Transition: database.WorkspaceTransitionCreate, - }) - require.NoError(t, err) - _, err = client.WorkspaceHistoryLogs(context.Background(), "", workspace.Name, history.Name) - require.NoError(t, err) - }) -} - -func TestFollowWorkspaceHistoryLogsAfter(t *testing.T) { - t.Parallel() - t.Run("Error", func(t *testing.T) { - t.Parallel() - client := coderdtest.New(t) - _, err := client.FollowWorkspaceHistoryLogsAfter(context.Background(), "", "", "", time.Time{}) - require.Error(t, err) - }) - - t.Run("Stream", func(t *testing.T) { - t.Parallel() - client := coderdtest.New(t) - user := coderdtest.CreateInitialUser(t, client) - _ = coderdtest.NewProvisionerDaemon(t, client) - project := coderdtest.CreateProject(t, client, user.Organization) - version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, &echo.Responses{ - Parse: echo.ParseComplete, - Provision: []*proto.Provision_Response{{ - Type: &proto.Provision_Response_Log{ - Log: &proto.Log{ - Output: "hello", - }, - }, - }, { - Type: &proto.Provision_Response_Complete{ - Complete: &proto.Provision_Complete{}, - }, - }}, - }) - coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name) - workspace := coderdtest.CreateWorkspace(t, client, "", project.ID) - after := database.Now() - history, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{ - ProjectVersionID: version.ID, - Transition: database.WorkspaceTransitionCreate, - }) - require.NoError(t, err) - logs, err := client.FollowWorkspaceHistoryLogsAfter(context.Background(), "", workspace.Name, history.Name, after) - require.NoError(t, err) - _, ok := <-logs - require.True(t, ok) - _, ok = <-logs - require.False(t, ok) - }) -} diff --git a/database/databasefake/databasefake.go b/database/databasefake/databasefake.go index 2ee1bf965e878..3b3d09cd25ab5 100644 --- a/database/databasefake/databasefake.go +++ b/database/databasefake/databasefake.go @@ -19,18 +19,18 @@ func New() database.Store { organizationMembers: make([]database.OrganizationMember, 0), users: make([]database.User, 0), - parameterValue: make([]database.ParameterValue, 0), - project: make([]database.Project, 0), - projectVersion: make([]database.ProjectVersion, 0), - projectVersionLog: make([]database.ProjectVersionLog, 0), - projectParameter: make([]database.ProjectParameter, 0), - provisionerDaemons: make([]database.ProvisionerDaemon, 0), - provisionerJobs: make([]database.ProvisionerJob, 0), - workspace: make([]database.Workspace, 0), - workspaceResource: make([]database.WorkspaceResource, 0), - workspaceHistory: make([]database.WorkspaceHistory, 0), - workspaceHistoryLog: make([]database.WorkspaceHistoryLog, 0), - workspaceAgent: make([]database.WorkspaceAgent, 0), + files: make([]database.File, 0), + parameterValue: make([]database.ParameterValue, 0), + project: make([]database.Project, 0), + projectVersion: make([]database.ProjectVersion, 0), + projectParameter: make([]database.ProjectParameter, 0), + provisionerDaemons: make([]database.ProvisionerDaemon, 0), + provisionerJobs: make([]database.ProvisionerJob, 0), + provisionerJobLog: make([]database.ProvisionerJobLog, 0), + workspace: make([]database.Workspace, 0), + workspaceResource: make([]database.WorkspaceResource, 0), + workspaceHistory: make([]database.WorkspaceHistory, 0), + workspaceAgent: make([]database.WorkspaceAgent, 0), } } @@ -40,23 +40,23 @@ type fakeQuerier struct { // Legacy tables apiKeys []database.APIKey + files []database.File organizations []database.Organization organizationMembers []database.OrganizationMember users []database.User // New tables - parameterValue []database.ParameterValue - project []database.Project - projectVersion []database.ProjectVersion - projectVersionLog []database.ProjectVersionLog - projectParameter []database.ProjectParameter - provisionerDaemons []database.ProvisionerDaemon - provisionerJobs []database.ProvisionerJob - workspace []database.Workspace - workspaceAgent []database.WorkspaceAgent - workspaceHistory []database.WorkspaceHistory - workspaceHistoryLog []database.WorkspaceHistoryLog - workspaceResource []database.WorkspaceResource + parameterValue []database.ParameterValue + project []database.Project + projectVersion []database.ProjectVersion + projectParameter []database.ProjectParameter + provisionerDaemons []database.ProvisionerDaemon + provisionerJobs []database.ProvisionerJob + provisionerJobLog []database.ProvisionerJobLog + workspace []database.Workspace + workspaceAgent []database.WorkspaceAgent + workspaceHistory []database.WorkspaceHistory + workspaceResource []database.WorkspaceResource } // InTx doesn't rollback data properly for in-memory yet. @@ -104,6 +104,18 @@ func (q *fakeQuerier) GetAPIKeyByID(_ context.Context, id string) (database.APIK return database.APIKey{}, sql.ErrNoRows } +func (q *fakeQuerier) GetFileByID(_ context.Context, id uuid.UUID) (database.File, error) { + q.mutex.Lock() + defer q.mutex.Unlock() + + for _, file := range q.files { + if file.ID.String() == id.String() { + return file, nil + } + } + return database.File{}, sql.ErrNoRows +} + func (q *fakeQuerier) GetUserByEmailOrUsername(_ context.Context, arg database.GetUserByEmailOrUsernameParams) (database.User, error) { q.mutex.Lock() defer q.mutex.Unlock() @@ -224,29 +236,6 @@ func (q *fakeQuerier) GetWorkspaceHistoryByWorkspaceIDWithoutAfter(_ context.Con return database.WorkspaceHistory{}, sql.ErrNoRows } -func (q *fakeQuerier) GetWorkspaceHistoryLogsByIDBetween(_ context.Context, arg database.GetWorkspaceHistoryLogsByIDBetweenParams) ([]database.WorkspaceHistoryLog, error) { - q.mutex.Lock() - defer q.mutex.Unlock() - - logs := make([]database.WorkspaceHistoryLog, 0) - for _, workspaceHistoryLog := range q.workspaceHistoryLog { - if workspaceHistoryLog.WorkspaceHistoryID.String() != arg.WorkspaceHistoryID.String() { - continue - } - if workspaceHistoryLog.CreatedAt.After(arg.CreatedBefore) { - continue - } - if workspaceHistoryLog.CreatedAt.Before(arg.CreatedAfter) { - continue - } - logs = append(logs, workspaceHistoryLog) - } - if len(logs) == 0 { - return nil, sql.ErrNoRows - } - return logs, nil -} - func (q *fakeQuerier) GetWorkspaceHistoryByWorkspaceID(_ context.Context, workspaceID uuid.UUID) ([]database.WorkspaceHistory, error) { q.mutex.Lock() defer q.mutex.Unlock() @@ -443,29 +432,6 @@ func (q *fakeQuerier) GetProjectVersionByProjectIDAndName(_ context.Context, arg return database.ProjectVersion{}, sql.ErrNoRows } -func (q *fakeQuerier) GetProjectVersionLogsByIDBetween(_ context.Context, arg database.GetProjectVersionLogsByIDBetweenParams) ([]database.ProjectVersionLog, error) { - q.mutex.Lock() - defer q.mutex.Unlock() - - logs := make([]database.ProjectVersionLog, 0) - for _, projectVersionLog := range q.projectVersionLog { - if projectVersionLog.ProjectVersionID.String() != arg.ProjectVersionID.String() { - continue - } - if projectVersionLog.CreatedAt.After(arg.CreatedBefore) { - continue - } - if projectVersionLog.CreatedAt.Before(arg.CreatedAfter) { - continue - } - logs = append(logs, projectVersionLog) - } - if len(logs) == 0 { - return nil, sql.ErrNoRows - } - return logs, nil -} - func (q *fakeQuerier) GetProjectVersionByID(_ context.Context, projectVersionID uuid.UUID) (database.ProjectVersion, error) { q.mutex.Lock() defer q.mutex.Unlock() @@ -567,6 +533,29 @@ func (q *fakeQuerier) GetProvisionerJobByID(_ context.Context, id uuid.UUID) (da return database.ProvisionerJob{}, sql.ErrNoRows } +func (q *fakeQuerier) GetProvisionerLogsByIDBetween(_ context.Context, arg database.GetProvisionerLogsByIDBetweenParams) ([]database.ProvisionerJobLog, error) { + q.mutex.Lock() + defer q.mutex.Unlock() + + logs := make([]database.ProvisionerJobLog, 0) + for _, jobLog := range q.provisionerJobLog { + if jobLog.JobID.String() != arg.JobID.String() { + continue + } + if jobLog.CreatedAt.After(arg.CreatedBefore) { + continue + } + if jobLog.CreatedAt.Before(arg.CreatedAfter) { + continue + } + logs = append(logs, jobLog) + } + if len(logs) == 0 { + return nil, sql.ErrNoRows + } + return logs, nil +} + func (q *fakeQuerier) InsertAPIKey(_ context.Context, arg database.InsertAPIKeyParams) (database.APIKey, error) { q.mutex.Lock() defer q.mutex.Unlock() @@ -593,6 +582,20 @@ func (q *fakeQuerier) InsertAPIKey(_ context.Context, arg database.InsertAPIKeyP return key, nil } +func (q *fakeQuerier) InsertFile(_ context.Context, arg database.InsertFileParams) (database.File, error) { + q.mutex.Lock() + defer q.mutex.Unlock() + + //nolint:gosimple + file := database.File{ + ID: arg.ID, + CreatedAt: arg.CreatedAt, + Data: arg.Data, + } + q.files = append(q.files, file) + return file, nil +} + func (q *fakeQuerier) InsertOrganization(_ context.Context, arg database.InsertOrganizationParams) (database.Organization, error) { q.mutex.Lock() defer q.mutex.Unlock() @@ -680,22 +683,22 @@ func (q *fakeQuerier) InsertProjectVersion(_ context.Context, arg database.Inser return version, nil } -func (q *fakeQuerier) InsertProjectVersionLogs(_ context.Context, arg database.InsertProjectVersionLogsParams) ([]database.ProjectVersionLog, error) { +func (q *fakeQuerier) InsertProvisionerJobLogs(_ context.Context, arg database.InsertProvisionerJobLogsParams) ([]database.ProvisionerJobLog, error) { q.mutex.Lock() defer q.mutex.Unlock() - logs := make([]database.ProjectVersionLog, 0) + logs := make([]database.ProvisionerJobLog, 0) for index, output := range arg.Output { - logs = append(logs, database.ProjectVersionLog{ - ProjectVersionID: arg.ProjectVersionID, - ID: arg.ID[index], - CreatedAt: arg.CreatedAt[index], - Source: arg.Source[index], - Level: arg.Level[index], - Output: output, + logs = append(logs, database.ProvisionerJobLog{ + JobID: arg.JobID, + ID: arg.ID[index], + CreatedAt: arg.CreatedAt[index], + Source: arg.Source[index], + Level: arg.Level[index], + Output: output, }) } - q.projectVersionLog = append(q.projectVersionLog, logs...) + q.provisionerJobLog = append(q.provisionerJobLog, logs...) return logs, nil } @@ -751,7 +754,6 @@ func (q *fakeQuerier) InsertProvisionerJob(_ context.Context, arg database.Inser UpdatedAt: arg.UpdatedAt, InitiatorID: arg.InitiatorID, Provisioner: arg.Provisioner, - ProjectID: arg.ProjectID, Type: arg.Type, Input: arg.Input, } @@ -832,25 +834,6 @@ func (q *fakeQuerier) InsertWorkspaceHistory(_ context.Context, arg database.Ins return workspaceHistory, nil } -func (q *fakeQuerier) InsertWorkspaceHistoryLogs(_ context.Context, arg database.InsertWorkspaceHistoryLogsParams) ([]database.WorkspaceHistoryLog, error) { - q.mutex.Lock() - defer q.mutex.Unlock() - - logs := make([]database.WorkspaceHistoryLog, 0) - for index, output := range arg.Output { - logs = append(logs, database.WorkspaceHistoryLog{ - WorkspaceHistoryID: arg.WorkspaceHistoryID, - ID: arg.ID[index], - CreatedAt: arg.CreatedAt[index], - Source: arg.Source[index], - Level: arg.Level[index], - Output: output, - }) - } - q.workspaceHistoryLog = append(q.workspaceHistoryLog, logs...) - return logs, nil -} - func (q *fakeQuerier) InsertWorkspaceResource(_ context.Context, arg database.InsertWorkspaceResourceParams) (database.WorkspaceResource, error) { q.mutex.Lock() defer q.mutex.Unlock() diff --git a/database/dump.sql b/database/dump.sql index 62ca942f03e67..8034b3d0ad517 100644 --- a/database/dump.sql +++ b/database/dump.sql @@ -87,6 +87,12 @@ CREATE TABLE api_keys ( devurl_token boolean DEFAULT false NOT NULL ); +CREATE TABLE files ( + id uuid NOT NULL, + created_at timestamp with time zone NOT NULL, + data bytea NOT NULL +); + CREATE TABLE licenses ( id integer NOT NULL, license jsonb NOT NULL, @@ -169,15 +175,6 @@ CREATE TABLE project_version ( import_job_id uuid NOT NULL ); -CREATE TABLE project_version_log ( - id uuid NOT NULL, - project_version_id uuid NOT NULL, - created_at timestamp with time zone NOT NULL, - source log_source NOT NULL, - level log_level NOT NULL, - output character varying(1024) NOT NULL -); - CREATE TABLE provisioner_daemon ( id uuid NOT NULL, created_at timestamp with time zone NOT NULL, @@ -197,11 +194,19 @@ CREATE TABLE provisioner_job ( initiator_id text NOT NULL, provisioner provisioner_type NOT NULL, type provisioner_job_type NOT NULL, - project_id uuid NOT NULL, input jsonb NOT NULL, worker_id uuid ); +CREATE TABLE provisioner_job_log ( + id uuid NOT NULL, + job_id uuid NOT NULL, + created_at timestamp with time zone NOT NULL, + source log_source NOT NULL, + level log_level NOT NULL, + output character varying(1024) NOT NULL +); + CREATE TABLE users ( id text NOT NULL, email text NOT NULL, @@ -257,15 +262,6 @@ CREATE TABLE workspace_history ( provision_job_id uuid NOT NULL ); -CREATE TABLE workspace_history_log ( - id uuid NOT NULL, - workspace_history_id uuid NOT NULL, - created_at timestamp with time zone NOT NULL, - source log_source NOT NULL, - level log_level NOT NULL, - output character varying(1024) NOT NULL -); - CREATE TABLE workspace_resource ( id uuid NOT NULL, created_at timestamp with time zone NOT NULL, @@ -276,6 +272,9 @@ CREATE TABLE workspace_resource ( workspace_agent_id uuid ); +ALTER TABLE ONLY files + ADD CONSTRAINT files_id_key UNIQUE (id); + ALTER TABLE ONLY parameter_value ADD CONSTRAINT parameter_value_id_key UNIQUE (id); @@ -297,9 +296,6 @@ ALTER TABLE ONLY project_parameter ALTER TABLE ONLY project_version ADD CONSTRAINT project_version_id_key UNIQUE (id); -ALTER TABLE ONLY project_version_log - ADD CONSTRAINT project_version_log_id_key UNIQUE (id); - ALTER TABLE ONLY project_version ADD CONSTRAINT project_version_project_id_name_key UNIQUE (project_id, name); @@ -312,15 +308,15 @@ ALTER TABLE ONLY provisioner_daemon ALTER TABLE ONLY provisioner_job ADD CONSTRAINT provisioner_job_id_key UNIQUE (id); +ALTER TABLE ONLY provisioner_job_log + ADD CONSTRAINT provisioner_job_log_id_key UNIQUE (id); + ALTER TABLE ONLY workspace_agent ADD CONSTRAINT workspace_agent_id_key UNIQUE (id); ALTER TABLE ONLY workspace_history ADD CONSTRAINT workspace_history_id_key UNIQUE (id); -ALTER TABLE ONLY workspace_history_log - ADD CONSTRAINT workspace_history_log_id_key UNIQUE (id); - ALTER TABLE ONLY workspace_history ADD CONSTRAINT workspace_history_workspace_id_name_key UNIQUE (workspace_id, name); @@ -342,21 +338,15 @@ ALTER TABLE ONLY workspace_resource ALTER TABLE ONLY project_parameter ADD CONSTRAINT project_parameter_project_version_id_fkey FOREIGN KEY (project_version_id) REFERENCES project_version(id) ON DELETE CASCADE; -ALTER TABLE ONLY project_version_log - ADD CONSTRAINT project_version_log_project_version_id_fkey FOREIGN KEY (project_version_id) REFERENCES project_version(id) ON DELETE CASCADE; - ALTER TABLE ONLY project_version ADD CONSTRAINT project_version_project_id_fkey FOREIGN KEY (project_id) REFERENCES project(id); -ALTER TABLE ONLY provisioner_job - ADD CONSTRAINT provisioner_job_project_id_fkey FOREIGN KEY (project_id) REFERENCES project(id) ON DELETE CASCADE; +ALTER TABLE ONLY provisioner_job_log + ADD CONSTRAINT provisioner_job_log_job_id_fkey FOREIGN KEY (job_id) REFERENCES provisioner_job(id) ON DELETE CASCADE; ALTER TABLE ONLY workspace_agent ADD CONSTRAINT workspace_agent_workspace_resource_id_fkey FOREIGN KEY (workspace_resource_id) REFERENCES workspace_resource(id) ON DELETE CASCADE; -ALTER TABLE ONLY workspace_history_log - ADD CONSTRAINT workspace_history_log_workspace_history_id_fkey FOREIGN KEY (workspace_history_id) REFERENCES workspace_history(id) ON DELETE CASCADE; - ALTER TABLE ONLY workspace_history ADD CONSTRAINT workspace_history_project_version_id_fkey FOREIGN KEY (project_version_id) REFERENCES project_version(id) ON DELETE CASCADE; diff --git a/database/migrations/000002_projects.up.sql b/database/migrations/000002_projects.up.sql index c33f2f1e725b8..21194025fc0b2 100644 --- a/database/migrations/000002_projects.up.sql +++ b/database/migrations/000002_projects.up.sql @@ -1,3 +1,10 @@ +-- Store arbitrary data like project source code or avatars. +CREATE TABLE files ( + id uuid NOT NULL UNIQUE, + created_at timestamptz NOT NULL, + data bytea NOT NULL +); + CREATE TYPE provisioner_type AS ENUM ('echo', 'terraform'); -- Project defines infrastructure that your software project @@ -90,25 +97,3 @@ CREATE TABLE project_parameter ( validation_value_type varchar(64) NOT NULL, UNIQUE(project_version_id, name) ); - -CREATE TYPE log_level AS ENUM ( - 'trace', - 'debug', - 'info', - 'warn', - 'error' -); - -CREATE TYPE log_source AS ENUM ( - 'provisioner_daemon', - 'provisioner' -); - -CREATE TABLE project_version_log ( - id uuid NOT NULL UNIQUE, - project_version_id uuid NOT NULL REFERENCES project_version (id) ON DELETE CASCADE, - created_at timestamptz NOT NULL, - source log_source NOT NULL, - level log_level NOT NULL, - output varchar(1024) NOT NULL -); \ No newline at end of file diff --git a/database/migrations/000003_workspaces.up.sql b/database/migrations/000003_workspaces.up.sql index fe7bfe6ea2822..22dbda07aa0ba 100644 --- a/database/migrations/000003_workspaces.up.sql +++ b/database/migrations/000003_workspaces.up.sql @@ -64,12 +64,3 @@ CREATE TABLE workspace_agent ( -- Identifies resources. resource_metadata jsonb NOT NULL ); - -CREATE TABLE workspace_history_log ( - id uuid NOT NULL UNIQUE, - workspace_history_id uuid NOT NULL REFERENCES workspace_history (id) ON DELETE CASCADE, - created_at timestamptz NOT NULL, - source log_source NOT NULL, - level log_level NOT NULL, - output varchar(1024) NOT NULL -); \ No newline at end of file diff --git a/database/migrations/000004_jobs.up.sql b/database/migrations/000004_jobs.up.sql index 7f767e7dfd983..6588aaa6c9aeb 100644 --- a/database/migrations/000004_jobs.up.sql +++ b/database/migrations/000004_jobs.up.sql @@ -24,11 +24,32 @@ CREATE TABLE IF NOT EXISTS provisioner_job ( initiator_id text NOT NULL, provisioner provisioner_type NOT NULL, type provisioner_job_type NOT NULL, - project_id uuid NOT NULL REFERENCES project(id) ON DELETE CASCADE, input jsonb NOT NULL, worker_id uuid ); +CREATE TYPE log_level AS ENUM ( + 'trace', + 'debug', + 'info', + 'warn', + 'error' +); + +CREATE TYPE log_source AS ENUM ( + 'provisioner_daemon', + 'provisioner' +); + +CREATE TABLE IF NOT EXISTS provisioner_job_log ( + id uuid NOT NULL UNIQUE, + job_id uuid NOT NULL REFERENCES provisioner_job (id) ON DELETE CASCADE, + created_at timestamptz NOT NULL, + source log_source NOT NULL, + level log_level NOT NULL, + output varchar(1024) NOT NULL +); + CREATE TYPE parameter_scope AS ENUM ( 'organization', 'project', diff --git a/database/models.go b/database/models.go index 72e7007e176c5..18d0677940a65 100644 --- a/database/models.go +++ b/database/models.go @@ -266,6 +266,12 @@ type APIKey struct { DevurlToken bool `db:"devurl_token" json:"devurl_token"` } +type File struct { + ID uuid.UUID `db:"id" json:"id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + Data []byte `db:"data" json:"data"` +} + type License struct { ID int32 `db:"id" json:"id"` License json.RawMessage `db:"license" json:"license"` @@ -348,15 +354,6 @@ type ProjectVersion struct { ImportJobID uuid.UUID `db:"import_job_id" json:"import_job_id"` } -type ProjectVersionLog struct { - ID uuid.UUID `db:"id" json:"id"` - ProjectVersionID uuid.UUID `db:"project_version_id" json:"project_version_id"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - Source LogSource `db:"source" json:"source"` - Level LogLevel `db:"level" json:"level"` - Output string `db:"output" json:"output"` -} - type ProvisionerDaemon struct { ID uuid.UUID `db:"id" json:"id"` CreatedAt time.Time `db:"created_at" json:"created_at"` @@ -376,11 +373,19 @@ type ProvisionerJob struct { InitiatorID string `db:"initiator_id" json:"initiator_id"` Provisioner ProvisionerType `db:"provisioner" json:"provisioner"` Type ProvisionerJobType `db:"type" json:"type"` - ProjectID uuid.UUID `db:"project_id" json:"project_id"` Input json.RawMessage `db:"input" json:"input"` WorkerID uuid.NullUUID `db:"worker_id" json:"worker_id"` } +type ProvisionerJobLog struct { + ID uuid.UUID `db:"id" json:"id"` + JobID uuid.UUID `db:"job_id" json:"job_id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + Source LogSource `db:"source" json:"source"` + Level LogLevel `db:"level" json:"level"` + Output string `db:"output" json:"output"` +} + type User struct { ID string `db:"id" json:"id"` Email string `db:"email" json:"email"` @@ -436,15 +441,6 @@ type WorkspaceHistory struct { ProvisionJobID uuid.UUID `db:"provision_job_id" json:"provision_job_id"` } -type WorkspaceHistoryLog struct { - ID uuid.UUID `db:"id" json:"id"` - WorkspaceHistoryID uuid.UUID `db:"workspace_history_id" json:"workspace_history_id"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - Source LogSource `db:"source" json:"source"` - Level LogLevel `db:"level" json:"level"` - Output string `db:"output" json:"output"` -} - type WorkspaceResource struct { ID uuid.UUID `db:"id" json:"id"` CreatedAt time.Time `db:"created_at" json:"created_at"` diff --git a/database/querier.go b/database/querier.go index 20c859cbe3f67..a9a0d9a67dce2 100644 --- a/database/querier.go +++ b/database/querier.go @@ -11,6 +11,7 @@ import ( type querier interface { AcquireProvisionerJob(ctx context.Context, arg AcquireProvisionerJobParams) (ProvisionerJob, error) GetAPIKeyByID(ctx context.Context, id string) (APIKey, error) + GetFileByID(ctx context.Context, id uuid.UUID) (File, error) GetOrganizationByID(ctx context.Context, id string) (Organization, error) GetOrganizationByName(ctx context.Context, name string) (Organization, error) GetOrganizationMemberByUserID(ctx context.Context, arg GetOrganizationMemberByUserIDParams) (OrganizationMember, error) @@ -22,11 +23,11 @@ type querier interface { GetProjectVersionByID(ctx context.Context, id uuid.UUID) (ProjectVersion, error) GetProjectVersionByProjectID(ctx context.Context, projectID uuid.UUID) ([]ProjectVersion, error) GetProjectVersionByProjectIDAndName(ctx context.Context, arg GetProjectVersionByProjectIDAndNameParams) (ProjectVersion, error) - GetProjectVersionLogsByIDBetween(ctx context.Context, arg GetProjectVersionLogsByIDBetweenParams) ([]ProjectVersionLog, error) GetProjectsByOrganizationIDs(ctx context.Context, ids []string) ([]Project, error) GetProvisionerDaemonByID(ctx context.Context, id uuid.UUID) (ProvisionerDaemon, error) GetProvisionerDaemons(ctx context.Context) ([]ProvisionerDaemon, error) GetProvisionerJobByID(ctx context.Context, id uuid.UUID) (ProvisionerJob, error) + GetProvisionerLogsByIDBetween(ctx context.Context, arg GetProvisionerLogsByIDBetweenParams) ([]ProvisionerJobLog, error) GetUserByEmailOrUsername(ctx context.Context, arg GetUserByEmailOrUsernameParams) (User, error) GetUserByID(ctx context.Context, id string) (User, error) GetUserCount(ctx context.Context) (int64, error) @@ -37,25 +38,24 @@ type querier interface { GetWorkspaceHistoryByWorkspaceID(ctx context.Context, workspaceID uuid.UUID) ([]WorkspaceHistory, error) GetWorkspaceHistoryByWorkspaceIDAndName(ctx context.Context, arg GetWorkspaceHistoryByWorkspaceIDAndNameParams) (WorkspaceHistory, error) GetWorkspaceHistoryByWorkspaceIDWithoutAfter(ctx context.Context, workspaceID uuid.UUID) (WorkspaceHistory, error) - GetWorkspaceHistoryLogsByIDBetween(ctx context.Context, arg GetWorkspaceHistoryLogsByIDBetweenParams) ([]WorkspaceHistoryLog, error) GetWorkspaceResourcesByHistoryID(ctx context.Context, workspaceHistoryID uuid.UUID) ([]WorkspaceResource, error) GetWorkspacesByProjectAndUserID(ctx context.Context, arg GetWorkspacesByProjectAndUserIDParams) ([]Workspace, error) GetWorkspacesByUserID(ctx context.Context, ownerID string) ([]Workspace, error) InsertAPIKey(ctx context.Context, arg InsertAPIKeyParams) (APIKey, error) + InsertFile(ctx context.Context, arg InsertFileParams) (File, error) InsertOrganization(ctx context.Context, arg InsertOrganizationParams) (Organization, error) InsertOrganizationMember(ctx context.Context, arg InsertOrganizationMemberParams) (OrganizationMember, error) InsertParameterValue(ctx context.Context, arg InsertParameterValueParams) (ParameterValue, error) InsertProject(ctx context.Context, arg InsertProjectParams) (Project, error) InsertProjectParameter(ctx context.Context, arg InsertProjectParameterParams) (ProjectParameter, error) InsertProjectVersion(ctx context.Context, arg InsertProjectVersionParams) (ProjectVersion, error) - InsertProjectVersionLogs(ctx context.Context, arg InsertProjectVersionLogsParams) ([]ProjectVersionLog, error) InsertProvisionerDaemon(ctx context.Context, arg InsertProvisionerDaemonParams) (ProvisionerDaemon, error) InsertProvisionerJob(ctx context.Context, arg InsertProvisionerJobParams) (ProvisionerJob, error) + InsertProvisionerJobLogs(ctx context.Context, arg InsertProvisionerJobLogsParams) ([]ProvisionerJobLog, error) InsertUser(ctx context.Context, arg InsertUserParams) (User, error) InsertWorkspace(ctx context.Context, arg InsertWorkspaceParams) (Workspace, error) InsertWorkspaceAgent(ctx context.Context, arg InsertWorkspaceAgentParams) (WorkspaceAgent, error) InsertWorkspaceHistory(ctx context.Context, arg InsertWorkspaceHistoryParams) (WorkspaceHistory, error) - InsertWorkspaceHistoryLogs(ctx context.Context, arg InsertWorkspaceHistoryLogsParams) ([]WorkspaceHistoryLog, error) InsertWorkspaceResource(ctx context.Context, arg InsertWorkspaceResourceParams) (WorkspaceResource, error) UpdateAPIKeyByID(ctx context.Context, arg UpdateAPIKeyByIDParams) error UpdateProvisionerDaemonByID(ctx context.Context, arg UpdateProvisionerDaemonByIDParams) error diff --git a/database/query.sql b/database/query.sql index 3a9889c299453..9134bca88404d 100644 --- a/database/query.sql +++ b/database/query.sql @@ -46,6 +46,16 @@ WHERE LIMIT 1; +-- name: GetFileByID :one +SELECT + * +FROM + files +WHERE + id = $1 +LIMIT + 1; + -- name: GetUserByID :one SELECT * @@ -188,13 +198,13 @@ FROM WHERE id = $1; --- name: GetProjectVersionLogsByIDBetween :many +-- name: GetProvisionerLogsByIDBetween :many SELECT * FROM - project_version_log + provisioner_job_log WHERE - project_version_id = @project_version_id + job_id = @job_id AND ( created_at >= @created_after OR created_at <= @created_before @@ -298,20 +308,6 @@ WHERE LIMIT 1; --- name: GetWorkspaceHistoryLogsByIDBetween :many -SELECT - * -FROM - workspace_history_log -WHERE - workspace_history_id = @workspace_history_id - AND ( - created_at >= @created_after - OR created_at <= @created_before - ) -ORDER BY - created_at; - -- name: GetWorkspaceResourcesByHistoryID :many SELECT * @@ -366,6 +362,23 @@ VALUES $15 ) RETURNING *; +-- name: InsertFile :one +INSERT INTO + files (id, created_at, data) +VALUES + ($1, $2, $3) RETURNING *; + +-- name: InsertProvisionerJobLogs :many +INSERT INTO + provisioner_job_log +SELECT + @job_id :: uuid AS job_id, + unnest(@id :: uuid [ ]) AS id, + unnest(@created_at :: timestamptz [ ]) AS created_at, + unnest(@source :: log_source [ ]) as source, + unnest(@level :: log_level [ ]) as level, + unnest(@output :: varchar(1024) [ ]) as output RETURNING *; + -- name: InsertOrganization :one INSERT INTO organizations (id, name, description, created_at, updated_at) @@ -430,17 +443,6 @@ INSERT INTO VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING *; --- name: InsertProjectVersionLogs :many -INSERT INTO - project_version_log -SELECT - @project_version_id :: uuid AS project_version_id, - unnest(@id :: uuid [ ]) AS id, - unnest(@created_at :: timestamptz [ ]) AS created_at, - unnest(@source :: log_source [ ]) as source, - unnest(@level :: log_level [ ]) as level, - unnest(@output :: varchar(1024) [ ]) as output RETURNING *; - -- name: InsertProjectParameter :one INSERT INTO project_parameter ( @@ -498,11 +500,10 @@ INSERT INTO initiator_id, provisioner, type, - project_id, input ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *; + ($1, $2, $3, $4, $5, $6, $7) RETURNING *; -- name: InsertUser :one INSERT INTO @@ -564,17 +565,6 @@ INSERT INTO VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING *; --- name: InsertWorkspaceHistoryLogs :many -INSERT INTO - workspace_history_log -SELECT - unnest(@id :: uuid [ ]) AS id, - @workspace_history_id :: uuid AS workspace_history_id, - unnest(@created_at :: timestamptz [ ]) AS created_at, - unnest(@source :: log_source [ ]) as source, - unnest(@level :: log_level [ ]) as level, - unnest(@output :: varchar(1024) [ ]) as output RETURNING *; - -- name: InsertWorkspaceResource :one INSERT INTO workspace_resource ( diff --git a/database/query.sql.go b/database/query.sql.go index 363c5a3aae896..444e57cc26c8c 100644 --- a/database/query.sql.go +++ b/database/query.sql.go @@ -37,7 +37,7 @@ WHERE SKIP LOCKED LIMIT 1 - ) RETURNING id, created_at, updated_at, started_at, cancelled_at, completed_at, error, initiator_id, provisioner, type, project_id, input, worker_id + ) RETURNING id, created_at, updated_at, started_at, cancelled_at, completed_at, error, initiator_id, provisioner, type, input, worker_id ` type AcquireProvisionerJobParams struct { @@ -66,7 +66,6 @@ func (q *sqlQuerier) AcquireProvisionerJob(ctx context.Context, arg AcquireProvi &i.InitiatorID, &i.Provisioner, &i.Type, - &i.ProjectID, &i.Input, &i.WorkerID, ) @@ -107,6 +106,24 @@ func (q *sqlQuerier) GetAPIKeyByID(ctx context.Context, id string) (APIKey, erro return i, err } +const getFileByID = `-- name: GetFileByID :one +SELECT + id, created_at, data +FROM + files +WHERE + id = $1 +LIMIT + 1 +` + +func (q *sqlQuerier) GetFileByID(ctx context.Context, id uuid.UUID) (File, error) { + row := q.db.QueryRowContext(ctx, getFileByID, id) + var i File + err := row.Scan(&i.ID, &i.CreatedAt, &i.Data) + return i, err +} + const getOrganizationByID = `-- name: GetOrganizationByID :one SELECT id, name, description, created_at, updated_at, "default", auto_off_threshold, cpu_provisioning_rate, memory_provisioning_rate, workspace_auto_off @@ -500,57 +517,6 @@ func (q *sqlQuerier) GetProjectVersionByProjectIDAndName(ctx context.Context, ar return i, err } -const getProjectVersionLogsByIDBetween = `-- name: GetProjectVersionLogsByIDBetween :many -SELECT - id, project_version_id, created_at, source, level, output -FROM - project_version_log -WHERE - project_version_id = $1 - AND ( - created_at >= $2 - OR created_at <= $3 - ) -ORDER BY - created_at -` - -type GetProjectVersionLogsByIDBetweenParams struct { - ProjectVersionID uuid.UUID `db:"project_version_id" json:"project_version_id"` - CreatedAfter time.Time `db:"created_after" json:"created_after"` - CreatedBefore time.Time `db:"created_before" json:"created_before"` -} - -func (q *sqlQuerier) GetProjectVersionLogsByIDBetween(ctx context.Context, arg GetProjectVersionLogsByIDBetweenParams) ([]ProjectVersionLog, error) { - rows, err := q.db.QueryContext(ctx, getProjectVersionLogsByIDBetween, arg.ProjectVersionID, arg.CreatedAfter, arg.CreatedBefore) - if err != nil { - return nil, err - } - defer rows.Close() - var items []ProjectVersionLog - for rows.Next() { - var i ProjectVersionLog - if err := rows.Scan( - &i.ID, - &i.ProjectVersionID, - &i.CreatedAt, - &i.Source, - &i.Level, - &i.Output, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const getProjectsByOrganizationIDs = `-- name: GetProjectsByOrganizationIDs :many SELECT id, created_at, updated_at, organization_id, name, provisioner, active_version_id @@ -651,7 +617,7 @@ func (q *sqlQuerier) GetProvisionerDaemons(ctx context.Context) ([]ProvisionerDa const getProvisionerJobByID = `-- name: GetProvisionerJobByID :one SELECT - id, created_at, updated_at, started_at, cancelled_at, completed_at, error, initiator_id, provisioner, type, project_id, input, worker_id + id, created_at, updated_at, started_at, cancelled_at, completed_at, error, initiator_id, provisioner, type, input, worker_id FROM provisioner_job WHERE @@ -672,13 +638,63 @@ func (q *sqlQuerier) GetProvisionerJobByID(ctx context.Context, id uuid.UUID) (P &i.InitiatorID, &i.Provisioner, &i.Type, - &i.ProjectID, &i.Input, &i.WorkerID, ) return i, err } +const getProvisionerLogsByIDBetween = `-- name: GetProvisionerLogsByIDBetween :many +SELECT + id, job_id, created_at, source, level, output +FROM + provisioner_job_log +WHERE + job_id = $1 + AND ( + created_at >= $2 + OR created_at <= $3 + ) +ORDER BY + created_at +` + +type GetProvisionerLogsByIDBetweenParams struct { + JobID uuid.UUID `db:"job_id" json:"job_id"` + CreatedAfter time.Time `db:"created_after" json:"created_after"` + CreatedBefore time.Time `db:"created_before" json:"created_before"` +} + +func (q *sqlQuerier) GetProvisionerLogsByIDBetween(ctx context.Context, arg GetProvisionerLogsByIDBetweenParams) ([]ProvisionerJobLog, error) { + rows, err := q.db.QueryContext(ctx, getProvisionerLogsByIDBetween, arg.JobID, arg.CreatedAfter, arg.CreatedBefore) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ProvisionerJobLog + for rows.Next() { + var i ProvisionerJobLog + if err := rows.Scan( + &i.ID, + &i.JobID, + &i.CreatedAt, + &i.Source, + &i.Level, + &i.Output, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getUserByEmailOrUsername = `-- name: GetUserByEmailOrUsername :one SELECT id, email, name, revoked, login_type, hashed_password, created_at, updated_at, temporary_password, avatar_hash, ssh_key_regenerated_at, username, dotfiles_git_uri, roles, status, relatime, gpg_key_regenerated_at, _decomissioned, shell @@ -1011,57 +1027,6 @@ func (q *sqlQuerier) GetWorkspaceHistoryByWorkspaceIDWithoutAfter(ctx context.Co return i, err } -const getWorkspaceHistoryLogsByIDBetween = `-- name: GetWorkspaceHistoryLogsByIDBetween :many -SELECT - id, workspace_history_id, created_at, source, level, output -FROM - workspace_history_log -WHERE - workspace_history_id = $1 - AND ( - created_at >= $2 - OR created_at <= $3 - ) -ORDER BY - created_at -` - -type GetWorkspaceHistoryLogsByIDBetweenParams struct { - WorkspaceHistoryID uuid.UUID `db:"workspace_history_id" json:"workspace_history_id"` - CreatedAfter time.Time `db:"created_after" json:"created_after"` - CreatedBefore time.Time `db:"created_before" json:"created_before"` -} - -func (q *sqlQuerier) GetWorkspaceHistoryLogsByIDBetween(ctx context.Context, arg GetWorkspaceHistoryLogsByIDBetweenParams) ([]WorkspaceHistoryLog, error) { - rows, err := q.db.QueryContext(ctx, getWorkspaceHistoryLogsByIDBetween, arg.WorkspaceHistoryID, arg.CreatedAfter, arg.CreatedBefore) - if err != nil { - return nil, err - } - defer rows.Close() - var items []WorkspaceHistoryLog - for rows.Next() { - var i WorkspaceHistoryLog - if err := rows.Scan( - &i.ID, - &i.WorkspaceHistoryID, - &i.CreatedAt, - &i.Source, - &i.Level, - &i.Output, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const getWorkspaceResourcesByHistoryID = `-- name: GetWorkspaceResourcesByHistoryID :many SELECT id, created_at, workspace_history_id, type, name, workspace_agent_token, workspace_agent_id @@ -1282,6 +1247,26 @@ func (q *sqlQuerier) InsertAPIKey(ctx context.Context, arg InsertAPIKeyParams) ( return i, err } +const insertFile = `-- name: InsertFile :one +INSERT INTO + files (id, created_at, data) +VALUES + ($1, $2, $3) RETURNING id, created_at, data +` + +type InsertFileParams struct { + ID uuid.UUID `db:"id" json:"id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + Data []byte `db:"data" json:"data"` +} + +func (q *sqlQuerier) InsertFile(ctx context.Context, arg InsertFileParams) (File, error) { + row := q.db.QueryRowContext(ctx, insertFile, arg.ID, arg.CreatedAt, arg.Data) + var i File + err := row.Scan(&i.ID, &i.CreatedAt, &i.Data) + return i, err +} + const insertOrganization = `-- name: InsertOrganization :one INSERT INTO organizations (id, name, description, created_at, updated_at) @@ -1628,64 +1613,6 @@ func (q *sqlQuerier) InsertProjectVersion(ctx context.Context, arg InsertProject return i, err } -const insertProjectVersionLogs = `-- name: InsertProjectVersionLogs :many -INSERT INTO - project_version_log -SELECT - $1 :: uuid AS project_version_id, - unnest($2 :: uuid [ ]) AS id, - unnest($3 :: timestamptz [ ]) AS created_at, - unnest($4 :: log_source [ ]) as source, - unnest($5 :: log_level [ ]) as level, - unnest($6 :: varchar(1024) [ ]) as output RETURNING id, project_version_id, created_at, source, level, output -` - -type InsertProjectVersionLogsParams struct { - ProjectVersionID uuid.UUID `db:"project_version_id" json:"project_version_id"` - ID []uuid.UUID `db:"id" json:"id"` - CreatedAt []time.Time `db:"created_at" json:"created_at"` - Source []LogSource `db:"source" json:"source"` - Level []LogLevel `db:"level" json:"level"` - Output []string `db:"output" json:"output"` -} - -func (q *sqlQuerier) InsertProjectVersionLogs(ctx context.Context, arg InsertProjectVersionLogsParams) ([]ProjectVersionLog, error) { - rows, err := q.db.QueryContext(ctx, insertProjectVersionLogs, - arg.ProjectVersionID, - pq.Array(arg.ID), - pq.Array(arg.CreatedAt), - pq.Array(arg.Source), - pq.Array(arg.Level), - pq.Array(arg.Output), - ) - if err != nil { - return nil, err - } - defer rows.Close() - var items []ProjectVersionLog - for rows.Next() { - var i ProjectVersionLog - if err := rows.Scan( - &i.ID, - &i.ProjectVersionID, - &i.CreatedAt, - &i.Source, - &i.Level, - &i.Output, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const insertProvisionerDaemon = `-- name: InsertProvisionerDaemon :one INSERT INTO provisioner_daemon (id, created_at, name, provisioners) @@ -1727,11 +1654,10 @@ INSERT INTO initiator_id, provisioner, type, - project_id, input ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id, created_at, updated_at, started_at, cancelled_at, completed_at, error, initiator_id, provisioner, type, project_id, input, worker_id + ($1, $2, $3, $4, $5, $6, $7) RETURNING id, created_at, updated_at, started_at, cancelled_at, completed_at, error, initiator_id, provisioner, type, input, worker_id ` type InsertProvisionerJobParams struct { @@ -1741,7 +1667,6 @@ type InsertProvisionerJobParams struct { InitiatorID string `db:"initiator_id" json:"initiator_id"` Provisioner ProvisionerType `db:"provisioner" json:"provisioner"` Type ProvisionerJobType `db:"type" json:"type"` - ProjectID uuid.UUID `db:"project_id" json:"project_id"` Input json.RawMessage `db:"input" json:"input"` } @@ -1753,7 +1678,6 @@ func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisi arg.InitiatorID, arg.Provisioner, arg.Type, - arg.ProjectID, arg.Input, ) var i ProvisionerJob @@ -1768,13 +1692,70 @@ func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisi &i.InitiatorID, &i.Provisioner, &i.Type, - &i.ProjectID, &i.Input, &i.WorkerID, ) return i, err } +const insertProvisionerJobLogs = `-- name: InsertProvisionerJobLogs :many +INSERT INTO + provisioner_job_log +SELECT + $1 :: uuid AS job_id, + unnest($2 :: uuid [ ]) AS id, + unnest($3 :: timestamptz [ ]) AS created_at, + unnest($4 :: log_source [ ]) as source, + unnest($5 :: log_level [ ]) as level, + unnest($6 :: varchar(1024) [ ]) as output RETURNING id, job_id, created_at, source, level, output +` + +type InsertProvisionerJobLogsParams struct { + JobID uuid.UUID `db:"job_id" json:"job_id"` + ID []uuid.UUID `db:"id" json:"id"` + CreatedAt []time.Time `db:"created_at" json:"created_at"` + Source []LogSource `db:"source" json:"source"` + Level []LogLevel `db:"level" json:"level"` + Output []string `db:"output" json:"output"` +} + +func (q *sqlQuerier) InsertProvisionerJobLogs(ctx context.Context, arg InsertProvisionerJobLogsParams) ([]ProvisionerJobLog, error) { + rows, err := q.db.QueryContext(ctx, insertProvisionerJobLogs, + arg.JobID, + pq.Array(arg.ID), + pq.Array(arg.CreatedAt), + pq.Array(arg.Source), + pq.Array(arg.Level), + pq.Array(arg.Output), + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ProvisionerJobLog + for rows.Next() { + var i ProvisionerJobLog + if err := rows.Scan( + &i.ID, + &i.JobID, + &i.CreatedAt, + &i.Source, + &i.Level, + &i.Output, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const insertUser = `-- name: InsertUser :one INSERT INTO users ( @@ -1992,64 +1973,6 @@ func (q *sqlQuerier) InsertWorkspaceHistory(ctx context.Context, arg InsertWorks return i, err } -const insertWorkspaceHistoryLogs = `-- name: InsertWorkspaceHistoryLogs :many -INSERT INTO - workspace_history_log -SELECT - unnest($1 :: uuid [ ]) AS id, - $2 :: uuid AS workspace_history_id, - unnest($3 :: timestamptz [ ]) AS created_at, - unnest($4 :: log_source [ ]) as source, - unnest($5 :: log_level [ ]) as level, - unnest($6 :: varchar(1024) [ ]) as output RETURNING id, workspace_history_id, created_at, source, level, output -` - -type InsertWorkspaceHistoryLogsParams struct { - ID []uuid.UUID `db:"id" json:"id"` - WorkspaceHistoryID uuid.UUID `db:"workspace_history_id" json:"workspace_history_id"` - CreatedAt []time.Time `db:"created_at" json:"created_at"` - Source []LogSource `db:"source" json:"source"` - Level []LogLevel `db:"level" json:"level"` - Output []string `db:"output" json:"output"` -} - -func (q *sqlQuerier) InsertWorkspaceHistoryLogs(ctx context.Context, arg InsertWorkspaceHistoryLogsParams) ([]WorkspaceHistoryLog, error) { - rows, err := q.db.QueryContext(ctx, insertWorkspaceHistoryLogs, - pq.Array(arg.ID), - arg.WorkspaceHistoryID, - pq.Array(arg.CreatedAt), - pq.Array(arg.Source), - pq.Array(arg.Level), - pq.Array(arg.Output), - ) - if err != nil { - return nil, err - } - defer rows.Close() - var items []WorkspaceHistoryLog - for rows.Next() { - var i WorkspaceHistoryLog - if err := rows.Scan( - &i.ID, - &i.WorkspaceHistoryID, - &i.CreatedAt, - &i.Source, - &i.Level, - &i.Output, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const insertWorkspaceResource = `-- name: InsertWorkspaceResource :one INSERT INTO workspace_resource ( diff --git a/httpmw/provisionerjobparam.go b/httpmw/provisionerjobparam.go new file mode 100644 index 0000000000000..1490dfbcde17c --- /dev/null +++ b/httpmw/provisionerjobparam.go @@ -0,0 +1,64 @@ +package httpmw + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/google/uuid" + + "github.com/coder/coder/database" + "github.com/coder/coder/httpapi" +) + +type provisionerJobParamContextKey struct{} + +// ProvisionerJobParam returns the project from the ExtractProjectParam handler. +func ProvisionerJobParam(r *http.Request) database.ProvisionerJob { + provisionerJob, ok := r.Context().Value(provisionerJobParamContextKey{}).(database.ProvisionerJob) + if !ok { + panic("developer error: provisioner job param middleware not provided") + } + return provisionerJob +} + +// ExtractProvisionerJobParam grabs a provisioner job from the "provisionerjob" URL parameter. +func ExtractProvisionerJobParam(db database.Store) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + jobID := chi.URLParam(r, "provisionerjob") + if jobID == "" { + httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{ + Message: "provisioner job must be provided", + }) + return + } + jobUUID, err := uuid.Parse(jobID) + if err != nil { + httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{ + Message: "job id must be a uuid", + }) + return + } + job, err := db.GetProvisionerJobByID(r.Context(), jobUUID) + if errors.Is(err, sql.ErrNoRows) { + httpapi.Write(rw, http.StatusNotFound, httpapi.Response{ + Message: "job doesn't exist with that id", + }) + return + } + if err != nil { + httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ + Message: fmt.Sprintf("get provisioner job: %s", err), + }) + return + } + + ctx := context.WithValue(r.Context(), provisionerJobParamContextKey{}, job) + next.ServeHTTP(rw, r.WithContext(ctx)) + }) + } +} diff --git a/httpmw/provisionerjobparam_test.go b/httpmw/provisionerjobparam_test.go new file mode 100644 index 0000000000000..45aa268bf63eb --- /dev/null +++ b/httpmw/provisionerjobparam_test.go @@ -0,0 +1,109 @@ +package httpmw_test + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/go-chi/chi/v5" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/database" + "github.com/coder/coder/database/databasefake" + "github.com/coder/coder/httpmw" +) + +func TestProvisionerJobParam(t *testing.T) { + t.Parallel() + + setup := func(db database.Store) (*http.Request, database.ProvisionerJob) { + r := httptest.NewRequest("GET", "/", nil) + provisionerJob, err := db.InsertProvisionerJob(context.Background(), database.InsertProvisionerJobParams{ + ID: uuid.New(), + }) + require.NoError(t, err) + + ctx := chi.NewRouteContext() + r = r.WithContext(context.WithValue(r.Context(), chi.RouteCtxKey, ctx)) + return r, provisionerJob + } + + t.Run("None", func(t *testing.T) { + t.Parallel() + db := databasefake.New() + rtr := chi.NewRouter() + rtr.Use( + httpmw.ExtractProvisionerJobParam(db), + ) + rtr.Get("/", nil) + r, _ := setup(db) + rw := httptest.NewRecorder() + rtr.ServeHTTP(rw, r) + + res := rw.Result() + defer res.Body.Close() + require.Equal(t, http.StatusBadRequest, res.StatusCode) + }) + + t.Run("BadUUID", func(t *testing.T) { + t.Parallel() + db := databasefake.New() + rtr := chi.NewRouter() + rtr.Use( + httpmw.ExtractProvisionerJobParam(db), + ) + rtr.Get("/", nil) + + r, _ := setup(db) + chi.RouteContext(r.Context()).URLParams.Add("provisionerjob", "nothin") + rw := httptest.NewRecorder() + rtr.ServeHTTP(rw, r) + + res := rw.Result() + defer res.Body.Close() + require.Equal(t, http.StatusBadRequest, res.StatusCode) + }) + + t.Run("NotFound", func(t *testing.T) { + t.Parallel() + db := databasefake.New() + rtr := chi.NewRouter() + rtr.Use( + httpmw.ExtractProvisionerJobParam(db), + ) + rtr.Get("/", nil) + + r, _ := setup(db) + chi.RouteContext(r.Context()).URLParams.Add("provisionerjob", uuid.NewString()) + rw := httptest.NewRecorder() + rtr.ServeHTTP(rw, r) + + res := rw.Result() + defer res.Body.Close() + require.Equal(t, http.StatusNotFound, res.StatusCode) + }) + + t.Run("ProvisionerJob", func(t *testing.T) { + t.Parallel() + db := databasefake.New() + rtr := chi.NewRouter() + rtr.Use( + httpmw.ExtractProvisionerJobParam(db), + ) + rtr.Get("/", func(rw http.ResponseWriter, r *http.Request) { + _ = httpmw.ProvisionerJobParam(r) + rw.WriteHeader(http.StatusOK) + }) + + r, job := setup(db) + chi.RouteContext(r.Context()).URLParams.Add("provisionerjob", job.ID.String()) + rw := httptest.NewRecorder() + rtr.ServeHTTP(rw, r) + + res := rw.Result() + defer res.Body.Close() + require.Equal(t, http.StatusOK, res.StatusCode) + }) +} diff --git a/provisionerd/proto/provisionerd.pb.go b/provisionerd/proto/provisionerd.pb.go index c7b59d6f6fcdc..184e6a0e4bedb 100644 --- a/provisionerd/proto/provisionerd.pb.go +++ b/provisionerd/proto/provisionerd.pb.go @@ -116,10 +116,8 @@ type AcquiredJob struct { JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` CreatedAt int64 `protobuf:"varint,2,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` Provisioner string `protobuf:"bytes,3,opt,name=provisioner,proto3" json:"provisioner,omitempty"` - OrganizationName string `protobuf:"bytes,4,opt,name=organization_name,json=organizationName,proto3" json:"organization_name,omitempty"` - ProjectName string `protobuf:"bytes,5,opt,name=project_name,json=projectName,proto3" json:"project_name,omitempty"` - UserName string `protobuf:"bytes,6,opt,name=user_name,json=userName,proto3" json:"user_name,omitempty"` - ProjectSourceArchive []byte `protobuf:"bytes,7,opt,name=project_source_archive,json=projectSourceArchive,proto3" json:"project_source_archive,omitempty"` + UserName string `protobuf:"bytes,4,opt,name=user_name,json=userName,proto3" json:"user_name,omitempty"` + ProjectSourceArchive []byte `protobuf:"bytes,5,opt,name=project_source_archive,json=projectSourceArchive,proto3" json:"project_source_archive,omitempty"` // Types that are assignable to Type: // *AcquiredJob_WorkspaceProvision_ // *AcquiredJob_ProjectImport_ @@ -179,20 +177,6 @@ func (x *AcquiredJob) GetProvisioner() string { return "" } -func (x *AcquiredJob) GetOrganizationName() string { - if x != nil { - return x.OrganizationName - } - return "" -} - -func (x *AcquiredJob) GetProjectName() string { - if x != nil { - return x.ProjectName - } - return "" -} - func (x *AcquiredJob) GetUserName() string { if x != nil { return x.UserName @@ -233,11 +217,11 @@ type isAcquiredJob_Type interface { } type AcquiredJob_WorkspaceProvision_ struct { - WorkspaceProvision *AcquiredJob_WorkspaceProvision `protobuf:"bytes,8,opt,name=workspace_provision,json=workspaceProvision,proto3,oneof"` + WorkspaceProvision *AcquiredJob_WorkspaceProvision `protobuf:"bytes,6,opt,name=workspace_provision,json=workspaceProvision,proto3,oneof"` } type AcquiredJob_ProjectImport_ struct { - ProjectImport *AcquiredJob_ProjectImport `protobuf:"bytes,9,opt,name=project_import,json=projectImport,proto3,oneof"` + ProjectImport *AcquiredJob_ProjectImport `protobuf:"bytes,7,opt,name=project_import,json=projectImport,proto3,oneof"` } func (*AcquiredJob_WorkspaceProvision_) isAcquiredJob_Type() {} @@ -468,9 +452,8 @@ type JobUpdate struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` - WorkspaceProvisionLogs []*Log `protobuf:"bytes,2,rep,name=workspace_provision_logs,json=workspaceProvisionLogs,proto3" json:"workspace_provision_logs,omitempty"` - ProjectImportLogs []*Log `protobuf:"bytes,3,rep,name=project_import_logs,json=projectImportLogs,proto3" json:"project_import_logs,omitempty"` + JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + Logs []*Log `protobuf:"bytes,2,rep,name=logs,proto3" json:"logs,omitempty"` } func (x *JobUpdate) Reset() { @@ -512,16 +495,9 @@ func (x *JobUpdate) GetJobId() string { return "" } -func (x *JobUpdate) GetWorkspaceProvisionLogs() []*Log { +func (x *JobUpdate) GetLogs() []*Log { if x != nil { - return x.WorkspaceProvisionLogs - } - return nil -} - -func (x *JobUpdate) GetProjectImportLogs() []*Log { - if x != nil { - return x.ProjectImportLogs + return x.Logs } return nil } @@ -602,8 +578,7 @@ type AcquiredJob_ProjectImport struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ProjectVersionId string `protobuf:"bytes,1,opt,name=project_version_id,json=projectVersionId,proto3" json:"project_version_id,omitempty"` - ProjectVersionName string `protobuf:"bytes,2,opt,name=project_version_name,json=projectVersionName,proto3" json:"project_version_name,omitempty"` + ProjectName string `protobuf:"bytes,2,opt,name=project_name,json=projectName,proto3" json:"project_name,omitempty"` } func (x *AcquiredJob_ProjectImport) Reset() { @@ -638,16 +613,9 @@ func (*AcquiredJob_ProjectImport) Descriptor() ([]byte, []int) { return file_provisionerd_proto_provisionerd_proto_rawDescGZIP(), []int{1, 1} } -func (x *AcquiredJob_ProjectImport) GetProjectVersionId() string { - if x != nil { - return x.ProjectVersionId - } - return "" -} - -func (x *AcquiredJob_ProjectImport) GetProjectVersionName() string { +func (x *AcquiredJob_ProjectImport) GetProjectName() string { if x != nil { - return x.ProjectVersionName + return x.ProjectName } return "" } @@ -763,31 +731,26 @@ var file_provisionerd_proto_provisionerd_proto_rawDesc = []byte{ 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x1a, 0x26, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x73, 0x64, 0x6b, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x07, 0x0a, - 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x82, 0x06, 0x0a, 0x0b, 0x41, 0x63, 0x71, 0x75, 0x69, + 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0xf5, 0x04, 0x0a, 0x0b, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x12, 0x15, 0x0a, 0x06, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x12, 0x2b, - 0x0a, 0x11, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6f, 0x72, 0x67, 0x61, 0x6e, - 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x70, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1b, - 0x0a, 0x09, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x12, 0x1b, + 0x0a, 0x09, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x16, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x61, 0x72, - 0x63, 0x68, 0x69, 0x76, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x14, 0x70, 0x72, 0x6f, + 0x63, 0x68, 0x69, 0x76, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x14, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x12, 0x5f, 0x0a, 0x13, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x70, - 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, + 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x12, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x50, 0x0a, 0x0e, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, 0x6d, - 0x70, 0x6f, 0x72, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x70, 0x72, 0x6f, + 0x70, 0x6f, 0x72, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x2e, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x6d, @@ -804,89 +767,78 @@ var file_provisionerd_proto_provisionerd_proto_rawDesc = []byte{ 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x1a, 0x6f, 0x0a, 0x0d, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x6d, 0x70, - 0x6f, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x10, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, - 0x64, 0x12, 0x30, 0x0a, 0x14, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x12, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4e, - 0x61, 0x6d, 0x65, 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x3b, 0x0a, 0x0c, 0x43, - 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x12, 0x15, 0x0a, 0x06, 0x6a, - 0x6f, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, - 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x9f, 0x03, 0x0a, 0x0c, 0x43, 0x6f, 0x6d, - 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x12, 0x15, 0x0a, 0x06, 0x6a, 0x6f, 0x62, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, - 0x12, 0x60, 0x0a, 0x13, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x70, 0x72, - 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, + 0x74, 0x65, 0x1a, 0x32, 0x0a, 0x0d, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x6d, 0x70, + 0x6f, 0x72, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x72, 0x6f, 0x6a, 0x65, + 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x3b, + 0x0a, 0x0c, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x12, 0x15, + 0x0a, 0x06, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x9f, 0x03, 0x0a, 0x0c, + 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x12, 0x15, 0x0a, 0x06, + 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, + 0x62, 0x49, 0x64, 0x12, 0x60, 0x0a, 0x13, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x2d, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, + 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x2e, 0x57, 0x6f, 0x72, + 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x48, + 0x00, 0x52, 0x12, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x76, + 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x51, 0x0a, 0x0e, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x5f, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43, 0x6f, 0x6d, - 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x12, - 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, - 0x6f, 0x6e, 0x12, 0x51, 0x0a, 0x0e, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, 0x6d, - 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x70, 0x72, 0x6f, - 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, - 0x74, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x2e, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x6d, - 0x70, 0x6f, 0x72, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, - 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x1a, 0x5f, 0x0a, 0x12, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x33, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, - 0x65, 0x72, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x09, 0x72, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x1a, 0x5a, 0x0a, 0x0d, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, - 0x74, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x49, 0x0a, 0x11, 0x70, 0x61, 0x72, 0x61, 0x6d, - 0x65, 0x74, 0x65, 0x72, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, - 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x52, 0x10, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x53, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x73, 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x9a, 0x01, 0x0a, 0x03, 0x4c, - 0x6f, 0x67, 0x12, 0x2f, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, - 0x64, 0x2e, 0x4c, 0x6f, 0x67, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x06, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x12, 0x2b, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, - 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, - 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, - 0x16, 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x22, 0xb2, 0x01, 0x0a, 0x09, 0x4a, 0x6f, 0x62, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x12, 0x4b, 0x0a, 0x18, - 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, - 0x69, 0x6f, 0x6e, 0x5f, 0x6c, 0x6f, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, - 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x4c, 0x6f, - 0x67, 0x52, 0x16, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x76, - 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x41, 0x0a, 0x13, 0x70, 0x72, 0x6f, - 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x6c, 0x6f, 0x67, 0x73, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, - 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x4c, 0x6f, 0x67, 0x52, 0x11, 0x70, 0x72, 0x6f, 0x6a, 0x65, - 0x63, 0x74, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x2a, 0x34, 0x0a, 0x09, - 0x4c, 0x6f, 0x67, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x16, 0x0a, 0x12, 0x50, 0x52, 0x4f, - 0x56, 0x49, 0x53, 0x49, 0x4f, 0x4e, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x45, 0x4d, 0x4f, 0x4e, 0x10, - 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x52, 0x4f, 0x56, 0x49, 0x53, 0x49, 0x4f, 0x4e, 0x45, 0x52, - 0x10, 0x01, 0x32, 0x8c, 0x02, 0x0a, 0x11, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, - 0x65, 0x72, 0x44, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x12, 0x3c, 0x0a, 0x0a, 0x41, 0x63, 0x71, 0x75, - 0x69, 0x72, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, - 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x19, 0x2e, 0x70, 0x72, - 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, - 0x72, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x12, 0x3b, 0x0a, 0x09, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x4a, 0x6f, 0x62, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, - 0x72, 0x64, 0x2e, 0x4a, 0x6f, 0x62, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x1a, 0x13, 0x2e, 0x70, - 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, - 0x79, 0x28, 0x01, 0x12, 0x3c, 0x0a, 0x09, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x4a, 0x6f, 0x62, - 0x12, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, - 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x1a, 0x13, 0x2e, 0x70, - 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, - 0x79, 0x12, 0x3e, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x4a, 0x6f, 0x62, - 0x12, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, - 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x1a, 0x13, 0x2e, 0x70, - 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, - 0x79, 0x42, 0x2b, 0x5a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x76, - 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x2e, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, + 0x74, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x70, 0x72, 0x6f, 0x6a, 0x65, + 0x63, 0x74, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x1a, 0x5f, 0x0a, 0x12, 0x57, 0x6f, 0x72, 0x6b, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, + 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x33, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, + 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x09, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x1a, 0x5a, 0x0a, 0x0d, 0x50, 0x72, 0x6f, + 0x6a, 0x65, 0x63, 0x74, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x49, 0x0a, 0x11, 0x70, 0x61, + 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, + 0x6e, 0x65, 0x72, 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x53, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x52, 0x10, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x53, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x73, 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x9a, 0x01, + 0x0a, 0x03, 0x4c, 0x6f, 0x67, 0x12, 0x2f, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, + 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x4c, 0x6f, 0x67, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x06, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x2b, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, + 0x6e, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, + 0x76, 0x65, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, + 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, + 0x41, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x22, 0x49, 0x0a, 0x09, 0x4a, 0x6f, + 0x62, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x6a, 0x6f, 0x62, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x12, 0x25, + 0x0a, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x70, + 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x4c, 0x6f, 0x67, 0x52, + 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x2a, 0x34, 0x0a, 0x09, 0x4c, 0x6f, 0x67, 0x53, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x12, 0x16, 0x0a, 0x12, 0x50, 0x52, 0x4f, 0x56, 0x49, 0x53, 0x49, 0x4f, 0x4e, 0x45, + 0x52, 0x5f, 0x44, 0x41, 0x45, 0x4d, 0x4f, 0x4e, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x52, + 0x4f, 0x56, 0x49, 0x53, 0x49, 0x4f, 0x4e, 0x45, 0x52, 0x10, 0x01, 0x32, 0x8c, 0x02, 0x0a, 0x11, + 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x44, 0x61, 0x65, 0x6d, 0x6f, + 0x6e, 0x12, 0x3c, 0x0a, 0x0a, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4a, 0x6f, 0x62, 0x12, + 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, + 0x65, 0x72, 0x64, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x12, + 0x3b, 0x0a, 0x09, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x17, 0x2e, 0x70, + 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x4a, 0x6f, 0x62, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x1a, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, + 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x28, 0x01, 0x12, 0x3c, 0x0a, 0x09, + 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x4a, 0x6f, 0x62, 0x12, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x76, + 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, + 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x1a, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, + 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x3e, 0x0a, 0x0b, 0x43, 0x6f, + 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x76, + 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, + 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x1a, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, + 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x2b, 0x5a, 0x29, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x63, + 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, + 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -927,24 +879,23 @@ var file_provisionerd_proto_provisionerd_proto_depIdxs = []int32{ 10, // 3: provisionerd.CompletedJob.project_import:type_name -> provisionerd.CompletedJob.ProjectImport 0, // 4: provisionerd.Log.source:type_name -> provisionerd.LogSource 11, // 5: provisionerd.Log.level:type_name -> provisioner.LogLevel - 5, // 6: provisionerd.JobUpdate.workspace_provision_logs:type_name -> provisionerd.Log - 5, // 7: provisionerd.JobUpdate.project_import_logs:type_name -> provisionerd.Log - 12, // 8: provisionerd.AcquiredJob.WorkspaceProvision.parameter_values:type_name -> provisioner.ParameterValue - 13, // 9: provisionerd.CompletedJob.WorkspaceProvision.resources:type_name -> provisioner.Resource - 14, // 10: provisionerd.CompletedJob.ProjectImport.parameter_schemas:type_name -> provisioner.ParameterSchema - 1, // 11: provisionerd.ProvisionerDaemon.AcquireJob:input_type -> provisionerd.Empty - 6, // 12: provisionerd.ProvisionerDaemon.UpdateJob:input_type -> provisionerd.JobUpdate - 3, // 13: provisionerd.ProvisionerDaemon.CancelJob:input_type -> provisionerd.CancelledJob - 4, // 14: provisionerd.ProvisionerDaemon.CompleteJob:input_type -> provisionerd.CompletedJob - 2, // 15: provisionerd.ProvisionerDaemon.AcquireJob:output_type -> provisionerd.AcquiredJob - 1, // 16: provisionerd.ProvisionerDaemon.UpdateJob:output_type -> provisionerd.Empty - 1, // 17: provisionerd.ProvisionerDaemon.CancelJob:output_type -> provisionerd.Empty - 1, // 18: provisionerd.ProvisionerDaemon.CompleteJob:output_type -> provisionerd.Empty - 15, // [15:19] is the sub-list for method output_type - 11, // [11:15] is the sub-list for method input_type - 11, // [11:11] is the sub-list for extension type_name - 11, // [11:11] is the sub-list for extension extendee - 0, // [0:11] is the sub-list for field type_name + 5, // 6: provisionerd.JobUpdate.logs:type_name -> provisionerd.Log + 12, // 7: provisionerd.AcquiredJob.WorkspaceProvision.parameter_values:type_name -> provisioner.ParameterValue + 13, // 8: provisionerd.CompletedJob.WorkspaceProvision.resources:type_name -> provisioner.Resource + 14, // 9: provisionerd.CompletedJob.ProjectImport.parameter_schemas:type_name -> provisioner.ParameterSchema + 1, // 10: provisionerd.ProvisionerDaemon.AcquireJob:input_type -> provisionerd.Empty + 6, // 11: provisionerd.ProvisionerDaemon.UpdateJob:input_type -> provisionerd.JobUpdate + 3, // 12: provisionerd.ProvisionerDaemon.CancelJob:input_type -> provisionerd.CancelledJob + 4, // 13: provisionerd.ProvisionerDaemon.CompleteJob:input_type -> provisionerd.CompletedJob + 2, // 14: provisionerd.ProvisionerDaemon.AcquireJob:output_type -> provisionerd.AcquiredJob + 1, // 15: provisionerd.ProvisionerDaemon.UpdateJob:output_type -> provisionerd.Empty + 1, // 16: provisionerd.ProvisionerDaemon.CancelJob:output_type -> provisionerd.Empty + 1, // 17: provisionerd.ProvisionerDaemon.CompleteJob:output_type -> provisionerd.Empty + 14, // [14:18] is the sub-list for method output_type + 10, // [10:14] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name } func init() { file_provisionerd_proto_provisionerd_proto_init() } diff --git a/provisionerd/proto/provisionerd.proto b/provisionerd/proto/provisionerd.proto index accb69e99f542..f7c6bdc545775 100644 --- a/provisionerd/proto/provisionerd.proto +++ b/provisionerd/proto/provisionerd.proto @@ -18,19 +18,16 @@ message AcquiredJob { bytes state = 4; } message ProjectImport { - string project_version_id = 1; - string project_version_name = 2; + string project_name = 2; } string job_id = 1; int64 created_at = 2; string provisioner = 3; - string organization_name = 4; - string project_name = 5; - string user_name = 6; - bytes project_source_archive = 7; + string user_name = 4; + bytes project_source_archive = 5; oneof type { - WorkspaceProvision workspace_provision = 8; - ProjectImport project_import = 9; + WorkspaceProvision workspace_provision = 6; + ProjectImport project_import = 7; } } @@ -74,8 +71,7 @@ message Log { // should still be sent periodically as a heartbeat. message JobUpdate { string job_id = 1; - repeated Log workspace_provision_logs = 2; - repeated Log project_import_logs = 3; + repeated Log logs = 2; } service ProvisionerDaemon { diff --git a/provisionerd/provisionerd.go b/provisionerd/provisionerd.go index 78f8c40f4c374..15cce82b1bfcb 100644 --- a/provisionerd/provisionerd.go +++ b/provisionerd/provisionerd.go @@ -200,9 +200,7 @@ func (p *provisionerDaemon) acquireJob(ctx context.Context) { p.jobID = job.JobId p.opts.Logger.Info(context.Background(), "acquired job", - slog.F("organization_name", job.OrganizationName), - slog.F("project_name", job.ProjectName), - slog.F("username", job.UserName), + slog.F("initiator_username", job.UserName), slog.F("provisioner", job.Provisioner), slog.F("id", job.JobId), ) @@ -322,7 +320,7 @@ func (p *provisionerDaemon) runJob(ctx context.Context, job *proto.AcquiredJob) switch jobType := job.Type.(type) { case *proto.AcquiredJob_ProjectImport_: p.opts.Logger.Debug(context.Background(), "acquired job is project import", - slog.F("project_version_name", jobType.ProjectImport.ProjectVersionName), + slog.F("project_name", jobType.ProjectImport.ProjectName), ) p.runProjectImport(ctx, provisioner, job) @@ -366,12 +364,11 @@ func (p *provisionerDaemon) runProjectImport(ctx context.Context, provisioner sd p.opts.Logger.Debug(context.Background(), "parse job logged", slog.F("level", msgType.Log.Level), slog.F("output", msgType.Log.Output), - slog.F("project_version_id", job.GetProjectImport().ProjectVersionId), ) err = p.updateStream.Send(&proto.JobUpdate{ JobId: job.JobId, - ProjectImportLogs: []*proto.Log{{ + Logs: []*proto.Log{{ Source: proto.LogSource_PROVISIONER, Level: msgType.Log.Level, CreatedAt: time.Now().UTC().UnixMilli(), @@ -436,7 +433,7 @@ func (p *provisionerDaemon) runWorkspaceProvision(ctx context.Context, provision err = p.updateStream.Send(&proto.JobUpdate{ JobId: job.JobId, - WorkspaceProvisionLogs: []*proto.Log{{ + Logs: []*proto.Log{{ Source: proto.LogSource_PROVISIONER, Level: msgType.Log.Level, CreatedAt: time.Now().UTC().UnixMilli(), diff --git a/provisionerd/provisionerd_test.go b/provisionerd/provisionerd_test.go index 5a32b6fb2030e..46d064ad7f958 100644 --- a/provisionerd/provisionerd_test.go +++ b/provisionerd/provisionerd_test.go @@ -228,7 +228,7 @@ func TestProvisionerd(t *testing.T) { if err != nil { return err } - if len(msg.ProjectImportLogs) == 0 { + if len(msg.Logs) == 0 { continue } @@ -308,7 +308,7 @@ func TestProvisionerd(t *testing.T) { if err != nil { return err } - if len(msg.WorkspaceProvisionLogs) == 0 { + if len(msg.Logs) == 0 { continue } From 6e6beb471c9f8517f1088993199aa42a4024079c Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 7 Feb 2022 20:34:55 +0000 Subject: [PATCH 2/4] Improve file table schema by using hash --- database/databasefake/databasefake.go | 7 ++-- database/dump.sql | 9 ++--- database/migrations/000002_projects.up.sql | 5 +-- database/models.go | 3 +- database/querier.go | 2 +- database/query.sql | 10 +++--- database/query.sql.go | 40 +++++++++++++++------- 7 files changed, 48 insertions(+), 28 deletions(-) diff --git a/database/databasefake/databasefake.go b/database/databasefake/databasefake.go index 3b3d09cd25ab5..a1ee7edf3a9b1 100644 --- a/database/databasefake/databasefake.go +++ b/database/databasefake/databasefake.go @@ -104,12 +104,12 @@ func (q *fakeQuerier) GetAPIKeyByID(_ context.Context, id string) (database.APIK return database.APIKey{}, sql.ErrNoRows } -func (q *fakeQuerier) GetFileByID(_ context.Context, id uuid.UUID) (database.File, error) { +func (q *fakeQuerier) GetFileByHash(_ context.Context, hash string) (database.File, error) { q.mutex.Lock() defer q.mutex.Unlock() for _, file := range q.files { - if file.ID.String() == id.String() { + if file.Hash == hash { return file, nil } } @@ -588,8 +588,9 @@ func (q *fakeQuerier) InsertFile(_ context.Context, arg database.InsertFileParam //nolint:gosimple file := database.File{ - ID: arg.ID, + Hash: arg.Hash, CreatedAt: arg.CreatedAt, + Mimetype: arg.Mimetype, Data: arg.Data, } q.files = append(q.files, file) diff --git a/database/dump.sql b/database/dump.sql index 8034b3d0ad517..721f1c034c803 100644 --- a/database/dump.sql +++ b/database/dump.sql @@ -87,9 +87,10 @@ CREATE TABLE api_keys ( devurl_token boolean DEFAULT false NOT NULL ); -CREATE TABLE files ( - id uuid NOT NULL, +CREATE TABLE file ( + hash character varying(32) NOT NULL, created_at timestamp with time zone NOT NULL, + mimetype character varying(64) NOT NULL, data bytea NOT NULL ); @@ -272,8 +273,8 @@ CREATE TABLE workspace_resource ( workspace_agent_id uuid ); -ALTER TABLE ONLY files - ADD CONSTRAINT files_id_key UNIQUE (id); +ALTER TABLE ONLY file + ADD CONSTRAINT file_hash_key UNIQUE (hash); ALTER TABLE ONLY parameter_value ADD CONSTRAINT parameter_value_id_key UNIQUE (id); diff --git a/database/migrations/000002_projects.up.sql b/database/migrations/000002_projects.up.sql index 21194025fc0b2..f8c3a85ca3d4b 100644 --- a/database/migrations/000002_projects.up.sql +++ b/database/migrations/000002_projects.up.sql @@ -1,7 +1,8 @@ -- Store arbitrary data like project source code or avatars. -CREATE TABLE files ( - id uuid NOT NULL UNIQUE, +CREATE TABLE file ( + hash varchar(32) NOT NULL UNIQUE, created_at timestamptz NOT NULL, + mimetype varchar(64) NOT NULL, data bytea NOT NULL ); diff --git a/database/models.go b/database/models.go index 18d0677940a65..6c0534c3e1809 100644 --- a/database/models.go +++ b/database/models.go @@ -267,8 +267,9 @@ type APIKey struct { } type File struct { - ID uuid.UUID `db:"id" json:"id"` + Hash string `db:"hash" json:"hash"` CreatedAt time.Time `db:"created_at" json:"created_at"` + Mimetype string `db:"mimetype" json:"mimetype"` Data []byte `db:"data" json:"data"` } diff --git a/database/querier.go b/database/querier.go index a9a0d9a67dce2..43ba8f66dbd7e 100644 --- a/database/querier.go +++ b/database/querier.go @@ -11,7 +11,7 @@ import ( type querier interface { AcquireProvisionerJob(ctx context.Context, arg AcquireProvisionerJobParams) (ProvisionerJob, error) GetAPIKeyByID(ctx context.Context, id string) (APIKey, error) - GetFileByID(ctx context.Context, id uuid.UUID) (File, error) + GetFileByHash(ctx context.Context, hash string) (File, error) GetOrganizationByID(ctx context.Context, id string) (Organization, error) GetOrganizationByName(ctx context.Context, name string) (Organization, error) GetOrganizationMemberByUserID(ctx context.Context, arg GetOrganizationMemberByUserIDParams) (OrganizationMember, error) diff --git a/database/query.sql b/database/query.sql index 9134bca88404d..cba1c5ed91119 100644 --- a/database/query.sql +++ b/database/query.sql @@ -46,13 +46,13 @@ WHERE LIMIT 1; --- name: GetFileByID :one +-- name: GetFileByHash :one SELECT * FROM - files + file WHERE - id = $1 + hash = $1 LIMIT 1; @@ -364,9 +364,9 @@ VALUES -- name: InsertFile :one INSERT INTO - files (id, created_at, data) + file (hash, created_at, mimetype, data) VALUES - ($1, $2, $3) RETURNING *; + ($1, $2, $3, $4) RETURNING *; -- name: InsertProvisionerJobLogs :many INSERT INTO diff --git a/database/query.sql.go b/database/query.sql.go index 444e57cc26c8c..2b54235a5b1b3 100644 --- a/database/query.sql.go +++ b/database/query.sql.go @@ -106,21 +106,26 @@ func (q *sqlQuerier) GetAPIKeyByID(ctx context.Context, id string) (APIKey, erro return i, err } -const getFileByID = `-- name: GetFileByID :one +const getFileByHash = `-- name: GetFileByHash :one SELECT - id, created_at, data + hash, created_at, mimetype, data FROM - files + file WHERE - id = $1 + hash = $1 LIMIT 1 ` -func (q *sqlQuerier) GetFileByID(ctx context.Context, id uuid.UUID) (File, error) { - row := q.db.QueryRowContext(ctx, getFileByID, id) +func (q *sqlQuerier) GetFileByHash(ctx context.Context, hash string) (File, error) { + row := q.db.QueryRowContext(ctx, getFileByHash, hash) var i File - err := row.Scan(&i.ID, &i.CreatedAt, &i.Data) + err := row.Scan( + &i.Hash, + &i.CreatedAt, + &i.Mimetype, + &i.Data, + ) return i, err } @@ -1249,21 +1254,32 @@ func (q *sqlQuerier) InsertAPIKey(ctx context.Context, arg InsertAPIKeyParams) ( const insertFile = `-- name: InsertFile :one INSERT INTO - files (id, created_at, data) + file (hash, created_at, mimetype, data) VALUES - ($1, $2, $3) RETURNING id, created_at, data + ($1, $2, $3, $4) RETURNING hash, created_at, mimetype, data ` type InsertFileParams struct { - ID uuid.UUID `db:"id" json:"id"` + Hash string `db:"hash" json:"hash"` CreatedAt time.Time `db:"created_at" json:"created_at"` + Mimetype string `db:"mimetype" json:"mimetype"` Data []byte `db:"data" json:"data"` } func (q *sqlQuerier) InsertFile(ctx context.Context, arg InsertFileParams) (File, error) { - row := q.db.QueryRowContext(ctx, insertFile, arg.ID, arg.CreatedAt, arg.Data) + row := q.db.QueryRowContext(ctx, insertFile, + arg.Hash, + arg.CreatedAt, + arg.Mimetype, + arg.Data, + ) var i File - err := row.Scan(&i.ID, &i.CreatedAt, &i.Data) + err := row.Scan( + &i.Hash, + &i.CreatedAt, + &i.Mimetype, + &i.Data, + ) return i, err } From b168e3d742eca660bc2acfbae46f1c100261588c Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 7 Feb 2022 20:36:12 +0000 Subject: [PATCH 3/4] Fix racey test by allowing logs before --- coderd/provisionerjoblogs_test.go | 15 ++++----------- database/query.sql | 2 +- database/query.sql.go | 8 ++++---- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/coderd/provisionerjoblogs_test.go b/coderd/provisionerjoblogs_test.go index 454b3bb85caec..cf67d93002c10 100644 --- a/coderd/provisionerjoblogs_test.go +++ b/coderd/provisionerjoblogs_test.go @@ -14,7 +14,7 @@ import ( "github.com/coder/coder/provisionersdk/proto" ) -func TestWorkspaceHistoryLogsByName(t *testing.T) { +func TestProvisionerJobLogsByName(t *testing.T) { t.Parallel() t.Run("List", func(t *testing.T) { t.Parallel() @@ -44,17 +44,10 @@ func TestWorkspaceHistoryLogsByName(t *testing.T) { Transition: database.WorkspaceTransitionCreate, }) require.NoError(t, err) - - // Successfully return empty logs before the job starts! - logs, err := client.ProvisionerJobLogs(context.Background(), history.Provision.ID) - require.NoError(t, err) - require.NotNil(t, logs) - require.Len(t, logs, 0) - coderdtest.AwaitWorkspaceHistoryProvisioned(t, client, "", workspace.Name, history.Name) // Return the log after completion! - logs, err = client.ProvisionerJobLogs(context.Background(), history.Provision.ID) + logs, err := client.ProvisionerJobLogs(context.Background(), history.Provision.ID) require.NoError(t, err) require.NotNil(t, logs) require.Len(t, logs, 1) @@ -124,13 +117,13 @@ func TestWorkspaceHistoryLogsByName(t *testing.T) { }) coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name) workspace := coderdtest.CreateWorkspace(t, client, "me", project.ID) + before := database.Now() history, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{ ProjectVersionID: version.ID, Transition: database.WorkspaceTransitionCreate, }) require.NoError(t, err) - - logs, err := client.FollowProvisionerJobLogsAfter(context.Background(), history.Provision.ID, time.Time{}) + logs, err := client.FollowProvisionerJobLogsAfter(context.Background(), history.Provision.ID, before) require.NoError(t, err) log := <-logs require.Equal(t, "log-output", log.Output) diff --git a/database/query.sql b/database/query.sql index cba1c5ed91119..ffd619b17483a 100644 --- a/database/query.sql +++ b/database/query.sql @@ -372,8 +372,8 @@ VALUES INSERT INTO provisioner_job_log SELECT - @job_id :: uuid AS job_id, unnest(@id :: uuid [ ]) AS id, + @job_id :: uuid AS job_id, unnest(@created_at :: timestamptz [ ]) AS created_at, unnest(@source :: log_source [ ]) as source, unnest(@level :: log_level [ ]) as level, diff --git a/database/query.sql.go b/database/query.sql.go index 2b54235a5b1b3..547330860c32f 100644 --- a/database/query.sql.go +++ b/database/query.sql.go @@ -1718,8 +1718,8 @@ const insertProvisionerJobLogs = `-- name: InsertProvisionerJobLogs :many INSERT INTO provisioner_job_log SELECT - $1 :: uuid AS job_id, - unnest($2 :: uuid [ ]) AS id, + unnest($1 :: uuid [ ]) AS id, + $2 :: uuid AS job_id, unnest($3 :: timestamptz [ ]) AS created_at, unnest($4 :: log_source [ ]) as source, unnest($5 :: log_level [ ]) as level, @@ -1727,8 +1727,8 @@ SELECT ` type InsertProvisionerJobLogsParams struct { - JobID uuid.UUID `db:"job_id" json:"job_id"` ID []uuid.UUID `db:"id" json:"id"` + JobID uuid.UUID `db:"job_id" json:"job_id"` CreatedAt []time.Time `db:"created_at" json:"created_at"` Source []LogSource `db:"source" json:"source"` Level []LogLevel `db:"level" json:"level"` @@ -1737,8 +1737,8 @@ type InsertProvisionerJobLogsParams struct { func (q *sqlQuerier) InsertProvisionerJobLogs(ctx context.Context, arg InsertProvisionerJobLogsParams) ([]ProvisionerJobLog, error) { rows, err := q.db.QueryContext(ctx, insertProvisionerJobLogs, - arg.JobID, pq.Array(arg.ID), + arg.JobID, pq.Array(arg.CreatedAt), pq.Array(arg.Source), pq.Array(arg.Level), From a70695680cc0ae0479c3dcbfc88e16ab1fc13299 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 7 Feb 2022 21:25:29 +0000 Subject: [PATCH 4/4] Add debug logging for PostgreSQL insert --- coderd/provisionerdaemons.go | 1 + 1 file changed, 1 insertion(+) diff --git a/coderd/provisionerdaemons.go b/coderd/provisionerdaemons.go index d3c1e7590b80d..fa29121e865a5 100644 --- a/coderd/provisionerdaemons.go +++ b/coderd/provisionerdaemons.go @@ -307,6 +307,7 @@ func (server *provisionerdServer) UpdateJob(stream proto.DRPCProvisionerDaemon_U } logs, err := server.Database.InsertProvisionerJobLogs(stream.Context(), insertParams) if err != nil { + server.Logger.Error(stream.Context(), "insert provisioner job logs", slog.Error(err)) return xerrors.Errorf("insert job logs: %w", err) } data, err := json.Marshal(logs)