Skip to content

feat: Add buffering to provisioner job logs #4918

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 7, 2022
Merged

feat: Add buffering to provisioner job logs #4918

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d7d4a33
feat: Add bufferring to provisioner job logs
kylecarbs Nov 6, 2022
e2830c2
Fix log bufferring
kylecarbs Nov 6, 2022
4dee930
Fix frontend log streaming
kylecarbs Nov 6, 2022
060c244
Merge branch 'main' into bufferlogs
kylecarbs Nov 6, 2022
d4513eb
Fix JS test
kylecarbs Nov 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cli/cliui/provisionerjob.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@ import (
"github.com/coder/coder/codersdk"
)

func WorkspaceBuild(ctx context.Context, writer io.Writer, client *codersdk.Client, build uuid.UUID, before time.Time) error {
func WorkspaceBuild(ctx context.Context, writer io.Writer, client *codersdk.Client, build uuid.UUID) error {
return ProvisionerJob(ctx, writer, ProvisionerJobOptions{
Fetch: func() (codersdk.ProvisionerJob, error) {
build, err := client.WorkspaceBuild(ctx, build)
return build.Job, err
},
Logs: func() (<-chan codersdk.ProvisionerJobLog, io.Closer, error) {
return client.WorkspaceBuildLogsAfter(ctx, build, before)
return client.WorkspaceBuildLogsAfter(ctx, build, 0)
},
})
}
Expand Down
6 changes: 2 additions & 4 deletions cli/create.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,6 @@ func create() *cobra.Command {
return err
}

after := time.Now()
workspace, err := client.CreateWorkspace(cmd.Context(), organization.ID, codersdk.Me, codersdk.CreateWorkspaceRequest{
TemplateID: template.ID,
Name: workspaceName,
Expand All @@ -151,7 +150,7 @@ func create() *cobra.Command {
return err
}

err = cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStdout(), client, workspace.LatestBuild.ID, after)
err = cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStdout(), client, workspace.LatestBuild.ID)
if err != nil {
return err
}
Expand Down Expand Up @@ -238,7 +237,6 @@ PromptParamLoop:
_, _ = fmt.Fprintln(cmd.OutOrStdout())

// Run a dry-run with the given parameters to check correctness
after := time.Now()
dryRun, err := client.CreateTemplateVersionDryRun(cmd.Context(), templateVersion.ID, codersdk.CreateTemplateVersionDryRunRequest{
WorkspaceName: args.NewWorkspaceName,
ParameterValues: parameters,
Expand All @@ -255,7 +253,7 @@ PromptParamLoop:
return client.CancelTemplateVersionDryRun(cmd.Context(), templateVersion.ID, dryRun.ID)
},
Logs: func() (<-chan codersdk.ProvisionerJobLog, io.Closer, error) {
return client.TemplateVersionDryRunLogsAfter(cmd.Context(), templateVersion.ID, dryRun.ID, after)
return client.TemplateVersionDryRunLogsAfter(cmd.Context(), templateVersion.ID, dryRun.ID, 0)
},
// Don't show log output for the dry-run unless there's an error.
Silent: true,
Expand Down
3 changes: 1 addition & 2 deletions cli/delete.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ func deleteWorkspace() *cobra.Command {
)
}

before := time.Now()
build, err := client.CreateWorkspaceBuild(cmd.Context(), workspace.ID, codersdk.CreateWorkspaceBuildRequest{
Transition: codersdk.WorkspaceTransitionDelete,
ProvisionerState: state,
Expand All @@ -57,7 +56,7 @@ func deleteWorkspace() *cobra.Command {
return err
}

err = cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStdout(), client, build.ID, before)
err = cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStdout(), client, build.ID)
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion cli/portforward.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func portForward() *cobra.Command {
return xerrors.New("workspace must be in start transition to port-forward")
}
if workspace.LatestBuild.Job.CompletedAt == nil {
err = cliui.WorkspaceBuild(ctx, cmd.ErrOrStderr(), client, workspace.LatestBuild.ID, workspace.CreatedAt)
err = cliui.WorkspaceBuild(ctx, cmd.ErrOrStderr(), client, workspace.LatestBuild.ID)
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion cli/ssh.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@ func getWorkspaceAndAgent(ctx context.Context, cmd *cobra.Command, client *coder
return codersdk.Workspace{}, codersdk.WorkspaceAgent{}, xerrors.New("workspace must be in start transition to ssh")
}
if workspace.LatestBuild.Job.CompletedAt == nil {
err := cliui.WorkspaceBuild(ctx, cmd.ErrOrStderr(), client, workspace.LatestBuild.ID, workspace.CreatedAt)
err := cliui.WorkspaceBuild(ctx, cmd.ErrOrStderr(), client, workspace.LatestBuild.ID)
if err != nil {
return codersdk.Workspace{}, codersdk.WorkspaceAgent{}, err
}
Expand Down
3 changes: 1 addition & 2 deletions cli/start.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,14 @@ func start() *cobra.Command {
if err != nil {
return err
}
before := time.Now()
build, err := client.CreateWorkspaceBuild(cmd.Context(), workspace.ID, codersdk.CreateWorkspaceBuildRequest{
Transition: codersdk.WorkspaceTransitionStart,
})
if err != nil {
return err
}

err = cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStdout(), client, build.ID, before)
err = cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStdout(), client, build.ID)
if err != nil {
return err
}
Expand Down
4 changes: 1 addition & 3 deletions cli/state.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"io"
"os"
"strconv"
"time"

"github.com/spf13/cobra"

Expand Down Expand Up @@ -100,7 +99,6 @@ func statePush() *cobra.Command {
return err
}

before := time.Now()
build, err = client.CreateWorkspaceBuild(cmd.Context(), workspace.ID, codersdk.CreateWorkspaceBuildRequest{
TemplateVersionID: build.TemplateVersionID,
Transition: build.Transition,
Expand All @@ -109,7 +107,7 @@ func statePush() *cobra.Command {
if err != nil {
return err
}
return cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStderr(), client, build.ID, before)
return cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStderr(), client, build.ID)
},
}
cmd.Flags().IntVarP(&buildNumber, "build", "b", 0, "Specify a workspace build to target by name.")
Expand Down
3 changes: 1 addition & 2 deletions cli/stop.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,14 @@ func stop() *cobra.Command {
if err != nil {
return err
}
before := time.Now()
build, err := client.CreateWorkspaceBuild(cmd.Context(), workspace.ID, codersdk.CreateWorkspaceBuildRequest{
Transition: codersdk.WorkspaceTransitionStop,
})
if err != nil {
return err
}

err = cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStdout(), client, build.ID, before)
err = cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStdout(), client, build.ID)
if err != nil {
return err
}
Expand Down
3 changes: 1 addition & 2 deletions cli/templatecreate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,6 @@ type createValidTemplateVersionArgs struct {
}

func createValidTemplateVersion(cmd *cobra.Command, args createValidTemplateVersionArgs, parameters ...codersdk.CreateParameterRequest) (*codersdk.TemplateVersion, []codersdk.CreateParameterRequest, error) {
before := time.Now()
client := args.Client

req := codersdk.CreateTemplateVersionRequest{
Expand All @@ -187,7 +186,7 @@ func createValidTemplateVersion(cmd *cobra.Command, args createValidTemplateVers
return client.CancelTemplateVersion(cmd.Context(), version.ID)
},
Logs: func() (<-chan codersdk.ProvisionerJobLog, io.Closer, error) {
return client.TemplateVersionLogsAfter(cmd.Context(), version.ID, before)
return client.TemplateVersionLogsAfter(cmd.Context(), version.ID, 0)
},
})
if err != nil {
Expand Down
4 changes: 1 addition & 3 deletions cli/update.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package cli

import (
"fmt"
"time"

"github.com/spf13/cobra"

Expand Down Expand Up @@ -57,7 +56,6 @@ func update() *cobra.Command {
return nil
}

before := time.Now()
build, err := client.CreateWorkspaceBuild(cmd.Context(), workspace.ID, codersdk.CreateWorkspaceBuildRequest{
TemplateVersionID: template.ActiveVersionID,
Transition: workspace.LatestBuild.Transition,
Expand All @@ -66,7 +64,7 @@ func update() *cobra.Command {
if err != nil {
return err
}
logs, closer, err := client.WorkspaceBuildLogsAfter(cmd.Context(), build.ID, before)
logs, closer, err := client.WorkspaceBuildLogsAfter(cmd.Context(), build.ID, 0)
if err != nil {
return err
}
Expand Down
14 changes: 8 additions & 6 deletions coderd/database/databasefake/databasefake.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2052,17 +2052,14 @@ func (q *fakeQuerier) GetProvisionerLogsByIDBetween(_ context.Context, arg datab
if jobLog.JobID != arg.JobID {
continue
}
if !arg.CreatedBefore.IsZero() && jobLog.CreatedAt.After(arg.CreatedBefore) {
if arg.CreatedBefore != 0 && jobLog.ID > arg.CreatedBefore {
continue
}
if !arg.CreatedAfter.IsZero() && jobLog.CreatedAt.Before(arg.CreatedAfter) {
if arg.CreatedAfter != 0 && jobLog.ID < arg.CreatedAfter {
continue
}
logs = append(logs, jobLog)
}
if len(logs) == 0 {
return nil, sql.ErrNoRows
}
return logs, nil
}

Expand Down Expand Up @@ -2212,10 +2209,15 @@ func (q *fakeQuerier) InsertProvisionerJobLogs(_ context.Context, arg database.I
defer q.mutex.Unlock()

logs := make([]database.ProvisionerJobLog, 0)
id := int64(1)
if len(q.provisionerJobLogs) > 0 {
id = q.provisionerJobLogs[len(q.provisionerJobLogs)-1].ID
}
for index, output := range arg.Output {
id++
logs = append(logs, database.ProvisionerJobLog{
ID: id,
JobID: arg.JobID,
ID: arg.ID[index],
CreatedAt: arg.CreatedAt[index],
Source: arg.Source[index],
Level: arg.Level[index],
Expand Down
15 changes: 13 additions & 2 deletions coderd/database/dump.sql
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions coderd/database/migrations/000071_provisioner_log_lines.down.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
ALTER TABLE provisioner_job_logs DROP COLUMN id;

ALTER TABLE provisioner_job_logs ADD COLUMN id uuid NOT NULL DEFAULT gen_random_uuid();
3 changes: 3 additions & 0 deletions coderd/database/migrations/000071_provisioner_log_lines.up.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
ALTER TABLE provisioner_job_logs DROP COLUMN id;

ALTER TABLE provisioner_job_logs ADD COLUMN id BIGSERIAL PRIMARY KEY;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we guarantee that adding this bigserial column will keep the same order of logs?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It likely won't in all cases, but it seems to in most.

Having a safe migration seems unreasonable here, so we settle for pretty great.

2 changes: 1 addition & 1 deletion coderd/database/models.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

33 changes: 14 additions & 19 deletions coderd/database/queries.sql.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 3 additions & 6 deletions coderd/database/queries/provisionerjoblogs.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,14 @@ FROM
WHERE
job_id = @job_id
AND (
created_at >= @created_after
OR created_at <= @created_before
)
ORDER BY
created_at DESC;
id > @created_after
OR id < @created_before
) ORDER BY id;

-- name: InsertProvisionerJobLogs :many
INSERT INTO
provisioner_job_logs
SELECT
unnest(@id :: uuid [ ]) AS id,
@job_id :: uuid AS job_id,
unnest(@created_at :: timestamptz [ ]) AS created_at,
unnest(@source :: log_source [ ]) AS source,
Expand Down
Loading