From 663dee0e3d1cdf01ce4d30620f7fc6717b78f072 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 25 Jul 2023 14:13:26 +0100 Subject: [PATCH 01/29] feat(cli): add dashboard loadtest command --- cli/exp_scaletest.go | 122 ++++++++++++++++++++++++++++++++++ cli/exp_scaletest_test.go | 25 +++++++ scaletest/dashboard/config.go | 36 ++++++++++ scaletest/dashboard/run.go | 79 ++++++++++++++++++++++ 4 files changed, 262 insertions(+) create mode 100644 scaletest/dashboard/config.go create mode 100644 scaletest/dashboard/run.go diff --git a/cli/exp_scaletest.go b/cli/exp_scaletest.go index f74cc49eb693f..90bcd74c17fd6 100644 --- a/cli/exp_scaletest.go +++ b/cli/exp_scaletest.go @@ -30,6 +30,7 @@ import ( "github.com/coder/coder/cryptorand" "github.com/coder/coder/scaletest/agentconn" "github.com/coder/coder/scaletest/createworkspaces" + "github.com/coder/coder/scaletest/dashboard" "github.com/coder/coder/scaletest/harness" "github.com/coder/coder/scaletest/reconnectingpty" "github.com/coder/coder/scaletest/workspacebuild" @@ -47,6 +48,7 @@ func (r *RootCmd) scaletestCmd() *clibase.Cmd { }, Children: []*clibase.Cmd{ r.scaletestCleanup(), + r.scaletestDashboard(), r.scaletestCreateWorkspaces(), r.scaletestWorkspaceTraffic(), }, @@ -1033,6 +1035,126 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd { return cmd } +func (r *RootCmd) scaletestDashboard() *clibase.Cmd { + var ( + count int64 + minWait time.Duration + maxWait time.Duration + + client = &codersdk.Client{} + tracingFlags = &scaletestTracingFlags{} + strategy = &scaletestStrategyFlags{} + cleanupStrategy = &scaletestStrategyFlags{cleanup: true} + output = &scaletestOutputFlags{} + ) + + cmd := &clibase.Cmd{ + Use: "dashboard", + Short: "Generate traffic to the HTTP API to simulate use of the dashboard.", + Middleware: clibase.Chain( + r.InitClient(client), + ), + Handler: func(inv *clibase.Invocation) error { + ctx := inv.Context() + logger := slog.Make(sloghuman.Sink(inv.Stdout)).Leveled(slog.LevelInfo) + tracerProvider, closeTracing, tracingEnabled, err := tracingFlags.provider(ctx) + if err != nil { + return xerrors.Errorf("create tracer provider: %w", err) + } + defer func() { + // Allow time for traces to flush even if command context is + // canceled. This is a no-op if tracing is not enabled. + _, _ = fmt.Fprintln(inv.Stderr, "\nUploading traces...") + if err := closeTracing(ctx); err != nil { + _, _ = fmt.Fprintf(inv.Stderr, "\nError uploading traces: %+v\n", err) + } + }() + tracer := tracerProvider.Tracer(scaletestTracerName) + outputs, err := output.parse() + if err != nil { + return xerrors.Errorf("could not parse --output flags") + } + + th := harness.NewTestHarness(strategy.toStrategy(), cleanupStrategy.toStrategy()) + config := dashboard.Config{ + MinWait: minWait, + MaxWait: maxWait, + Trace: tracingEnabled, + Logger: logger, + } + if err := config.Validate(); err != nil { + return err + } + + for i := int64(0); i < count; i++ { + name := fmt.Sprintf("dashboard-%d", i) + var runner harness.Runnable = dashboard.NewRunner(client, config) + if tracingEnabled { + runner = &runnableTraceWrapper{ + tracer: tracer, + spanName: name, + runner: runner, + } + } + th.AddRun("dashboard", name, runner) + } + + _, _ = fmt.Fprintln(inv.Stderr, "Running load test...") + testCtx, testCancel := strategy.toContext(ctx) + defer testCancel() + err = th.Run(testCtx) + if err != nil { + return xerrors.Errorf("run test harness (harness failure, not a test failure): %w", err) + } + + res := th.Results() + for _, o := range outputs { + err = o.write(res, inv.Stdout) + if err != nil { + return xerrors.Errorf("write output %q to %q: %w", o.format, o.path, err) + } + } + + if res.TotalFail > 0 { + return xerrors.New("load test failed, see above for more details") + } + + return nil + }, + } + + cmd.Options = []clibase.Option{ + { + Flag: "count", + Env: "CODER_SCALETEST_DASHBOARD_COUNT", + Default: "1", + Description: "Number of concurrent workers.", + Value: clibase.Int64Of(&count), + }, + { + Flag: "min-wait", + Env: "CODER_SCALETEST_DASHBOARD_MIN_WAIT", + Default: "100ms", + Description: "Minimum wait between fetches.", + Value: clibase.DurationOf(&minWait), + }, + { + Flag: "max-wait", + Env: "CODER_SCALETEST_DASHBOARD_MAX_WAIT", + Default: "1s", + Description: "Maximum wait between fetches.", + Value: clibase.DurationOf(&maxWait), + }, + } + + tracingFlags.attach(&cmd.Options) + strategy.attach(&cmd.Options) + cleanupStrategy.attach(&cmd.Options) + output.attach(&cmd.Options) + + return cmd +} + type runnableTraceWrapper struct { tracer trace.Tracer spanName string diff --git a/cli/exp_scaletest_test.go b/cli/exp_scaletest_test.go index 940ba65eb9264..bfec179b77c97 100644 --- a/cli/exp_scaletest_test.go +++ b/cli/exp_scaletest_test.go @@ -78,3 +78,28 @@ func TestScaleTestWorkspaceTraffic(t *testing.T) { err := inv.WithContext(ctx).Run() require.ErrorContains(t, err, "no scaletest workspaces exist") } + +// This test just validates that the CLI command accepts its known arguments. +// A more comprehensive test is performed in dashboard/run_test.go +func TestScaleTestDashboard(t *testing.T) { + t.Parallel() + + ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitMedium) + defer cancelFunc() + + client := coderdtest.New(t, nil) + _ = coderdtest.CreateFirstUser(t, client) + + inv, root := clitest.New(t, "exp", "scaletest", "dashboard", + "--count", "1", + "--min-wait", "100ms", + "--max-wait", "1s", + "--timeout", "1s", + ) + clitest.SetupConfig(t, client, root) + var stdout, stderr bytes.Buffer + inv.Stdout = &stdout + inv.Stderr = &stderr + err := inv.WithContext(ctx).Run() + require.NoError(t, err, "") +} diff --git a/scaletest/dashboard/config.go b/scaletest/dashboard/config.go new file mode 100644 index 0000000000000..83232cf0667f4 --- /dev/null +++ b/scaletest/dashboard/config.go @@ -0,0 +1,36 @@ +package dashboard + +import ( + "time" + + "golang.org/x/xerrors" + + "cdr.dev/slog" +) + +type Config struct { + // MinWait is the minimum interval between fetches. + MinWait time.Duration `json:"duration_min"` + // MaxWait is the maximum interval between fetches. + MaxWait time.Duration `json:"duration_max"` + // Trace is whether to trace the requests. + Trace bool `json:"trace"` + // Logger is the logger to use. + Logger slog.Logger `json:"-"` +} + +func (c Config) Validate() error { + if c.MinWait <= 0 { + return xerrors.Errorf("validate duration_min: must be greater than zero") + } + + if c.MaxWait <= 0 { + return xerrors.Errorf("validate duration_max: must be greater than zero") + } + + if c.MinWait > c.MaxWait { + return xerrors.Errorf("validate duration_min: must be less than duration_max") + } + + return nil +} diff --git a/scaletest/dashboard/run.go b/scaletest/dashboard/run.go new file mode 100644 index 0000000000000..1da78aac8c9df --- /dev/null +++ b/scaletest/dashboard/run.go @@ -0,0 +1,79 @@ +package dashboard + +import ( + "context" + "io" + "math/rand" + "time" + + "cdr.dev/slog" + "github.com/coder/coder/codersdk" + "github.com/coder/coder/scaletest/harness" +) + +type Runner struct { + client *codersdk.Client + cfg Config +} + +var ( + _ harness.Runnable = &Runner{} + _ harness.Cleanable = &Runner{} +) + +func NewRunner(client *codersdk.Client, cfg Config) *Runner { + client.Trace = cfg.Trace + return &Runner{ + client: client, + cfg: cfg, + } +} + +func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error { + go r.do(ctx, "fetch workspaces", func(client *codersdk.Client) error { + _, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{}) + return err + }) + go r.do(ctx, "fetch users", func(client *codersdk.Client) error { + _, err := client.Users(ctx, codersdk.UsersRequest{}) + return err + }) + go r.do(ctx, "fetch templates", func(client *codersdk.Client) error { + me, err := client.User(ctx, codersdk.Me) + if err != nil { + return err + } + _, err = client.TemplatesByOrganization(ctx, me.OrganizationIDs[0]) + return err + }) + <-ctx.Done() + return nil +} + +func (*Runner) Cleanup(_ context.Context, _ string) error { + return nil +} + +func (r *Runner) do(ctx context.Context, label string, fn func(client *codersdk.Client) error) { + t := time.NewTicker(r.randWait()) + defer t.Stop() + for { + select { + case <-ctx.Done(): + r.cfg.Logger.Info(ctx, "context done, stopping") + return + case <-t.C: + r.cfg.Logger.Info(ctx, "running function", slog.F("fn", label)) + if err := fn(r.client); err != nil { + r.cfg.Logger.Error(ctx, "function returned error", slog.Error(err), slog.F("fn", label)) + } + t.Reset(r.randWait()) + } + } +} + +func (r *Runner) randWait() time.Duration { + // nolint:gosec // This is not for cryptographic purposes. Chill, gosec. Chill. + wait := time.Duration(rand.Intn(int(r.cfg.MaxWait) - int(r.cfg.MinWait))) + return r.cfg.MinWait + wait +} From 8d95283c7156e5e5269e252b8c31203b328d7cef Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 25 Jul 2023 14:16:01 +0100 Subject: [PATCH 02/29] testing From 3eac7cd9d5cc3ab995cbcb731bf883b86ec7be7c Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 25 Jul 2023 14:17:32 +0100 Subject: [PATCH 03/29] testing again From c382f4c073234b674a5bb8db0872919f7cbc701a Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 25 Jul 2023 14:18:03 +0100 Subject: [PATCH 04/29] testing again From 73191706df93e6aa1cf91a6e391b6d38346ec489 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 25 Jul 2023 14:21:56 +0100 Subject: [PATCH 05/29] testing again From e310af7be0d54617cd4ab7c96d4c9b651ac7f0ae Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 25 Jul 2023 14:27:25 +0100 Subject: [PATCH 06/29] testing again From c6b5322d9737cf9556fff06ec2396ae245da421c Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 25 Jul 2023 15:13:28 +0100 Subject: [PATCH 07/29] ensure we are logged in --- scaletest/dashboard/run.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scaletest/dashboard/run.go b/scaletest/dashboard/run.go index 1da78aac8c9df..043ee27ef8bcd 100644 --- a/scaletest/dashboard/run.go +++ b/scaletest/dashboard/run.go @@ -30,6 +30,10 @@ func NewRunner(client *codersdk.Client, cfg Config) *Runner { } func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error { + _, err := r.client.User(ctx, codersdk.Me) + if err != nil { + return err + } go r.do(ctx, "fetch workspaces", func(client *codersdk.Client) error { _, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{}) return err From 298a4f1f83adeb012b3428e0052e7cf2b6b02887 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 25 Jul 2023 15:17:08 +0100 Subject: [PATCH 08/29] log elapsed time --- scaletest/dashboard/run.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/scaletest/dashboard/run.go b/scaletest/dashboard/run.go index 043ee27ef8bcd..3d3ff510ccadb 100644 --- a/scaletest/dashboard/run.go +++ b/scaletest/dashboard/run.go @@ -67,9 +67,21 @@ func (r *Runner) do(ctx context.Context, label string, fn func(client *codersdk. r.cfg.Logger.Info(ctx, "context done, stopping") return case <-t.C: - r.cfg.Logger.Info(ctx, "running function", slog.F("fn", label)) - if err := fn(r.client); err != nil { - r.cfg.Logger.Error(ctx, "function returned error", slog.Error(err), slog.F("fn", label)) + start := time.Now() + err := fn(r.client) + elapsed := time.Since(start) + if err != nil { + r.cfg.Logger.Error( + ctx, "function returned error", + slog.Error(err), + slog.F("fn", label), + slog.F("elapsed", elapsed), + ) + } else { + r.cfg.Logger.Info(ctx, "completed successfully", + slog.F("fn", label), + slog.F("elapsed", elapsed), + ) } t.Reset(r.randWait()) } From fb183e996b69d960df0c51323a7cbf170f3ad6cd Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 25 Jul 2023 15:25:39 +0100 Subject: [PATCH 09/29] name logger --- cli/exp_scaletest.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cli/exp_scaletest.go b/cli/exp_scaletest.go index 90bcd74c17fd6..38418546b186e 100644 --- a/cli/exp_scaletest.go +++ b/cli/exp_scaletest.go @@ -1076,18 +1076,18 @@ func (r *RootCmd) scaletestDashboard() *clibase.Cmd { } th := harness.NewTestHarness(strategy.toStrategy(), cleanupStrategy.toStrategy()) - config := dashboard.Config{ - MinWait: minWait, - MaxWait: maxWait, - Trace: tracingEnabled, - Logger: logger, - } - if err := config.Validate(); err != nil { - return err - } for i := int64(0); i < count; i++ { name := fmt.Sprintf("dashboard-%d", i) + config := dashboard.Config{ + MinWait: minWait, + MaxWait: maxWait, + Trace: tracingEnabled, + Logger: logger.Named(name), + } + if err := config.Validate(); err != nil { + return err + } var runner harness.Runnable = dashboard.NewRunner(client, config) if tracingEnabled { runner = &runnableTraceWrapper{ From 7eeda394a08342d40fd42288bfe5e5e3c48af3ce Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 25 Jul 2023 15:49:53 +0100 Subject: [PATCH 10/29] add auth req --- codersdk/rbacresources.go | 41 +++++++++++++++++++++++++++++++++ scaletest/dashboard/run.go | 46 +++++++++++++++++++++++++++++++++----- 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/codersdk/rbacresources.go b/codersdk/rbacresources.go index 7db5fc0ec1c76..fc1a7b209b393 100644 --- a/codersdk/rbacresources.go +++ b/codersdk/rbacresources.go @@ -27,6 +27,47 @@ const ( ResourceSystem RBACResource = "system" ) +const ( + ActionCreate = "create" + ActionRead = "read" + ActionUpdate = "update" + ActionDelete = "delete" +) + +var ( + AllRBACResources = []RBACResource{ + ResourceWorkspace, + ResourceWorkspaceProxy, + ResourceWorkspaceExecution, + ResourceWorkspaceApplicationConnect, + ResourceAuditLog, + ResourceTemplate, + ResourceGroup, + ResourceFile, + ResourceProvisionerDaemon, + ResourceOrganization, + ResourceRoleAssignment, + ResourceOrgRoleAssignment, + ResourceAPIKey, + ResourceUser, + ResourceUserData, + ResourceOrganizationMember, + ResourceLicense, + ResourceDeploymentValues, + ResourceDeploymentStats, + ResourceReplicas, + ResourceDebugInfo, + ResourceSystem, + } + + AllRBACActions = []string{ + ActionCreate, + ActionRead, + ActionUpdate, + ActionDelete, + } +) + func (r RBACResource) String() string { return string(r) } diff --git a/scaletest/dashboard/run.go b/scaletest/dashboard/run.go index 3d3ff510ccadb..d31ccc4c278a3 100644 --- a/scaletest/dashboard/run.go +++ b/scaletest/dashboard/run.go @@ -2,10 +2,15 @@ package dashboard import ( "context" + "fmt" "io" "math/rand" "time" + "golang.org/x/xerrors" + + "github.com/google/uuid" + "cdr.dev/slog" "github.com/coder/coder/codersdk" "github.com/coder/coder/scaletest/harness" @@ -30,26 +35,35 @@ func NewRunner(client *codersdk.Client, cfg Config) *Runner { } func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error { - _, err := r.client.User(ctx, codersdk.Me) + me, err := r.client.User(ctx, codersdk.Me) if err != nil { return err } + if len(me.OrganizationIDs) == 0 { + return xerrors.Errorf("user has no organizations") + } + orgID := me.OrganizationIDs[0] + + go r.do(ctx, "auth check", func(client *codersdk.Client) error { + _, err := client.AuthCheck(ctx, randAuthReq(orgID)) + return err + }) + go r.do(ctx, "fetch workspaces", func(client *codersdk.Client) error { _, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{}) return err }) + go r.do(ctx, "fetch users", func(client *codersdk.Client) error { _, err := client.Users(ctx, codersdk.UsersRequest{}) return err }) + go r.do(ctx, "fetch templates", func(client *codersdk.Client) error { - me, err := client.User(ctx, codersdk.Me) - if err != nil { - return err - } - _, err = client.TemplatesByOrganization(ctx, me.OrganizationIDs[0]) + _, err = client.TemplatesByOrganization(ctx, orgID) return err }) + <-ctx.Done() return nil } @@ -93,3 +107,23 @@ func (r *Runner) randWait() time.Duration { wait := time.Duration(rand.Intn(int(r.cfg.MaxWait) - int(r.cfg.MinWait))) return r.cfg.MinWait + wait } + +// nolint: gosec +func randAuthReq(orgID uuid.UUID) codersdk.AuthorizationRequest { + objType := codersdk.AllRBACResources[rand.Intn(len(codersdk.AllRBACResources))] + action := codersdk.AllRBACActions[rand.Intn(len(codersdk.AllRBACActions))] + subjectID := uuid.New().String() + checkName := fmt.Sprintf("%s-%s-%s", action, objType, subjectID) + return codersdk.AuthorizationRequest{ + Checks: map[string]codersdk.AuthorizationCheck{ + checkName: { + Object: codersdk.AuthorizationObject{ + ResourceType: objType, + OrganizationID: orgID.String(), + ResourceID: subjectID, + }, + Action: action, + }, + }, + } +} From 8cbd90aeca07d9d8190f0527e0b8408bdbefa412 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 25 Jul 2023 16:12:47 +0100 Subject: [PATCH 11/29] add more auth checks --- scaletest/dashboard/run.go | 69 +++++++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 16 deletions(-) diff --git a/scaletest/dashboard/run.go b/scaletest/dashboard/run.go index d31ccc4c278a3..1213cfc229b89 100644 --- a/scaletest/dashboard/run.go +++ b/scaletest/dashboard/run.go @@ -2,7 +2,6 @@ package dashboard import ( "context" - "fmt" "io" "math/rand" "time" @@ -44,8 +43,13 @@ func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error { } orgID := me.OrganizationIDs[0] - go r.do(ctx, "auth check", func(client *codersdk.Client) error { - _, err := client.AuthCheck(ctx, randAuthReq(orgID)) + go r.do(ctx, "auth check - owner", func(client *codersdk.Client) error { + _, err := client.AuthCheck(ctx, randAuthReq(orgID, ownedByMe(me.ID), withAction(randAction()), withObjType(randObjectType()))) + return err + }) + + go r.do(ctx, "auth check - not owner", func(client *codersdk.Client) error { + _, err := client.AuthCheck(ctx, randAuthReq(orgID, ownedByMe(uuid.New()), withAction(randAction()), withObjType(randObjectType()))) return err }) @@ -109,21 +113,54 @@ func (r *Runner) randWait() time.Duration { } // nolint: gosec -func randAuthReq(orgID uuid.UUID) codersdk.AuthorizationRequest { - objType := codersdk.AllRBACResources[rand.Intn(len(codersdk.AllRBACResources))] - action := codersdk.AllRBACActions[rand.Intn(len(codersdk.AllRBACActions))] - subjectID := uuid.New().String() - checkName := fmt.Sprintf("%s-%s-%s", action, objType, subjectID) +func randAuthReq(orgID uuid.UUID, mut ...func(*codersdk.AuthorizationCheck)) codersdk.AuthorizationRequest { + var check codersdk.AuthorizationCheck + for _, m := range mut { + m(&check) + } return codersdk.AuthorizationRequest{ Checks: map[string]codersdk.AuthorizationCheck{ - checkName: { - Object: codersdk.AuthorizationObject{ - ResourceType: objType, - OrganizationID: orgID.String(), - ResourceID: subjectID, - }, - Action: action, - }, + "check": check, }, } } + +func ownedByMe(myID uuid.UUID) func(check *codersdk.AuthorizationCheck) { + return func(check *codersdk.AuthorizationCheck) { + check.Object.OwnerID = myID.String() + } +} + +func inOrg(orgID uuid.UUID) func(check *codersdk.AuthorizationCheck) { + return func(check *codersdk.AuthorizationCheck) { + check.Object.OrganizationID = orgID.String() + } +} + +func withResourceID(id uuid.UUID) func(check *codersdk.AuthorizationCheck) { + return func(check *codersdk.AuthorizationCheck) { + check.Object.ResourceID = id.String() + } +} + +func withObjType(objType codersdk.RBACResource) func(check *codersdk.AuthorizationCheck) { + return func(check *codersdk.AuthorizationCheck) { + check.Object.ResourceType = objType + } +} + +func withAction(action string) func(check *codersdk.AuthorizationCheck) { + return func(check *codersdk.AuthorizationCheck) { + check.Action = action + } +} + +func randAction() string { + // nolint:gosec + return codersdk.AllRBACActions[rand.Intn(len(codersdk.AllRBACActions))] +} + +func randObjectType() codersdk.RBACResource { + // nolint:gosec + return codersdk.AllRBACResources[rand.Intn(len(codersdk.AllRBACResources))] +} From 765db11bdf60abb9f3a0deaf7d65d7ad316c22fe Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 25 Jul 2023 16:29:45 +0100 Subject: [PATCH 12/29] move to a roll table so we have probabilities --- scaletest/dashboard/run.go | 137 ++++++++++++++++++++++--------------- 1 file changed, 83 insertions(+), 54 deletions(-) diff --git a/scaletest/dashboard/run.go b/scaletest/dashboard/run.go index 1213cfc229b89..e85d6ee4e469b 100644 --- a/scaletest/dashboard/run.go +++ b/scaletest/dashboard/run.go @@ -42,34 +42,68 @@ func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error { return xerrors.Errorf("user has no organizations") } orgID := me.OrganizationIDs[0] + rolls := make(chan int) + // Roll a die + go func() { + t := time.NewTicker(r.randWait()) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + rolls <- rand.Intn(6) + t.Reset(r.randWait()) + } + } + }() - go r.do(ctx, "auth check - owner", func(client *codersdk.Client) error { - _, err := client.AuthCheck(ctx, randAuthReq(orgID, ownedByMe(me.ID), withAction(randAction()), withObjType(randObjectType()))) - return err - }) - - go r.do(ctx, "auth check - not owner", func(client *codersdk.Client) error { - _, err := client.AuthCheck(ctx, randAuthReq(orgID, ownedByMe(uuid.New()), withAction(randAction()), withObjType(randObjectType()))) - return err - }) - - go r.do(ctx, "fetch workspaces", func(client *codersdk.Client) error { - _, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{}) - return err - }) - - go r.do(ctx, "fetch users", func(client *codersdk.Client) error { - _, err := client.Users(ctx, codersdk.UsersRequest{}) - return err - }) - - go r.do(ctx, "fetch templates", func(client *codersdk.Client) error { - _, err = client.TemplatesByOrganization(ctx, orgID) - return err - }) - - <-ctx.Done() - return nil + for { + select { + case <-ctx.Done(): + return nil + case n := <-rolls: + switch n { + case 0: + go r.do(ctx, "fetch workspaces", func(client *codersdk.Client) error { + _, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{}) + return err + }) + case 1: + go r.do(ctx, "fetch users", func(client *codersdk.Client) error { + _, err := client.Users(ctx, codersdk.UsersRequest{}) + return err + }) + case 2: + go r.do(ctx, "fetch templates", func(client *codersdk.Client) error { + _, err = client.TemplatesByOrganization(ctx, orgID) + return err + }) + case 3: + go r.do(ctx, "auth check - owner", func(client *codersdk.Client) error { + _, err := client.AuthCheck(ctx, randAuthReq( + ownedBy(me.ID), + withAction(randAction()), + withObjType(randObjectType()), + inOrg(orgID), + )) + return err + }) + case 4: + go r.do(ctx, "auth check - not owner", func(client *codersdk.Client) error { + _, err := client.AuthCheck(ctx, randAuthReq( + ownedBy(uuid.New()), + withAction(randAction()), + withObjType(randObjectType()), + inOrg(orgID), + )) + return err + }) + default: + r.cfg.Logger.Warn(ctx, "no action for roll - skipping my turn", slog.F("roll", n)) + } + } + } } func (*Runner) Cleanup(_ context.Context, _ string) error { @@ -77,31 +111,26 @@ func (*Runner) Cleanup(_ context.Context, _ string) error { } func (r *Runner) do(ctx context.Context, label string, fn func(client *codersdk.Client) error) { - t := time.NewTicker(r.randWait()) - defer t.Stop() - for { - select { - case <-ctx.Done(): - r.cfg.Logger.Info(ctx, "context done, stopping") - return - case <-t.C: - start := time.Now() - err := fn(r.client) - elapsed := time.Since(start) - if err != nil { - r.cfg.Logger.Error( - ctx, "function returned error", - slog.Error(err), - slog.F("fn", label), - slog.F("elapsed", elapsed), - ) - } else { - r.cfg.Logger.Info(ctx, "completed successfully", - slog.F("fn", label), - slog.F("elapsed", elapsed), - ) - } - t.Reset(r.randWait()) + select { + case <-ctx.Done(): + r.cfg.Logger.Info(ctx, "context done, stopping") + return + default: + start := time.Now() + err := fn(r.client) + elapsed := time.Since(start) + if err != nil { + r.cfg.Logger.Error( + ctx, "function returned error", + slog.Error(err), + slog.F("fn", label), + slog.F("elapsed", elapsed), + ) + } else { + r.cfg.Logger.Info(ctx, "completed successfully", + slog.F("fn", label), + slog.F("elapsed", elapsed), + ) } } } @@ -113,7 +142,7 @@ func (r *Runner) randWait() time.Duration { } // nolint: gosec -func randAuthReq(orgID uuid.UUID, mut ...func(*codersdk.AuthorizationCheck)) codersdk.AuthorizationRequest { +func randAuthReq(mut ...func(*codersdk.AuthorizationCheck)) codersdk.AuthorizationRequest { var check codersdk.AuthorizationCheck for _, m := range mut { m(&check) @@ -125,7 +154,7 @@ func randAuthReq(orgID uuid.UUID, mut ...func(*codersdk.AuthorizationCheck)) cod } } -func ownedByMe(myID uuid.UUID) func(check *codersdk.AuthorizationCheck) { +func ownedBy(myID uuid.UUID) func(check *codersdk.AuthorizationCheck) { return func(check *codersdk.AuthorizationCheck) { check.Object.OwnerID = myID.String() } From 08e6d418caf41eb85ef312d857ca8befbae99bb8 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 25 Jul 2023 18:09:01 +0100 Subject: [PATCH 13/29] extract to a roll table --- scaletest/dashboard/rolltable.go | 131 +++++++++++++++++++++++++++++++ scaletest/dashboard/run.go | 59 +++----------- 2 files changed, 143 insertions(+), 47 deletions(-) create mode 100644 scaletest/dashboard/rolltable.go diff --git a/scaletest/dashboard/rolltable.go b/scaletest/dashboard/rolltable.go new file mode 100644 index 0000000000000..ce7b24adfa9d5 --- /dev/null +++ b/scaletest/dashboard/rolltable.go @@ -0,0 +1,131 @@ +package dashboard + +import ( + "context" + + "github.com/google/uuid" + + "github.com/coder/coder/codersdk" +) + +// rollTable is a table of actions to perform. +// D&D nerds will feel right at home here :-) +// Note that the order of the table is important! +// Entries must be in ascending order. +var allActions rollTable = []rollTableEntry{ + {00, fetchWorkspaces, "fetch workspaces"}, + {10, fetchUsers, "fetch users"}, + {20, fetchTemplates, "fetch templates"}, + {30, authCheckAsOwner, "authcheck owner"}, + {40, authCheckAsNonOwner, "authcheck not owner"}, + {50, fetchAuditLog, "fetch audit log"}, +} + +// rollTable is a slice of rollTableEntry. +type rollTable []rollTableEntry + +// rollTableEntry is an entry in the roll table. +type rollTableEntry struct { + // roll is the minimum number required to perform the action. + roll int + // fn is the function to call. + fn func(ctx context.Context, p *params) error + // label is used for logging. + label string +} + +// choose returns the first entry in the table that is greater than or equal to n. +func (r rollTable) choose(n int) rollTableEntry { + for _, entry := range r { + if entry.roll >= n { + return entry + } + } + return rollTableEntry{} +} + +// max returns the maximum roll in the table. +// Important: this assumes that the table is sorted in ascending order. +func (r rollTable) max() int { + return r[len(r)-1].roll +} + +// params is a set of parameters to pass to the actions in a rollTable. +type params struct { + // client is the client to use for performing the action. + client *codersdk.Client + // me is the currently authenticated user. Lots of actions require this. + me codersdk.User +} + +// fetchWorkspaces fetches all workspaces. +func fetchWorkspaces(ctx context.Context, p *params) error { + _, err := p.client.Workspaces(ctx, codersdk.WorkspaceFilter{}) + return err +} + +// fetchUsers fetches all users. +func fetchUsers(ctx context.Context, p *params) error { + _, err := p.client.Users(ctx, codersdk.UsersRequest{}) + return err +} + +// fetchTemplates fetches all templates. +func fetchTemplates(ctx context.Context, p *params) error { + _, err := p.client.TemplatesByOrganization(ctx, p.me.OrganizationIDs[0]) + return err +} + +// fetchAuditLog fetches the audit log. +// As not all users have access to the audit log, we check first. +func fetchAuditLog(ctx context.Context, p *params) error { + res, err := p.client.AuthCheck(ctx, codersdk.AuthorizationRequest{ + Checks: map[string]codersdk.AuthorizationCheck{ + "auditlog": { + Object: codersdk.AuthorizationObject{ + ResourceType: codersdk.ResourceAuditLog, + }, + Action: codersdk.ActionRead, + }, + }, + }) + if err != nil { + return err + } + if !res["auditlog"] { + return nil // we are not authorized to read the audit log + } + + // Fetch the first 25 audit log entries. + _, err = p.client.AuditLogs(ctx, codersdk.AuditLogsRequest{ + Pagination: codersdk.Pagination{ + Offset: 0, + Limit: 25, + }, + }) + return err +} + +// authCheckAsOwner performs an auth check as the owner of a random +// resource type and action. +func authCheckAsOwner(ctx context.Context, p *params) error { + _, err := p.client.AuthCheck(ctx, randAuthReq( + ownedBy(p.me.ID), + withAction(randAction()), + withObjType(randObjectType()), + inOrg(p.me.OrganizationIDs[0]), + )) + return err +} + +// authCheckAsNonOwner performs an auth check as a non-owner of a random +// resource type and action. +func authCheckAsNonOwner(ctx context.Context, p *params) error { + _, err := p.client.AuthCheck(ctx, randAuthReq( + ownedBy(uuid.New()), + withAction(randAction()), + withObjType(randObjectType()), + inOrg(p.me.OrganizationIDs[0]), + )) + return err +} diff --git a/scaletest/dashboard/run.go b/scaletest/dashboard/run.go index e85d6ee4e469b..ce0da5d918a7f 100644 --- a/scaletest/dashboard/run.go +++ b/scaletest/dashboard/run.go @@ -41,9 +41,11 @@ func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error { if len(me.OrganizationIDs) == 0 { return xerrors.Errorf("user has no organizations") } - orgID := me.OrganizationIDs[0] + p := ¶ms{ + client: r.client, + me: me, + } rolls := make(chan int) - // Roll a die go func() { t := time.NewTicker(r.randWait()) defer t.Stop() @@ -52,7 +54,7 @@ func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error { case <-ctx.Done(): return case <-t.C: - rolls <- rand.Intn(6) + rolls <- rand.Intn(allActions.max()) // nolint:gosec t.Reset(r.randWait()) } } @@ -63,45 +65,8 @@ func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error { case <-ctx.Done(): return nil case n := <-rolls: - switch n { - case 0: - go r.do(ctx, "fetch workspaces", func(client *codersdk.Client) error { - _, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{}) - return err - }) - case 1: - go r.do(ctx, "fetch users", func(client *codersdk.Client) error { - _, err := client.Users(ctx, codersdk.UsersRequest{}) - return err - }) - case 2: - go r.do(ctx, "fetch templates", func(client *codersdk.Client) error { - _, err = client.TemplatesByOrganization(ctx, orgID) - return err - }) - case 3: - go r.do(ctx, "auth check - owner", func(client *codersdk.Client) error { - _, err := client.AuthCheck(ctx, randAuthReq( - ownedBy(me.ID), - withAction(randAction()), - withObjType(randObjectType()), - inOrg(orgID), - )) - return err - }) - case 4: - go r.do(ctx, "auth check - not owner", func(client *codersdk.Client) error { - _, err := client.AuthCheck(ctx, randAuthReq( - ownedBy(uuid.New()), - withAction(randAction()), - withObjType(randObjectType()), - inOrg(orgID), - )) - return err - }) - default: - r.cfg.Logger.Warn(ctx, "no action for roll - skipping my turn", slog.F("roll", n)) - } + act := allActions.choose(n) + go r.do(ctx, act, p) } } } @@ -110,25 +75,25 @@ func (*Runner) Cleanup(_ context.Context, _ string) error { return nil } -func (r *Runner) do(ctx context.Context, label string, fn func(client *codersdk.Client) error) { +func (r *Runner) do(ctx context.Context, act rollTableEntry, p *params) { select { case <-ctx.Done(): r.cfg.Logger.Info(ctx, "context done, stopping") return default: start := time.Now() - err := fn(r.client) + err := act.fn(ctx, p) elapsed := time.Since(start) if err != nil { r.cfg.Logger.Error( - ctx, "function returned error", + ctx, "action failed", slog.Error(err), - slog.F("fn", label), + slog.F("action", act.label), slog.F("elapsed", elapsed), ) } else { r.cfg.Logger.Info(ctx, "completed successfully", - slog.F("fn", label), + slog.F("action", act.label), slog.F("elapsed", elapsed), ) } From 4edeacfce7af85837473b871fcb391f8f78cd60d Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 25 Jul 2023 18:09:44 +0100 Subject: [PATCH 14/29] make fmt --- cli/exp_scaletest_test.go | 1 - scaletest/dashboard/rolltable.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cli/exp_scaletest_test.go b/cli/exp_scaletest_test.go index bfec179b77c97..00e344f15c3d6 100644 --- a/cli/exp_scaletest_test.go +++ b/cli/exp_scaletest_test.go @@ -80,7 +80,6 @@ func TestScaleTestWorkspaceTraffic(t *testing.T) { } // This test just validates that the CLI command accepts its known arguments. -// A more comprehensive test is performed in dashboard/run_test.go func TestScaleTestDashboard(t *testing.T) { t.Parallel() diff --git a/scaletest/dashboard/rolltable.go b/scaletest/dashboard/rolltable.go index ce7b24adfa9d5..6bf1dd3ef4765 100644 --- a/scaletest/dashboard/rolltable.go +++ b/scaletest/dashboard/rolltable.go @@ -13,7 +13,7 @@ import ( // Note that the order of the table is important! // Entries must be in ascending order. var allActions rollTable = []rollTableEntry{ - {00, fetchWorkspaces, "fetch workspaces"}, + {0, fetchWorkspaces, "fetch workspaces"}, {10, fetchUsers, "fetch users"}, {20, fetchTemplates, "fetch templates"}, {30, authCheckAsOwner, "authcheck owner"}, From f5b134a98b3e5c8e6ec9b1486fe46880c22b5836 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 25 Jul 2023 18:20:26 +0100 Subject: [PATCH 15/29] move main roll table closer to where it is used --- scaletest/dashboard/rolltable.go | 29 ++++++++++++++++------------- scaletest/dashboard/run.go | 23 ++++++++++++++++------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/scaletest/dashboard/rolltable.go b/scaletest/dashboard/rolltable.go index 6bf1dd3ef4765..179d884610c52 100644 --- a/scaletest/dashboard/rolltable.go +++ b/scaletest/dashboard/rolltable.go @@ -8,19 +8,6 @@ import ( "github.com/coder/coder/codersdk" ) -// rollTable is a table of actions to perform. -// D&D nerds will feel right at home here :-) -// Note that the order of the table is important! -// Entries must be in ascending order. -var allActions rollTable = []rollTableEntry{ - {0, fetchWorkspaces, "fetch workspaces"}, - {10, fetchUsers, "fetch users"}, - {20, fetchTemplates, "fetch templates"}, - {30, authCheckAsOwner, "authcheck owner"}, - {40, authCheckAsNonOwner, "authcheck not owner"}, - {50, fetchAuditLog, "fetch audit log"}, -} - // rollTable is a slice of rollTableEntry. type rollTable []rollTableEntry @@ -70,6 +57,22 @@ func fetchUsers(ctx context.Context, p *params) error { return err } +// fetchActiveUsers fetches all active users +func fetchActiveUsers(ctx context.Context, p *params) error { + _, err := p.client.Users(ctx, codersdk.UsersRequest{ + Status: codersdk.UserStatusActive, + }) + return err +} + +// fetchSuspendedUsers fetches all suspended users +func fetchSuspendedUsers(ctx context.Context, p *params) error { + _, err := p.client.Users(ctx, codersdk.UsersRequest{ + Status: codersdk.UserStatusSuspended, + }) + return err +} + // fetchTemplates fetches all templates. func fetchTemplates(ctx context.Context, p *params) error { _, err := p.client.TemplatesByOrganization(ctx, p.me.OrganizationIDs[0]) diff --git a/scaletest/dashboard/run.go b/scaletest/dashboard/run.go index ce0da5d918a7f..89b9f84db6223 100644 --- a/scaletest/dashboard/run.go +++ b/scaletest/dashboard/run.go @@ -15,6 +15,21 @@ import ( "github.com/coder/coder/scaletest/harness" ) +// rollTable is a table of actions to perform. +// D&D nerds will feel right at home here :-) +// Note that the order of the table is important! +// Entries must be in ascending order. +var allActions rollTable = []rollTableEntry{ + {0, fetchWorkspaces, "fetch workspaces"}, + {10, fetchUsers, "fetch users"}, + {20, fetchTemplates, "fetch templates"}, + {30, authCheckAsOwner, "authcheck owner"}, + {40, authCheckAsNonOwner, "authcheck not owner"}, + {50, fetchAuditLog, "fetch audit log"}, + {60, fetchActiveUsers, "fetch active users"}, + {70, fetchSuspendedUsers, "fetch suspended users"}, +} + type Runner struct { client *codersdk.Client cfg Config @@ -85,7 +100,7 @@ func (r *Runner) do(ctx context.Context, act rollTableEntry, p *params) { err := act.fn(ctx, p) elapsed := time.Since(start) if err != nil { - r.cfg.Logger.Error( + r.cfg.Logger.Error( //nolint:gocritic ctx, "action failed", slog.Error(err), slog.F("action", act.label), @@ -131,12 +146,6 @@ func inOrg(orgID uuid.UUID) func(check *codersdk.AuthorizationCheck) { } } -func withResourceID(id uuid.UUID) func(check *codersdk.AuthorizationCheck) { - return func(check *codersdk.AuthorizationCheck) { - check.Object.ResourceID = id.String() - } -} - func withObjType(objType codersdk.RBACResource) func(check *codersdk.AuthorizationCheck) { return func(check *codersdk.AuthorizationCheck) { check.Object.ResourceType = objType From 0c03f938573cb106cb90ef3bb2b03f7bdfeb8768 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 26 Jul 2023 10:35:10 +0100 Subject: [PATCH 16/29] add test for roll table ordering --- scaletest/dashboard/run.go | 2 +- scaletest/dashboard/run_internal_test.go | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 scaletest/dashboard/run_internal_test.go diff --git a/scaletest/dashboard/run.go b/scaletest/dashboard/run.go index 89b9f84db6223..e4753bd238f37 100644 --- a/scaletest/dashboard/run.go +++ b/scaletest/dashboard/run.go @@ -15,7 +15,7 @@ import ( "github.com/coder/coder/scaletest/harness" ) -// rollTable is a table of actions to perform. +// allActions is a table of actions to perform. // D&D nerds will feel right at home here :-) // Note that the order of the table is important! // Entries must be in ascending order. diff --git a/scaletest/dashboard/run_internal_test.go b/scaletest/dashboard/run_internal_test.go new file mode 100644 index 0000000000000..00be8912b90c8 --- /dev/null +++ b/scaletest/dashboard/run_internal_test.go @@ -0,0 +1,17 @@ +package dashboard + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_allActions_ordering(t *testing.T) { + t.Parallel() + + last := -1 + for idx, entry := range allActions { + require.Greater(t, entry.roll, last, "roll table must be in ascending order, entry %d is out of order", idx) + last = entry.roll + } +} From 70550aa78b47a8cdd0c4bcc6aa2100c61e0b35f7 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 26 Jul 2023 10:36:33 +0100 Subject: [PATCH 17/29] move --- scaletest/dashboard/rolltable.go | 15 +++++++++++++++ ...nternal_test.go => rolltable_internal_test.go} | 0 scaletest/dashboard/run.go | 15 --------------- 3 files changed, 15 insertions(+), 15 deletions(-) rename scaletest/dashboard/{run_internal_test.go => rolltable_internal_test.go} (100%) diff --git a/scaletest/dashboard/rolltable.go b/scaletest/dashboard/rolltable.go index 179d884610c52..a368a8ed0196f 100644 --- a/scaletest/dashboard/rolltable.go +++ b/scaletest/dashboard/rolltable.go @@ -8,6 +8,21 @@ import ( "github.com/coder/coder/codersdk" ) +// allActions is a table of actions to perform. +// D&D nerds will feel right at home here :-) +// Note that the order of the table is important! +// Entries must be in ascending order. +var allActions rollTable = []rollTableEntry{ + {0, fetchWorkspaces, "fetch workspaces"}, + {10, fetchUsers, "fetch users"}, + {20, fetchTemplates, "fetch templates"}, + {30, authCheckAsOwner, "authcheck owner"}, + {40, authCheckAsNonOwner, "authcheck not owner"}, + {50, fetchAuditLog, "fetch audit log"}, + {60, fetchActiveUsers, "fetch active users"}, + {70, fetchSuspendedUsers, "fetch suspended users"}, +} + // rollTable is a slice of rollTableEntry. type rollTable []rollTableEntry diff --git a/scaletest/dashboard/run_internal_test.go b/scaletest/dashboard/rolltable_internal_test.go similarity index 100% rename from scaletest/dashboard/run_internal_test.go rename to scaletest/dashboard/rolltable_internal_test.go diff --git a/scaletest/dashboard/run.go b/scaletest/dashboard/run.go index e4753bd238f37..08b307213f1cc 100644 --- a/scaletest/dashboard/run.go +++ b/scaletest/dashboard/run.go @@ -15,21 +15,6 @@ import ( "github.com/coder/coder/scaletest/harness" ) -// allActions is a table of actions to perform. -// D&D nerds will feel right at home here :-) -// Note that the order of the table is important! -// Entries must be in ascending order. -var allActions rollTable = []rollTableEntry{ - {0, fetchWorkspaces, "fetch workspaces"}, - {10, fetchUsers, "fetch users"}, - {20, fetchTemplates, "fetch templates"}, - {30, authCheckAsOwner, "authcheck owner"}, - {40, authCheckAsNonOwner, "authcheck not owner"}, - {50, fetchAuditLog, "fetch audit log"}, - {60, fetchActiveUsers, "fetch active users"}, - {70, fetchSuspendedUsers, "fetch suspended users"}, -} - type Runner struct { client *codersdk.Client cfg Config From c1291adf3b89618dc80f88800c8932fc084f679e Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 26 Jul 2023 11:31:12 +0100 Subject: [PATCH 18/29] add cache and lots more actions --- scaletest/dashboard/cache.go | 97 ++++++++++++++++++ scaletest/dashboard/rolltable.go | 167 +++++++++++++++++++++++++++++-- scaletest/dashboard/run.go | 58 ++--------- 3 files changed, 262 insertions(+), 60 deletions(-) create mode 100644 scaletest/dashboard/cache.go diff --git a/scaletest/dashboard/cache.go b/scaletest/dashboard/cache.go new file mode 100644 index 0000000000000..2ac31ff22a525 --- /dev/null +++ b/scaletest/dashboard/cache.go @@ -0,0 +1,97 @@ +package dashboard + +import ( + "context" + "math/rand" + "sync" + + "github.com/coder/coder/codersdk" +) + +type cache struct { + sync.RWMutex + workspaces []codersdk.Workspace + templates []codersdk.Template + users []codersdk.User +} + +func (c *cache) fill(ctx context.Context, client *codersdk.Client) error { + c.Lock() + defer c.Unlock() + me, err := client.User(ctx, codersdk.Me) + if err != nil { + return err + } + ws, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{}) + if err != nil { + return err + } + c.workspaces = ws.Workspaces + tpl, err := client.TemplatesByOrganization(ctx, me.OrganizationIDs[0]) + if err != nil { + return err + } + c.templates = tpl + users, err := client.Users(ctx, codersdk.UsersRequest{}) + if err != nil { + return err + } + c.users = users.Users + return nil +} + +func (c *cache) setWorkspaces(ws []codersdk.Workspace) { + c.Lock() + c.workspaces = ws + c.Unlock() +} + +func (c *cache) setTemplates(t []codersdk.Template) { + c.Lock() + c.templates = t + c.Unlock() +} + +func (c *cache) randWorkspace() codersdk.Workspace { + c.RLock() + defer c.RUnlock() + if len(c.workspaces) == 0 { + return codersdk.Workspace{} + } + return pick(c.workspaces) +} + +func (c *cache) randTemplate() codersdk.Template { + c.RLock() + defer c.RUnlock() + if len(c.templates) == 0 { + return codersdk.Template{} + } + return pick(c.templates) +} + +func (c *cache) setUsers(u []codersdk.User) { + c.Lock() + c.users = u + c.Unlock() +} + +func (c *cache) randUser() codersdk.User { + c.RLock() + defer c.RUnlock() + if len(c.users) == 0 { + return codersdk.User{} + } + return pick(c.users) +} + +// pick chooses a random element from a slice. +// If the slice is empty, it returns the zero value of the type. +func pick[T any](s []T) T { + if len(s) == 0 { + var zero T + return zero + } + // nolint:gosec + return s[rand.Intn(len(s))] +} diff --git a/scaletest/dashboard/rolltable.go b/scaletest/dashboard/rolltable.go index a368a8ed0196f..2871df5c3afcb 100644 --- a/scaletest/dashboard/rolltable.go +++ b/scaletest/dashboard/rolltable.go @@ -14,13 +14,23 @@ import ( // Entries must be in ascending order. var allActions rollTable = []rollTableEntry{ {0, fetchWorkspaces, "fetch workspaces"}, - {10, fetchUsers, "fetch users"}, - {20, fetchTemplates, "fetch templates"}, - {30, authCheckAsOwner, "authcheck owner"}, - {40, authCheckAsNonOwner, "authcheck not owner"}, - {50, fetchAuditLog, "fetch audit log"}, - {60, fetchActiveUsers, "fetch active users"}, - {70, fetchSuspendedUsers, "fetch suspended users"}, + {1, fetchUsers, "fetch users"}, + {2, fetchTemplates, "fetch templates"}, + {3, authCheckAsOwner, "authcheck owner"}, + {4, authCheckAsNonOwner, "authcheck not owner"}, + {5, fetchAuditLog, "fetch audit log"}, + {6, fetchActiveUsers, "fetch active users"}, + {7, fetchSuspendedUsers, "fetch suspended users"}, + {8, fetchTemplateVersion, "fetch template version"}, + {9, fetchWorkspace, "fetch workspace"}, + {10, fetchTemplate, "fetch template"}, + {11, fetchUserByID, "fetch user by ID"}, + {12, fetchUserByUsername, "fetch user by username"}, + {13, fetchWorkspaceBuild, "fetch workspace build"}, + {14, fetchDeploymentConfig, "fetch deployment config"}, + {15, fetchWorkspaceQuotaForUser, "fetch workspace quota for user"}, + {16, fetchDeploymentStats, "fetch deployment stats"}, + {17, fetchWorkspaceLogs, "fetch workspace logs"}, } // rollTable is a slice of rollTableEntry. @@ -58,17 +68,29 @@ type params struct { client *codersdk.Client // me is the currently authenticated user. Lots of actions require this. me codersdk.User + // For picking random resource IDs, we need to know what resources are + // present. We store them in a cache to avoid fetching them every time. + // This may seem counter-intuitive for load testing, but we want to avoid + // muddying results. + c *cache } // fetchWorkspaces fetches all workspaces. func fetchWorkspaces(ctx context.Context, p *params) error { - _, err := p.client.Workspaces(ctx, codersdk.WorkspaceFilter{}) + ws, err := p.client.Workspaces(ctx, codersdk.WorkspaceFilter{}) + if err != nil { + // store the workspaces for later use in case they change + p.c.setWorkspaces(ws.Workspaces) + } return err } // fetchUsers fetches all users. func fetchUsers(ctx context.Context, p *params) error { - _, err := p.client.Users(ctx, codersdk.UsersRequest{}) + users, err := p.client.Users(ctx, codersdk.UsersRequest{}) + if err != nil { + p.c.setUsers(users.Users) + } return err } @@ -90,7 +112,87 @@ func fetchSuspendedUsers(ctx context.Context, p *params) error { // fetchTemplates fetches all templates. func fetchTemplates(ctx context.Context, p *params) error { - _, err := p.client.TemplatesByOrganization(ctx, p.me.OrganizationIDs[0]) + templates, err := p.client.TemplatesByOrganization(ctx, p.me.OrganizationIDs[0]) + if err != nil { + p.c.setTemplates(templates) + } + return err +} + +// fetchTemplateBuild fetches a single template version at random. +func fetchTemplateVersion(ctx context.Context, p *params) error { + t := p.c.randTemplate() + _, err := p.client.TemplateVersion(ctx, t.ActiveVersionID) + return err +} + +// fetchWorkspace fetches a single workspace at random. +func fetchWorkspace(ctx context.Context, p *params) error { + w := p.c.randWorkspace() + _, err := p.client.WorkspaceByOwnerAndName(ctx, w.OwnerName, w.Name, codersdk.WorkspaceOptions{}) + return err +} + +// fetchWorkspaceBuild fetches a single workspace build at random. +func fetchWorkspaceBuild(ctx context.Context, p *params) error { + w := p.c.randWorkspace() + _, err := p.client.WorkspaceBuild(ctx, w.LatestBuild.ID) + return err +} + +// fetchTemplate fetches a single template at random. +func fetchTemplate(ctx context.Context, p *params) error { + t := p.c.randTemplate() + _, err := p.client.Template(ctx, t.ID) + return err +} + +// fetchUserByID fetches a single user at random by ID. +func fetchUserByID(ctx context.Context, p *params) error { + u := p.c.randUser() + _, err := p.client.User(ctx, u.ID.String()) + return err +} + +// fetchUserByUsername fetches a single user at random by username. +func fetchUserByUsername(ctx context.Context, p *params) error { + u := p.c.randUser() + _, err := p.client.User(ctx, u.Username) + return err +} + +// fetchDeploymentConfig fetches the deployment config. +func fetchDeploymentConfig(ctx context.Context, p *params) error { + _, err := p.client.DeploymentConfig(ctx) + return err +} + +// fetchWorkspaceQuotaForUser fetches the workspace quota for a random user. +func fetchWorkspaceQuotaForUser(ctx context.Context, p *params) error { + u := p.c.randUser() + _, err := p.client.WorkspaceQuota(ctx, u.ID.String()) + return err +} + +// fetchDeploymentStats fetches the deployment stats. +func fetchDeploymentStats(ctx context.Context, p *params) error { + _, err := p.client.DeploymentStats(ctx) + return err +} + +// fetchWorkspaceLogs fetches the logs for a random workspace. +func fetchWorkspaceLogs(ctx context.Context, p *params) error { + w := p.c.randWorkspace() + ch, closer, err := p.client.WorkspaceBuildLogsAfter(ctx, w.LatestBuild.ID, 0) + if err != nil { + return err + } + defer func() { + _ = closer.Close() + }() + for range ch { + // do nothing + } return err } @@ -147,3 +249,48 @@ func authCheckAsNonOwner(ctx context.Context, p *params) error { )) return err } + +// nolint: gosec +func randAuthReq(mut ...func(*codersdk.AuthorizationCheck)) codersdk.AuthorizationRequest { + var check codersdk.AuthorizationCheck + for _, m := range mut { + m(&check) + } + return codersdk.AuthorizationRequest{ + Checks: map[string]codersdk.AuthorizationCheck{ + "check": check, + }, + } +} + +func ownedBy(myID uuid.UUID) func(check *codersdk.AuthorizationCheck) { + return func(check *codersdk.AuthorizationCheck) { + check.Object.OwnerID = myID.String() + } +} + +func inOrg(orgID uuid.UUID) func(check *codersdk.AuthorizationCheck) { + return func(check *codersdk.AuthorizationCheck) { + check.Object.OrganizationID = orgID.String() + } +} + +func withObjType(objType codersdk.RBACResource) func(check *codersdk.AuthorizationCheck) { + return func(check *codersdk.AuthorizationCheck) { + check.Object.ResourceType = objType + } +} + +func withAction(action string) func(check *codersdk.AuthorizationCheck) { + return func(check *codersdk.AuthorizationCheck) { + check.Action = action + } +} + +func randAction() string { + return pick(codersdk.AllRBACActions) +} + +func randObjectType() codersdk.RBACResource { + return pick(codersdk.AllRBACResources) +} diff --git a/scaletest/dashboard/run.go b/scaletest/dashboard/run.go index 08b307213f1cc..1de7547a07589 100644 --- a/scaletest/dashboard/run.go +++ b/scaletest/dashboard/run.go @@ -8,8 +8,6 @@ import ( "golang.org/x/xerrors" - "github.com/google/uuid" - "cdr.dev/slog" "github.com/coder/coder/codersdk" "github.com/coder/coder/scaletest/harness" @@ -41,9 +39,16 @@ func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error { if len(me.OrganizationIDs) == 0 { return xerrors.Errorf("user has no organizations") } + + c := &cache{} + if err := c.fill(ctx, r.client); err != nil { + return err + } + p := ¶ms{ client: r.client, me: me, + c: c, } rolls := make(chan int) go func() { @@ -54,7 +59,7 @@ func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error { case <-ctx.Done(): return case <-t.C: - rolls <- rand.Intn(allActions.max()) // nolint:gosec + rolls <- rand.Intn(allActions.max() + 1) // nolint:gosec t.Reset(r.randWait()) } } @@ -105,50 +110,3 @@ func (r *Runner) randWait() time.Duration { wait := time.Duration(rand.Intn(int(r.cfg.MaxWait) - int(r.cfg.MinWait))) return r.cfg.MinWait + wait } - -// nolint: gosec -func randAuthReq(mut ...func(*codersdk.AuthorizationCheck)) codersdk.AuthorizationRequest { - var check codersdk.AuthorizationCheck - for _, m := range mut { - m(&check) - } - return codersdk.AuthorizationRequest{ - Checks: map[string]codersdk.AuthorizationCheck{ - "check": check, - }, - } -} - -func ownedBy(myID uuid.UUID) func(check *codersdk.AuthorizationCheck) { - return func(check *codersdk.AuthorizationCheck) { - check.Object.OwnerID = myID.String() - } -} - -func inOrg(orgID uuid.UUID) func(check *codersdk.AuthorizationCheck) { - return func(check *codersdk.AuthorizationCheck) { - check.Object.OrganizationID = orgID.String() - } -} - -func withObjType(objType codersdk.RBACResource) func(check *codersdk.AuthorizationCheck) { - return func(check *codersdk.AuthorizationCheck) { - check.Object.ResourceType = objType - } -} - -func withAction(action string) func(check *codersdk.AuthorizationCheck) { - return func(check *codersdk.AuthorizationCheck) { - check.Action = action - } -} - -func randAction() string { - // nolint:gosec - return codersdk.AllRBACActions[rand.Intn(len(codersdk.AllRBACActions))] -} - -func randObjectType() codersdk.RBACResource { - // nolint:gosec - return codersdk.AllRBACResources[rand.Intn(len(codersdk.AllRBACResources))] -} From 5d1739709766e26a286f665f584c9a9acc814752 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 26 Jul 2023 11:37:13 +0100 Subject: [PATCH 19/29] fixup! add cache and lots more actions --- scaletest/dashboard/rolltable.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scaletest/dashboard/rolltable.go b/scaletest/dashboard/rolltable.go index 2871df5c3afcb..1cd49dc0e9ea5 100644 --- a/scaletest/dashboard/rolltable.go +++ b/scaletest/dashboard/rolltable.go @@ -190,10 +190,18 @@ func fetchWorkspaceLogs(ctx context.Context, p *params) error { defer func() { _ = closer.Close() }() - for range ch { - // do nothing + // Drain the channel. + for { + select { + case <-ctx.Done(): + return nil + case l, ok := <-ch: + if !ok { + return nil + } + _ = l + } } - return err } // fetchAuditLog fetches the audit log. From e5f0bbbc205928855ac9ce71477593ec82109965 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 26 Jul 2023 13:22:31 +0100 Subject: [PATCH 20/29] add metrics --- cli/exp_scaletest.go | 65 ++++++++++++++++++++++------------ cli/exp_scaletest_test.go | 2 ++ scaletest/dashboard/metrics.go | 34 ++++++++++++++++++ scaletest/dashboard/run.go | 20 ++++++++--- 4 files changed, 93 insertions(+), 28 deletions(-) create mode 100644 scaletest/dashboard/metrics.go diff --git a/cli/exp_scaletest.go b/cli/exp_scaletest.go index 38418546b186e..60f6d9e25e995 100644 --- a/cli/exp_scaletest.go +++ b/cli/exp_scaletest.go @@ -319,6 +319,30 @@ func (s *scaletestOutputFlags) parse() ([]scaleTestOutput, error) { return out, nil } +type scaletestPrometheusFlags struct { + Address string + Wait time.Duration +} + +func (s *scaletestPrometheusFlags) attach(opts *clibase.OptionSet) { + *opts = append(*opts, + clibase.Option{ + Flag: "scaletest-prometheus-address", + Env: "CODER_SCALETEST_PROMETHEUS_ADDRESS", + Default: "0.0.0.0:21112", + Description: "Address on which to expose scaletest Prometheus metrics.", + Value: clibase.StringOf(&s.Address), + }, + clibase.Option{ + Flag: "scaletest-prometheus-wait", + Env: "CODER_SCALETEST_PROMETHEUS_WAIT", + Default: "5s", + Description: "How long to wait before exiting in order to allow Prometheus metrics to be scraped.", + Value: clibase.DurationOf(&s.Wait), + }, + ) +} + func requireAdmin(ctx context.Context, client *codersdk.Client) (codersdk.User, error) { me, err := client.User(ctx, codersdk.Me) if err != nil { @@ -848,17 +872,16 @@ func (r *RootCmd) scaletestCreateWorkspaces() *clibase.Cmd { func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd { var ( - tickInterval time.Duration - bytesPerTick int64 - ssh bool - scaletestPrometheusAddress string - scaletestPrometheusWait time.Duration + tickInterval time.Duration + bytesPerTick int64 + ssh bool client = &codersdk.Client{} tracingFlags = &scaletestTracingFlags{} strategy = &scaletestStrategyFlags{} cleanupStrategy = &scaletestStrategyFlags{cleanup: true} output = &scaletestOutputFlags{} + prometheusFlags = &scaletestPrometheusFlags{} ) cmd := &clibase.Cmd{ @@ -873,7 +896,7 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd { metrics := workspacetraffic.NewMetrics(reg, "username", "workspace_name", "agent_name") logger := slog.Make(sloghuman.Sink(io.Discard)) - prometheusSrvClose := ServeHandler(ctx, logger, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), scaletestPrometheusAddress, "prometheus") + prometheusSrvClose := ServeHandler(ctx, logger, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), prometheusFlags.Address, "prometheus") defer prometheusSrvClose() // Bypass rate limiting @@ -907,8 +930,8 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd { _, _ = fmt.Fprintf(inv.Stderr, "\nError uploading traces: %+v\n", err) } // Wait for prometheus metrics to be scraped - _, _ = fmt.Fprintf(inv.Stderr, "Waiting %s for prometheus metrics to be scraped\n", scaletestPrometheusWait) - <-time.After(scaletestPrometheusWait) + _, _ = fmt.Fprintf(inv.Stderr, "Waiting %s for prometheus metrics to be scraped\n", prometheusFlags.Wait) + <-time.After(prometheusFlags.Wait) }() tracer := tracerProvider.Tracer(scaletestTracerName) @@ -1011,26 +1034,13 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd { Description: "Send traffic over SSH.", Value: clibase.BoolOf(&ssh), }, - { - Flag: "scaletest-prometheus-address", - Env: "CODER_SCALETEST_PROMETHEUS_ADDRESS", - Default: "0.0.0.0:21112", - Description: "Address on which to expose scaletest Prometheus metrics.", - Value: clibase.StringOf(&scaletestPrometheusAddress), - }, - { - Flag: "scaletest-prometheus-wait", - Env: "CODER_SCALETEST_PROMETHEUS_WAIT", - Default: "5s", - Description: "How long to wait before exiting in order to allow Prometheus metrics to be scraped.", - Value: clibase.DurationOf(&scaletestPrometheusWait), - }, } tracingFlags.attach(&cmd.Options) strategy.attach(&cmd.Options) cleanupStrategy.attach(&cmd.Options) output.attach(&cmd.Options) + prometheusFlags.attach(&cmd.Options) return cmd } @@ -1046,6 +1056,7 @@ func (r *RootCmd) scaletestDashboard() *clibase.Cmd { strategy = &scaletestStrategyFlags{} cleanupStrategy = &scaletestStrategyFlags{cleanup: true} output = &scaletestOutputFlags{} + prometheusFlags = &scaletestPrometheusFlags{} ) cmd := &clibase.Cmd{ @@ -1068,12 +1079,19 @@ func (r *RootCmd) scaletestDashboard() *clibase.Cmd { if err := closeTracing(ctx); err != nil { _, _ = fmt.Fprintf(inv.Stderr, "\nError uploading traces: %+v\n", err) } + // Wait for prometheus metrics to be scraped + _, _ = fmt.Fprintf(inv.Stderr, "Waiting %s for prometheus metrics to be scraped\n", prometheusFlags.Wait) + <-time.After(prometheusFlags.Wait) }() tracer := tracerProvider.Tracer(scaletestTracerName) outputs, err := output.parse() if err != nil { return xerrors.Errorf("could not parse --output flags") } + reg := prometheus.NewRegistry() + prometheusSrvClose := ServeHandler(ctx, logger, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), prometheusFlags.Address, "prometheus") + defer prometheusSrvClose() + metrics := dashboard.NewMetrics(reg) th := harness.NewTestHarness(strategy.toStrategy(), cleanupStrategy.toStrategy()) @@ -1088,7 +1106,7 @@ func (r *RootCmd) scaletestDashboard() *clibase.Cmd { if err := config.Validate(); err != nil { return err } - var runner harness.Runnable = dashboard.NewRunner(client, config) + var runner harness.Runnable = dashboard.NewRunner(client, metrics, config) if tracingEnabled { runner = &runnableTraceWrapper{ tracer: tracer, @@ -1151,6 +1169,7 @@ func (r *RootCmd) scaletestDashboard() *clibase.Cmd { strategy.attach(&cmd.Options) cleanupStrategy.attach(&cmd.Options) output.attach(&cmd.Options) + prometheusFlags.attach(&cmd.Options) return cmd } diff --git a/cli/exp_scaletest_test.go b/cli/exp_scaletest_test.go index 00e344f15c3d6..4c10b722ca357 100644 --- a/cli/exp_scaletest_test.go +++ b/cli/exp_scaletest_test.go @@ -94,6 +94,8 @@ func TestScaleTestDashboard(t *testing.T) { "--min-wait", "100ms", "--max-wait", "1s", "--timeout", "1s", + "--scaletest-prometheus-address", "127.0.0.1:0", + "--scaletest-prometheus-wait", "0s", ) clitest.SetupConfig(t, client, root) var stdout, stderr bytes.Buffer diff --git a/scaletest/dashboard/metrics.go b/scaletest/dashboard/metrics.go new file mode 100644 index 0000000000000..b89896c50d3a3 --- /dev/null +++ b/scaletest/dashboard/metrics.go @@ -0,0 +1,34 @@ +package dashboard + +import "github.com/prometheus/client_golang/prometheus" + +type Metrics struct { + DurationSeconds *prometheus.HistogramVec + Errors *prometheus.CounterVec + Statuses *prometheus.CounterVec +} + +func NewMetrics(reg prometheus.Registerer) *Metrics { + m := &Metrics{ + DurationSeconds: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "coderd", + Subsystem: "scaletest_dashboard", + Name: "duration_seconds", + }, []string{"action"}), + Errors: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "coderd", + Subsystem: "scaletest_dashboard", + Name: "errors_total", + }, []string{"action"}), + Statuses: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "coderd", + Subsystem: "scaletest_dashboard", + Name: "statuses_total", + }, []string{"action", "code"}), + } + + reg.MustRegister(m.DurationSeconds) + reg.MustRegister(m.Errors) + reg.MustRegister(m.Statuses) + return m +} diff --git a/scaletest/dashboard/run.go b/scaletest/dashboard/run.go index 1de7547a07589..b6efe6054cc18 100644 --- a/scaletest/dashboard/run.go +++ b/scaletest/dashboard/run.go @@ -2,6 +2,7 @@ package dashboard import ( "context" + "fmt" "io" "math/rand" "time" @@ -14,8 +15,9 @@ import ( ) type Runner struct { - client *codersdk.Client - cfg Config + client *codersdk.Client + cfg Config + metrics *Metrics `json:"-"` } var ( @@ -23,11 +25,12 @@ var ( _ harness.Cleanable = &Runner{} ) -func NewRunner(client *codersdk.Client, cfg Config) *Runner { +func NewRunner(client *codersdk.Client, metrics *Metrics, cfg Config) *Runner { client.Trace = cfg.Trace return &Runner{ - client: client, - cfg: cfg, + client: client, + cfg: cfg, + metrics: metrics, } } @@ -102,6 +105,13 @@ func (r *Runner) do(ctx context.Context, act rollTableEntry, p *params) { slog.F("elapsed", elapsed), ) } + codeLabel := "200" + if apiErr, ok := codersdk.AsError(err); ok { + codeLabel = fmt.Sprintf("%d", apiErr.StatusCode()) + r.metrics.Errors.WithLabelValues(act.label).Add(1) + } + r.metrics.DurationSeconds.WithLabelValues(act.label).Observe(elapsed.Seconds()) + r.metrics.Statuses.WithLabelValues(act.label, codeLabel).Add(1) } } From a3b75cab6532ed90e5dbe4ef771831a9164676e5 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 26 Jul 2023 13:35:01 +0100 Subject: [PATCH 21/29] fixup! add metrics --- scaletest/dashboard/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scaletest/dashboard/run.go b/scaletest/dashboard/run.go index b6efe6054cc18..a71423892f022 100644 --- a/scaletest/dashboard/run.go +++ b/scaletest/dashboard/run.go @@ -17,7 +17,7 @@ import ( type Runner struct { client *codersdk.Client cfg Config - metrics *Metrics `json:"-"` + metrics *Metrics } var ( From 60b5a76c5e6d931b532d7028027201f5e7fa40dc Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 26 Jul 2023 16:23:39 +0100 Subject: [PATCH 22/29] improve metrics errors --- scaletest/dashboard/rolltable.go | 2 +- scaletest/dashboard/run.go | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/scaletest/dashboard/rolltable.go b/scaletest/dashboard/rolltable.go index 1cd49dc0e9ea5..8aa7416895089 100644 --- a/scaletest/dashboard/rolltable.go +++ b/scaletest/dashboard/rolltable.go @@ -194,7 +194,7 @@ func fetchWorkspaceLogs(ctx context.Context, p *params) error { for { select { case <-ctx.Done(): - return nil + return ctx.Err() case l, ok := <-ch: if !ok { return nil diff --git a/scaletest/dashboard/run.go b/scaletest/dashboard/run.go index a71423892f022..a4971aaec9ba2 100644 --- a/scaletest/dashboard/run.go +++ b/scaletest/dashboard/run.go @@ -89,10 +89,15 @@ func (r *Runner) do(ctx context.Context, act rollTableEntry, p *params) { r.cfg.Logger.Info(ctx, "context done, stopping") return default: + var errored bool + cancelCtx, cancel := context.WithTimeout(ctx, r.cfg.MaxWait) + defer cancel() start := time.Now() - err := act.fn(ctx, p) + err := act.fn(cancelCtx, p) + cancel() elapsed := time.Since(start) if err != nil { + errored = true r.cfg.Logger.Error( //nolint:gocritic ctx, "action failed", slog.Error(err), @@ -109,9 +114,14 @@ func (r *Runner) do(ctx context.Context, act rollTableEntry, p *params) { if apiErr, ok := codersdk.AsError(err); ok { codeLabel = fmt.Sprintf("%d", apiErr.StatusCode()) r.metrics.Errors.WithLabelValues(act.label).Add(1) + } else if xerrors.Is(err, context.Canceled) { + codeLabel = "timeout" } r.metrics.DurationSeconds.WithLabelValues(act.label).Observe(elapsed.Seconds()) r.metrics.Statuses.WithLabelValues(act.label, codeLabel).Add(1) + if errored { + r.metrics.Errors.WithLabelValues(act.label).Add(1) + } } } From 2674b2911ace7d60e570d279fc1dea2945c8a98a Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 26 Jul 2023 17:42:06 +0100 Subject: [PATCH 23/29] allow configuring actions --- cli/exp_scaletest.go | 9 +++++---- scaletest/dashboard/config.go | 2 ++ scaletest/dashboard/rolltable.go | 12 ++++++------ scaletest/dashboard/rolltable_internal_test.go | 2 +- scaletest/dashboard/run.go | 4 ++-- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/cli/exp_scaletest.go b/cli/exp_scaletest.go index 60f6d9e25e995..d8c7d2796aade 100644 --- a/cli/exp_scaletest.go +++ b/cli/exp_scaletest.go @@ -1098,10 +1098,11 @@ func (r *RootCmd) scaletestDashboard() *clibase.Cmd { for i := int64(0); i < count; i++ { name := fmt.Sprintf("dashboard-%d", i) config := dashboard.Config{ - MinWait: minWait, - MaxWait: maxWait, - Trace: tracingEnabled, - Logger: logger.Named(name), + MinWait: minWait, + MaxWait: maxWait, + Trace: tracingEnabled, + Logger: logger.Named(name), + RollTable: dashboard.DefaultActions, } if err := config.Validate(); err != nil { return err diff --git a/scaletest/dashboard/config.go b/scaletest/dashboard/config.go index 83232cf0667f4..b269ee7119320 100644 --- a/scaletest/dashboard/config.go +++ b/scaletest/dashboard/config.go @@ -17,6 +17,8 @@ type Config struct { Trace bool `json:"trace"` // Logger is the logger to use. Logger slog.Logger `json:"-"` + // RollTable is the set of actions to perform + RollTable RollTable `json:"roll_table"` } func (c Config) Validate() error { diff --git a/scaletest/dashboard/rolltable.go b/scaletest/dashboard/rolltable.go index 8aa7416895089..932990c1cea45 100644 --- a/scaletest/dashboard/rolltable.go +++ b/scaletest/dashboard/rolltable.go @@ -8,11 +8,11 @@ import ( "github.com/coder/coder/codersdk" ) -// allActions is a table of actions to perform. +// DefaultActions is a table of actions to perform. // D&D nerds will feel right at home here :-) // Note that the order of the table is important! // Entries must be in ascending order. -var allActions rollTable = []rollTableEntry{ +var DefaultActions RollTable = []rollTableEntry{ {0, fetchWorkspaces, "fetch workspaces"}, {1, fetchUsers, "fetch users"}, {2, fetchTemplates, "fetch templates"}, @@ -33,8 +33,8 @@ var allActions rollTable = []rollTableEntry{ {17, fetchWorkspaceLogs, "fetch workspace logs"}, } -// rollTable is a slice of rollTableEntry. -type rollTable []rollTableEntry +// RollTable is a slice of rollTableEntry. +type RollTable []rollTableEntry // rollTableEntry is an entry in the roll table. type rollTableEntry struct { @@ -47,7 +47,7 @@ type rollTableEntry struct { } // choose returns the first entry in the table that is greater than or equal to n. -func (r rollTable) choose(n int) rollTableEntry { +func (r RollTable) choose(n int) rollTableEntry { for _, entry := range r { if entry.roll >= n { return entry @@ -58,7 +58,7 @@ func (r rollTable) choose(n int) rollTableEntry { // max returns the maximum roll in the table. // Important: this assumes that the table is sorted in ascending order. -func (r rollTable) max() int { +func (r RollTable) max() int { return r[len(r)-1].roll } diff --git a/scaletest/dashboard/rolltable_internal_test.go b/scaletest/dashboard/rolltable_internal_test.go index 00be8912b90c8..883bc23a5e131 100644 --- a/scaletest/dashboard/rolltable_internal_test.go +++ b/scaletest/dashboard/rolltable_internal_test.go @@ -10,7 +10,7 @@ func Test_allActions_ordering(t *testing.T) { t.Parallel() last := -1 - for idx, entry := range allActions { + for idx, entry := range DefaultActions { require.Greater(t, entry.roll, last, "roll table must be in ascending order, entry %d is out of order", idx) last = entry.roll } diff --git a/scaletest/dashboard/run.go b/scaletest/dashboard/run.go index a4971aaec9ba2..795a1a179aba5 100644 --- a/scaletest/dashboard/run.go +++ b/scaletest/dashboard/run.go @@ -62,7 +62,7 @@ func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error { case <-ctx.Done(): return case <-t.C: - rolls <- rand.Intn(allActions.max() + 1) // nolint:gosec + rolls <- rand.Intn(r.cfg.RollTable.max() + 1) // nolint:gosec t.Reset(r.randWait()) } } @@ -73,7 +73,7 @@ func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error { case <-ctx.Done(): return nil case n := <-rolls: - act := allActions.choose(n) + act := r.cfg.RollTable.choose(n) go r.do(ctx, act, p) } } From 471f021cc125b3a53efb90b9128260b0a6962764 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 26 Jul 2023 18:18:25 +0100 Subject: [PATCH 24/29] add unit test --- scaletest/dashboard/metrics.go | 48 +++++-- scaletest/dashboard/rolltable.go | 70 +++++----- .../dashboard/rolltable_internal_test.go | 4 +- scaletest/dashboard/run.go | 21 ++- scaletest/dashboard/run_test.go | 120 ++++++++++++++++++ 5 files changed, 202 insertions(+), 61 deletions(-) create mode 100644 scaletest/dashboard/run_test.go diff --git a/scaletest/dashboard/metrics.go b/scaletest/dashboard/metrics.go index b89896c50d3a3..513a319a07bae 100644 --- a/scaletest/dashboard/metrics.go +++ b/scaletest/dashboard/metrics.go @@ -1,34 +1,56 @@ package dashboard -import "github.com/prometheus/client_golang/prometheus" +import ( + "time" -type Metrics struct { - DurationSeconds *prometheus.HistogramVec - Errors *prometheus.CounterVec - Statuses *prometheus.CounterVec + "github.com/prometheus/client_golang/prometheus" +) + +type Metrics interface { + ObserveDuration(action string, d time.Duration) + IncErrors(action string) + IncStatuses(action string, code string) +} + +type PromMetrics struct { + durationSeconds *prometheus.HistogramVec + errors *prometheus.CounterVec + statuses *prometheus.CounterVec } -func NewMetrics(reg prometheus.Registerer) *Metrics { - m := &Metrics{ - DurationSeconds: prometheus.NewHistogramVec(prometheus.HistogramOpts{ +func NewMetrics(reg prometheus.Registerer) *PromMetrics { + m := &PromMetrics{ + durationSeconds: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "coderd", Subsystem: "scaletest_dashboard", Name: "duration_seconds", }, []string{"action"}), - Errors: prometheus.NewCounterVec(prometheus.CounterOpts{ + errors: prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "coderd", Subsystem: "scaletest_dashboard", Name: "errors_total", }, []string{"action"}), - Statuses: prometheus.NewCounterVec(prometheus.CounterOpts{ + statuses: prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "coderd", Subsystem: "scaletest_dashboard", Name: "statuses_total", }, []string{"action", "code"}), } - reg.MustRegister(m.DurationSeconds) - reg.MustRegister(m.Errors) - reg.MustRegister(m.Statuses) + reg.MustRegister(m.durationSeconds) + reg.MustRegister(m.errors) + reg.MustRegister(m.statuses) return m } + +func (p *PromMetrics) ObserveDuration(action string, d time.Duration) { + p.durationSeconds.WithLabelValues(action).Observe(d.Seconds()) +} + +func (p *PromMetrics) IncErrors(action string) { + p.errors.WithLabelValues(action).Inc() +} + +func (p *PromMetrics) IncStatuses(action string, code string) { + p.statuses.WithLabelValues(action, code).Inc() +} diff --git a/scaletest/dashboard/rolltable.go b/scaletest/dashboard/rolltable.go index 932990c1cea45..e237cf6983878 100644 --- a/scaletest/dashboard/rolltable.go +++ b/scaletest/dashboard/rolltable.go @@ -12,7 +12,7 @@ import ( // D&D nerds will feel right at home here :-) // Note that the order of the table is important! // Entries must be in ascending order. -var DefaultActions RollTable = []rollTableEntry{ +var DefaultActions RollTable = []RollTableEntry{ {0, fetchWorkspaces, "fetch workspaces"}, {1, fetchUsers, "fetch users"}, {2, fetchTemplates, "fetch templates"}, @@ -34,36 +34,36 @@ var DefaultActions RollTable = []rollTableEntry{ } // RollTable is a slice of rollTableEntry. -type RollTable []rollTableEntry - -// rollTableEntry is an entry in the roll table. -type rollTableEntry struct { - // roll is the minimum number required to perform the action. - roll int - // fn is the function to call. - fn func(ctx context.Context, p *params) error - // label is used for logging. - label string +type RollTable []RollTableEntry + +// RollTableEntry is an entry in the roll table. +type RollTableEntry struct { + // Roll is the minimum number required to perform the action. + Roll int + // Fn is the function to call. + Fn func(ctx context.Context, p *Params) error + // Label is used for logging. + Label string } // choose returns the first entry in the table that is greater than or equal to n. -func (r RollTable) choose(n int) rollTableEntry { +func (r RollTable) choose(n int) RollTableEntry { for _, entry := range r { - if entry.roll >= n { + if entry.Roll >= n { return entry } } - return rollTableEntry{} + return RollTableEntry{} } // max returns the maximum roll in the table. // Important: this assumes that the table is sorted in ascending order. func (r RollTable) max() int { - return r[len(r)-1].roll + return r[len(r)-1].Roll } -// params is a set of parameters to pass to the actions in a rollTable. -type params struct { +// Params is a set of parameters to pass to the actions in a rollTable. +type Params struct { // client is the client to use for performing the action. client *codersdk.Client // me is the currently authenticated user. Lots of actions require this. @@ -76,7 +76,7 @@ type params struct { } // fetchWorkspaces fetches all workspaces. -func fetchWorkspaces(ctx context.Context, p *params) error { +func fetchWorkspaces(ctx context.Context, p *Params) error { ws, err := p.client.Workspaces(ctx, codersdk.WorkspaceFilter{}) if err != nil { // store the workspaces for later use in case they change @@ -86,7 +86,7 @@ func fetchWorkspaces(ctx context.Context, p *params) error { } // fetchUsers fetches all users. -func fetchUsers(ctx context.Context, p *params) error { +func fetchUsers(ctx context.Context, p *Params) error { users, err := p.client.Users(ctx, codersdk.UsersRequest{}) if err != nil { p.c.setUsers(users.Users) @@ -95,7 +95,7 @@ func fetchUsers(ctx context.Context, p *params) error { } // fetchActiveUsers fetches all active users -func fetchActiveUsers(ctx context.Context, p *params) error { +func fetchActiveUsers(ctx context.Context, p *Params) error { _, err := p.client.Users(ctx, codersdk.UsersRequest{ Status: codersdk.UserStatusActive, }) @@ -103,7 +103,7 @@ func fetchActiveUsers(ctx context.Context, p *params) error { } // fetchSuspendedUsers fetches all suspended users -func fetchSuspendedUsers(ctx context.Context, p *params) error { +func fetchSuspendedUsers(ctx context.Context, p *Params) error { _, err := p.client.Users(ctx, codersdk.UsersRequest{ Status: codersdk.UserStatusSuspended, }) @@ -111,7 +111,7 @@ func fetchSuspendedUsers(ctx context.Context, p *params) error { } // fetchTemplates fetches all templates. -func fetchTemplates(ctx context.Context, p *params) error { +func fetchTemplates(ctx context.Context, p *Params) error { templates, err := p.client.TemplatesByOrganization(ctx, p.me.OrganizationIDs[0]) if err != nil { p.c.setTemplates(templates) @@ -120,68 +120,68 @@ func fetchTemplates(ctx context.Context, p *params) error { } // fetchTemplateBuild fetches a single template version at random. -func fetchTemplateVersion(ctx context.Context, p *params) error { +func fetchTemplateVersion(ctx context.Context, p *Params) error { t := p.c.randTemplate() _, err := p.client.TemplateVersion(ctx, t.ActiveVersionID) return err } // fetchWorkspace fetches a single workspace at random. -func fetchWorkspace(ctx context.Context, p *params) error { +func fetchWorkspace(ctx context.Context, p *Params) error { w := p.c.randWorkspace() _, err := p.client.WorkspaceByOwnerAndName(ctx, w.OwnerName, w.Name, codersdk.WorkspaceOptions{}) return err } // fetchWorkspaceBuild fetches a single workspace build at random. -func fetchWorkspaceBuild(ctx context.Context, p *params) error { +func fetchWorkspaceBuild(ctx context.Context, p *Params) error { w := p.c.randWorkspace() _, err := p.client.WorkspaceBuild(ctx, w.LatestBuild.ID) return err } // fetchTemplate fetches a single template at random. -func fetchTemplate(ctx context.Context, p *params) error { +func fetchTemplate(ctx context.Context, p *Params) error { t := p.c.randTemplate() _, err := p.client.Template(ctx, t.ID) return err } // fetchUserByID fetches a single user at random by ID. -func fetchUserByID(ctx context.Context, p *params) error { +func fetchUserByID(ctx context.Context, p *Params) error { u := p.c.randUser() _, err := p.client.User(ctx, u.ID.String()) return err } // fetchUserByUsername fetches a single user at random by username. -func fetchUserByUsername(ctx context.Context, p *params) error { +func fetchUserByUsername(ctx context.Context, p *Params) error { u := p.c.randUser() _, err := p.client.User(ctx, u.Username) return err } // fetchDeploymentConfig fetches the deployment config. -func fetchDeploymentConfig(ctx context.Context, p *params) error { +func fetchDeploymentConfig(ctx context.Context, p *Params) error { _, err := p.client.DeploymentConfig(ctx) return err } // fetchWorkspaceQuotaForUser fetches the workspace quota for a random user. -func fetchWorkspaceQuotaForUser(ctx context.Context, p *params) error { +func fetchWorkspaceQuotaForUser(ctx context.Context, p *Params) error { u := p.c.randUser() _, err := p.client.WorkspaceQuota(ctx, u.ID.String()) return err } // fetchDeploymentStats fetches the deployment stats. -func fetchDeploymentStats(ctx context.Context, p *params) error { +func fetchDeploymentStats(ctx context.Context, p *Params) error { _, err := p.client.DeploymentStats(ctx) return err } // fetchWorkspaceLogs fetches the logs for a random workspace. -func fetchWorkspaceLogs(ctx context.Context, p *params) error { +func fetchWorkspaceLogs(ctx context.Context, p *Params) error { w := p.c.randWorkspace() ch, closer, err := p.client.WorkspaceBuildLogsAfter(ctx, w.LatestBuild.ID, 0) if err != nil { @@ -206,7 +206,7 @@ func fetchWorkspaceLogs(ctx context.Context, p *params) error { // fetchAuditLog fetches the audit log. // As not all users have access to the audit log, we check first. -func fetchAuditLog(ctx context.Context, p *params) error { +func fetchAuditLog(ctx context.Context, p *Params) error { res, err := p.client.AuthCheck(ctx, codersdk.AuthorizationRequest{ Checks: map[string]codersdk.AuthorizationCheck{ "auditlog": { @@ -236,7 +236,7 @@ func fetchAuditLog(ctx context.Context, p *params) error { // authCheckAsOwner performs an auth check as the owner of a random // resource type and action. -func authCheckAsOwner(ctx context.Context, p *params) error { +func authCheckAsOwner(ctx context.Context, p *Params) error { _, err := p.client.AuthCheck(ctx, randAuthReq( ownedBy(p.me.ID), withAction(randAction()), @@ -248,7 +248,7 @@ func authCheckAsOwner(ctx context.Context, p *params) error { // authCheckAsNonOwner performs an auth check as a non-owner of a random // resource type and action. -func authCheckAsNonOwner(ctx context.Context, p *params) error { +func authCheckAsNonOwner(ctx context.Context, p *Params) error { _, err := p.client.AuthCheck(ctx, randAuthReq( ownedBy(uuid.New()), withAction(randAction()), diff --git a/scaletest/dashboard/rolltable_internal_test.go b/scaletest/dashboard/rolltable_internal_test.go index 883bc23a5e131..53b646df119d6 100644 --- a/scaletest/dashboard/rolltable_internal_test.go +++ b/scaletest/dashboard/rolltable_internal_test.go @@ -11,7 +11,7 @@ func Test_allActions_ordering(t *testing.T) { last := -1 for idx, entry := range DefaultActions { - require.Greater(t, entry.roll, last, "roll table must be in ascending order, entry %d is out of order", idx) - last = entry.roll + require.Greater(t, entry.Roll, last, "roll table must be in ascending order, entry %d is out of order", idx) + last = entry.Roll } } diff --git a/scaletest/dashboard/run.go b/scaletest/dashboard/run.go index 795a1a179aba5..64c904177565e 100644 --- a/scaletest/dashboard/run.go +++ b/scaletest/dashboard/run.go @@ -17,7 +17,7 @@ import ( type Runner struct { client *codersdk.Client cfg Config - metrics *Metrics + metrics Metrics } var ( @@ -25,7 +25,7 @@ var ( _ harness.Cleanable = &Runner{} ) -func NewRunner(client *codersdk.Client, metrics *Metrics, cfg Config) *Runner { +func NewRunner(client *codersdk.Client, metrics Metrics, cfg Config) *Runner { client.Trace = cfg.Trace return &Runner{ client: client, @@ -48,7 +48,7 @@ func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error { return err } - p := ¶ms{ + p := &Params{ client: r.client, me: me, c: c, @@ -83,7 +83,7 @@ func (*Runner) Cleanup(_ context.Context, _ string) error { return nil } -func (r *Runner) do(ctx context.Context, act rollTableEntry, p *params) { +func (r *Runner) do(ctx context.Context, act RollTableEntry, p *Params) { select { case <-ctx.Done(): r.cfg.Logger.Info(ctx, "context done, stopping") @@ -93,7 +93,7 @@ func (r *Runner) do(ctx context.Context, act rollTableEntry, p *params) { cancelCtx, cancel := context.WithTimeout(ctx, r.cfg.MaxWait) defer cancel() start := time.Now() - err := act.fn(cancelCtx, p) + err := act.Fn(cancelCtx, p) cancel() elapsed := time.Since(start) if err != nil { @@ -101,26 +101,25 @@ func (r *Runner) do(ctx context.Context, act rollTableEntry, p *params) { r.cfg.Logger.Error( //nolint:gocritic ctx, "action failed", slog.Error(err), - slog.F("action", act.label), + slog.F("action", act.Label), slog.F("elapsed", elapsed), ) } else { r.cfg.Logger.Info(ctx, "completed successfully", - slog.F("action", act.label), + slog.F("action", act.Label), slog.F("elapsed", elapsed), ) } codeLabel := "200" if apiErr, ok := codersdk.AsError(err); ok { codeLabel = fmt.Sprintf("%d", apiErr.StatusCode()) - r.metrics.Errors.WithLabelValues(act.label).Add(1) } else if xerrors.Is(err, context.Canceled) { codeLabel = "timeout" } - r.metrics.DurationSeconds.WithLabelValues(act.label).Observe(elapsed.Seconds()) - r.metrics.Statuses.WithLabelValues(act.label, codeLabel).Add(1) + r.metrics.ObserveDuration(act.Label, elapsed) + r.metrics.IncStatuses(act.Label, codeLabel) if errored { - r.metrics.Errors.WithLabelValues(act.label).Add(1) + r.metrics.IncErrors(act.Label) } } } diff --git a/scaletest/dashboard/run_test.go b/scaletest/dashboard/run_test.go new file mode 100644 index 0000000000000..1cb24d9664909 --- /dev/null +++ b/scaletest/dashboard/run_test.go @@ -0,0 +1,120 @@ +package dashboard_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/xerrors" + + "cdr.dev/slog/sloggers/slogtest" + "github.com/coder/coder/coderd/coderdtest" + "github.com/coder/coder/scaletest/dashboard" + "github.com/coder/coder/testutil" +) + +func Test_Run(t *testing.T) { + t.Parallel() + if testutil.RaceEnabled() { + t.Skip("skipping timing-sensitive test because of race detector") + } + + client := coderdtest.New(t, nil) + _ = coderdtest.CreateFirstUser(t, client) + + successfulAction := func(context.Context, *dashboard.Params) error { + return nil + } + failingAction := func(context.Context, *dashboard.Params) error { + return xerrors.Errorf("failed") + } + hangingAction := func(ctx context.Context, _ *dashboard.Params) error { + <-ctx.Done(): + return ctx.Err() + } + + testActions := []dashboard.RollTableEntry{ + {0, successfulAction, "succeeds"}, + {1, failingAction, "fails"}, + {2, hangingAction, "hangs"}, + } + + log := slogtest.Make(t, &slogtest.Options{ + IgnoreErrors: true, + }) + m := &testMetrics{} + r := dashboard.NewRunner(client, m, dashboard.Config{ + MinWait: time.Millisecond, + MaxWait: 100 * time.Millisecond, + Logger: log, + RollTable: testActions, + }) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) + t.Cleanup(cancel) + done := make(chan error) + go func() { + defer close(done) + done <- r.Run(ctx, "", nil) + }() + err, ok := <-done + assert.True(t, ok) + require.NoError(t, err) + + if assert.NotEmpty(t, m.ObservedDurations["succeeds"]) { + assert.NotZero(t, m.ObservedDurations["succeeds"][0]) + } + + if assert.NotEmpty(t, m.ObservedDurations["fails"]) { + assert.NotZero(t, m.ObservedDurations["fails"][0]) + } + + if assert.NotEmpty(t, m.ObservedDurations["hangs"]) { + assert.NotZero(t, m.ObservedDurations["hangs"][0]) + } + assert.Zero(t, m.Errors["succeeds"]) + assert.NotZero(t, m.Errors["fails"]) + assert.NotZero(t, m.Errors["hangs"]) + assert.NotEmpty(t, m.Statuses["succeeds"]) + assert.NotEmpty(t, m.Statuses["fails"]) + assert.NotEmpty(t, m.Statuses["hangs"]) +} + +type testMetrics struct { + sync.RWMutex + ObservedDurations map[string][]float64 + Errors map[string]int + Statuses map[string]map[string]int +} + +func (m *testMetrics) ObserveDuration(action string, d time.Duration) { + m.Lock() + defer m.Unlock() + if m.ObservedDurations == nil { + m.ObservedDurations = make(map[string][]float64) + } + m.ObservedDurations[action] = append(m.ObservedDurations[action], d.Seconds()) +} + +func (m *testMetrics) IncErrors(action string) { + m.Lock() + defer m.Unlock() + if m.Errors == nil { + m.Errors = make(map[string]int) + } + m.Errors[action]++ +} + +func (m *testMetrics) IncStatuses(action string, code string) { + m.Lock() + defer m.Unlock() + if m.Statuses == nil { + m.Statuses = make(map[string]map[string]int) + } + if m.Statuses[action] == nil { + m.Statuses[action] = make(map[string]int) + } + m.Statuses[action][code]++ +} From 98b63967c1fd883820443b6168e82d3a2ca3693a Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 26 Jul 2023 20:43:26 +0100 Subject: [PATCH 25/29] fixup! add unit test --- scaletest/dashboard/run_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scaletest/dashboard/run_test.go b/scaletest/dashboard/run_test.go index 1cb24d9664909..8df7e47acd277 100644 --- a/scaletest/dashboard/run_test.go +++ b/scaletest/dashboard/run_test.go @@ -32,7 +32,7 @@ func Test_Run(t *testing.T) { return xerrors.Errorf("failed") } hangingAction := func(ctx context.Context, _ *dashboard.Params) error { - <-ctx.Done(): + <-ctx.Done() return ctx.Err() } From 61c3a0698fe2b6cea33d17b6ff832bd21a4713ea Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 26 Jul 2023 21:02:10 +0100 Subject: [PATCH 26/29] check latency in test --- scaletest/dashboard/run_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scaletest/dashboard/run_test.go b/scaletest/dashboard/run_test.go index 8df7e47acd277..cdb27ed9ba130 100644 --- a/scaletest/dashboard/run_test.go +++ b/scaletest/dashboard/run_test.go @@ -46,13 +46,14 @@ func Test_Run(t *testing.T) { IgnoreErrors: true, }) m := &testMetrics{} - r := dashboard.NewRunner(client, m, dashboard.Config{ + cfg := dashboard.Config{ MinWait: time.Millisecond, - MaxWait: 100 * time.Millisecond, + MaxWait: 10 * time.Millisecond, Logger: log, RollTable: testActions, - }) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) + } + r := dashboard.NewRunner(client, m, cfg) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) t.Cleanup(cancel) done := make(chan error) go func() { @@ -72,7 +73,7 @@ func Test_Run(t *testing.T) { } if assert.NotEmpty(t, m.ObservedDurations["hangs"]) { - assert.NotZero(t, m.ObservedDurations["hangs"][0]) + assert.GreaterOrEqual(t, m.ObservedDurations["hangs"][0], cfg.MaxWait.Seconds()) } assert.Zero(t, m.Errors["succeeds"]) assert.NotZero(t, m.Errors["fails"]) From 925a446b66013caa36a5745be6ee284cd1ef948d Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 26 Jul 2023 21:13:23 +0100 Subject: [PATCH 27/29] appease linter --- scaletest/dashboard/run_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scaletest/dashboard/run_test.go b/scaletest/dashboard/run_test.go index cdb27ed9ba130..bd52080388425 100644 --- a/scaletest/dashboard/run_test.go +++ b/scaletest/dashboard/run_test.go @@ -53,7 +53,7 @@ func Test_Run(t *testing.T) { RollTable: testActions, } r := dashboard.NewRunner(client, m, cfg) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) t.Cleanup(cancel) done := make(chan error) go func() { From 8f4c1fc58607bbf829fda676fafe78203a942486 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 26 Jul 2023 21:13:34 +0100 Subject: [PATCH 28/29] bump scrape wait timeout --- cli/exp_scaletest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/exp_scaletest.go b/cli/exp_scaletest.go index d8c7d2796aade..d2ee36c1819eb 100644 --- a/cli/exp_scaletest.go +++ b/cli/exp_scaletest.go @@ -336,7 +336,7 @@ func (s *scaletestPrometheusFlags) attach(opts *clibase.OptionSet) { clibase.Option{ Flag: "scaletest-prometheus-wait", Env: "CODER_SCALETEST_PROMETHEUS_WAIT", - Default: "5s", + Default: "15s", Description: "How long to wait before exiting in order to allow Prometheus metrics to be scraped.", Value: clibase.DurationOf(&s.Wait), }, From f7fa1c6d567f0c4803acbcdb1a786af883248045 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 26 Jul 2023 21:25:52 +0100 Subject: [PATCH 29/29] skip test on windows --- scaletest/dashboard/run_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scaletest/dashboard/run_test.go b/scaletest/dashboard/run_test.go index bd52080388425..d522ba1a6ec88 100644 --- a/scaletest/dashboard/run_test.go +++ b/scaletest/dashboard/run_test.go @@ -2,6 +2,7 @@ package dashboard_test import ( "context" + "runtime" "sync" "testing" "time" @@ -21,6 +22,9 @@ func Test_Run(t *testing.T) { if testutil.RaceEnabled() { t.Skip("skipping timing-sensitive test because of race detector") } + if runtime.GOOS == "windows" { + t.Skip("skipping test on Windows") + } client := coderdtest.New(t, nil) _ = coderdtest.CreateFirstUser(t, client)