Skip to content

feat(scaletest/dashboard): integrate chromedp #9927

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Oct 2, 2023
Merged

feat(scaletest/dashboard): integrate chromedp #9927

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
a8fc106
fix(scaletest/harness): fix div-zero error
johnstcn Sep 28, 2023
d4ceee7
feat(scaletest/dashboard): add chromedp actions
johnstcn Sep 28, 2023
0ae372d
feat(cli): integrate chromedp actions into scaletest dashboard
johnstcn Sep 28, 2023
5c07be6
feat(scaletest/dashboard): remove unused code
johnstcn Sep 28, 2023
6aa0a52
rm unused flag
johnstcn Sep 29, 2023
41f7bb8
validate and fail-fast
johnstcn Sep 29, 2023
89bf0ee
fixup! validate and fail-fast
johnstcn Sep 29, 2023
48a319c
fixup! validate and fail-fast
johnstcn Sep 29, 2023
cca6a0b
log error on removing temp user data dir
johnstcn Sep 29, 2023
cf10151
use prng, add --rand-seed argument
johnstcn Sep 29, 2023
0805f2e
replace --min-wait and --max-wait with --interval and --jitter
johnstcn Sep 29, 2023
3232a92
Merge remote-tracking branch 'origin/main' into cj/scaletest-dashboar…
johnstcn Oct 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 46 additions & 20 deletions cli/exp_scaletest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1046,9 +1046,9 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {

func (r *RootCmd) scaletestDashboard() *clibase.Cmd {
var (
count int64
minWait time.Duration
maxWait time.Duration
minWait time.Duration
maxWait time.Duration
headless bool

client = &codersdk.Client{}
tracingFlags = &scaletestTracingFlags{}
Expand All @@ -1065,6 +1065,12 @@ func (r *RootCmd) scaletestDashboard() *clibase.Cmd {
r.InitClient(client),
),
Handler: func(inv *clibase.Invocation) error {
if !(minWait > 0) {
return xerrors.Errorf("--min-wait must be greater than zero")
}
if !(maxWait > minWait) {
return xerrors.Errorf("--max-wait must be greater than --min-wait")
}
ctx := inv.Context()
logger := slog.Make(sloghuman.Sink(inv.Stdout)).Leveled(slog.LevelInfo)
tracerProvider, closeTracing, tracingEnabled, err := tracingFlags.provider(ctx)
Expand Down Expand Up @@ -1094,19 +1100,39 @@ func (r *RootCmd) scaletestDashboard() *clibase.Cmd {

th := harness.NewTestHarness(strategy.toStrategy(), cleanupStrategy.toStrategy())

for i := int64(0); i < count; i++ {
name := fmt.Sprintf("dashboard-%d", i)
users, err := getScaletestUsers(ctx, client)
if err != nil {
return xerrors.Errorf("get scaletest users")
}

for _, usr := range users {
name := fmt.Sprintf("dashboard-%s", usr.Username)
userTokResp, err := client.CreateToken(ctx, usr.ID.String(), codersdk.CreateTokenRequest{
Lifetime: 30 * 24 * time.Hour,
Scope: "",
TokenName: fmt.Sprintf("scaletest-%d", time.Now().Unix()),
})
if err != nil {
return xerrors.Errorf("create token for user: %w", err)
}

userClient := codersdk.New(client.URL)
userClient.SetSessionToken(userTokResp.Key)

config := dashboard.Config{
MinWait: minWait,
MaxWait: maxWait,
Trace: tracingEnabled,
Logger: logger.Named(name),
RollTable: dashboard.DefaultActions,
MinWait: minWait,
MaxWait: maxWait,
Trace: tracingEnabled,
Logger: logger.Named(name),
Headless: headless,
ActionFunc: dashboard.ClickRandomElement,
}
//nolint:gocritic
logger.Info(ctx, "runner config", slog.F("min_wait", minWait), slog.F("max_wait", maxWait), slog.F("headless", headless), slog.F("trace", tracingEnabled))
if err := config.Validate(); err != nil {
return err
}
var runner harness.Runnable = dashboard.NewRunner(client, metrics, config)
var runner harness.Runnable = dashboard.NewRunner(userClient, metrics, config)
if tracingEnabled {
runner = &runnableTraceWrapper{
tracer: tracer,
Expand Down Expand Up @@ -1142,27 +1168,27 @@ func (r *RootCmd) scaletestDashboard() *clibase.Cmd {
}

cmd.Options = []clibase.Option{
{
Flag: "count",
Env: "CODER_SCALETEST_DASHBOARD_COUNT",
Default: "1",
Description: "Number of concurrent workers.",
Value: clibase.Int64Of(&count),
},
{
Flag: "min-wait",
Env: "CODER_SCALETEST_DASHBOARD_MIN_WAIT",
Default: "100ms",
Default: "1s",
Description: "Minimum wait between fetches.",
Value: clibase.DurationOf(&minWait),
},
{
Flag: "max-wait",
Env: "CODER_SCALETEST_DASHBOARD_MAX_WAIT",
Default: "1s",
Default: "10s",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any chance we may be able to eliminate these options? It'd be nice if you didn't need to adjust these parameters to have a working load test, and that it'd continue working even if coderd was under too much load, etc.

We could instead track user experience impact when things slow down to a crawl.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This more just controls the interval between actions. Between 1 and 10 seconds I think is a reasonable default. I'll rename it to better reflect its semantics.

Description: "Maximum wait between fetches.",
Value: clibase.DurationOf(&maxWait),
},
{
Flag: "headless",
Env: "CODER_SCALETEST_DASHBOARD_HEADLESS",
Default: "true",
Description: "Controls headless mode. Setting to false is useful for debugging.",
Value: clibase.BoolOf(&headless),
},
}

tracingFlags.attach(&cmd.Options)
Expand Down
91 changes: 70 additions & 21 deletions cli/exp_scaletest_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,28 +92,77 @@ func TestScaleTestWorkspaceTraffic(t *testing.T) {
// This test just validates that the CLI command accepts its known arguments.
func TestScaleTestDashboard(t *testing.T) {
t.Parallel()
ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitMedium)
defer cancelFunc()

log := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
client := coderdtest.New(t, &coderdtest.Options{
Logger: &log,
t.Run("MinWait", func(t *testing.T) {
t.Parallel()
ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitShort)
defer cancelFunc()

log := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
client := coderdtest.New(t, &coderdtest.Options{
Logger: &log,
})
_ = coderdtest.CreateFirstUser(t, client)

inv, root := clitest.New(t, "exp", "scaletest", "dashboard",
"--min-wait", "0s",
)
clitest.SetupConfig(t, client, root)
pty := ptytest.New(t)
inv.Stdout = pty.Output()
inv.Stderr = pty.Output()

err := inv.WithContext(ctx).Run()
require.ErrorContains(t, err, "--min-wait must be greater than zero")
})
_ = coderdtest.CreateFirstUser(t, client)

inv, root := clitest.New(t, "exp", "scaletest", "dashboard",
"--count", "1",
"--min-wait", "100ms",
"--max-wait", "1s",
"--timeout", "5s",
"--scaletest-prometheus-address", "127.0.0.1:0",
"--scaletest-prometheus-wait", "0s",
)
clitest.SetupConfig(t, client, root)
pty := ptytest.New(t)
inv.Stdout = pty.Output()
inv.Stderr = pty.Output()
t.Run("MaxWait", func(t *testing.T) {
t.Parallel()
ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitShort)
defer cancelFunc()

log := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
client := coderdtest.New(t, &coderdtest.Options{
Logger: &log,
})
_ = coderdtest.CreateFirstUser(t, client)

inv, root := clitest.New(t, "exp", "scaletest", "dashboard",
"--min-wait", "1s",
"--max-wait", "1s",
)
clitest.SetupConfig(t, client, root)
pty := ptytest.New(t)
inv.Stdout = pty.Output()
inv.Stderr = pty.Output()

err := inv.WithContext(ctx).Run()
require.ErrorContains(t, err, "--max-wait must be greater than --min-wait")
})

err := inv.WithContext(ctx).Run()
require.NoError(t, err, "")
t.Run("OK", func(t *testing.T) {
t.Parallel()
ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitMedium)
defer cancelFunc()

log := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
client := coderdtest.New(t, &coderdtest.Options{
Logger: &log,
})
_ = coderdtest.CreateFirstUser(t, client)

inv, root := clitest.New(t, "exp", "scaletest", "dashboard",
"--min-wait", "100ms",
"--max-wait", "1s",
"--timeout", "5s",
"--scaletest-prometheus-address", "127.0.0.1:0",
"--scaletest-prometheus-wait", "0s",
)
clitest.SetupConfig(t, client, root)
pty := ptytest.New(t)
inv.Stdout = pty.Output()
inv.Stderr = pty.Output()

err := inv.WithContext(ctx).Run()
require.NoError(t, err, "")
})
}
6 changes: 6 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,9 @@ require (
github.com/bep/godartsass/v2 v2.0.0 // indirect
github.com/bep/golibsass v1.1.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89 // indirect
github.com/chromedp/chromedp v0.9.2 // indirect
github.com/chromedp/sysutil v1.0.0 // indirect
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These new imports add 5MB to the slim binary, perhaps we need to consider breaking off exp scaletest from slim 🤔

Copy link
Member Author

@johnstcn johnstcn Sep 29, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can do this here 👍

Edit: better done in a follow-up PR

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#9934

github.com/clbanning/mxj/v2 v2.7.0 // indirect
github.com/cloudflare/circl v1.3.3 // indirect
github.com/containerd/continuity v0.4.2-0.20230616210509-1e0d26eb2381 // indirect
Expand Down Expand Up @@ -273,6 +276,9 @@ require (
github.com/go-test/deep v1.0.8 // indirect
github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.2.1 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
Expand Down
Loading