Skip to content

Commit b3471bd

Browse files
johnstcnjohnstcn
authored
fix(scaletest/dashboard): increase viewport size and handle deadlines (#10197)
- Set viewport size to avoid responsive mode - Added way more debug logging - Added facility to write a screenshot on error in verbose mode. - Added a deadline for each iteraction of clicking on and waiting for a thing.
1 parent dc11705 commit b3471bd

File tree

5 files changed

+140
-28
lines changed

5 files changed

+140
-28
lines changed

‎cli/exp_scaletest.go

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1099,6 +1099,9 @@ func (r *RootCmd) scaletestDashboard() *clibase.Cmd {
10991099
}
11001100
ctx := inv.Context()
11011101
logger := slog.Make(sloghuman.Sink(inv.Stdout)).Leveled(slog.LevelInfo)
1102+
if r.verbose {
1103+
logger = logger.Leveled(slog.LevelDebug)
1104+
}
11021105
tracerProvider, closeTracing, tracingEnabled, err := tracingFlags.provider(ctx)
11031106
if err != nil {
11041107
return xerrors.Errorf("create tracer provider: %w", err)
@@ -1148,16 +1151,20 @@ func (r *RootCmd) scaletestDashboard() *clibase.Cmd {
11481151
userClient.SetSessionToken(userTokResp.Key)
11491152

11501153
config := dashboard.Config{
1151-
Interval: interval,
1152-
Jitter: jitter,
1153-
Trace: tracingEnabled,
1154-
Logger: logger.Named(name),
1155-
Headless: headless,
1156-
ActionFunc: dashboard.ClickRandomElement,
1157-
RandIntn: rndGen.Intn,
1154+
Interval: interval,
1155+
Jitter: jitter,
1156+
Trace: tracingEnabled,
1157+
Logger: logger.Named(name),
1158+
Headless: headless,
1159+
RandIntn: rndGen.Intn,
1160+
}
1161+
// Only take a screenshot if we're in verbose mode.
1162+
// This could be useful for debugging, but it will blow up the disk.
1163+
if r.verbose {
1164+
config.Screenshot = dashboard.Screenshot
11581165
}
11591166
//nolint:gocritic
1160-
logger.Info(ctx, "runner config", slog.F("min_wait", interval), slog.F("max_wait", jitter), slog.F("headless", headless), slog.F("trace", tracingEnabled))
1167+
logger.Info(ctx, "runner config", slog.F("interval", interval), slog.F("jitter", jitter), slog.F("headless", headless), slog.F("trace", tracingEnabled))
11611168
if err := config.Validate(); err != nil {
11621169
return err
11631170
}
@@ -1200,14 +1207,14 @@ func (r *RootCmd) scaletestDashboard() *clibase.Cmd {
12001207
{
12011208
Flag: "interval",
12021209
Env: "CODER_SCALETEST_DASHBOARD_INTERVAL",
1203-
Default: "3s",
1210+
Default: "10s",
12041211
Description: "Interval between actions.",
12051212
Value: clibase.DurationOf(&interval),
12061213
},
12071214
{
12081215
Flag: "jitter",
12091216
Env: "CODER_SCALETEST_DASHBOARD_JITTER",
1210-
Default: "2s",
1217+
Default: "5s",
12111218
Description: "Jitter between actions.",
12121219
Value: clibase.DurationOf(&jitter),
12131220
},

‎scaletest/dashboard/chromedp.go

Lines changed: 66 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,19 @@ package dashboard
22

33
import (
44
"context"
5+
"fmt"
56
"net/url"
67
"os"
8+
"path/filepath"
79
"time"
810

911
"github.com/chromedp/cdproto/cdp"
1012
"github.com/chromedp/cdproto/network"
1113
"github.com/chromedp/chromedp"
1214
"golang.org/x/xerrors"
1315

16+
"github.com/coder/coder/v2/cryptorand"
17+
1418
"cdr.dev/slog"
1519
)
1620

@@ -86,17 +90,17 @@ var defaultTargets = []Target{
8690
},
8791
}
8892

89-
// ClickRandomElement returns an action that will click an element from defaultTargets.
93+
// clickRandomElement returns an action that will click an element from defaultTargets.
9094
// If no elements are found, an error is returned.
9195
// If more than one element is found, one is chosen at random.
9296
// The label of the clicked element is returned.
93-
func ClickRandomElement(ctx context.Context, randIntn func(int) int) (Label, Action, error) {
97+
func clickRandomElement(ctx context.Context, log slog.Logger, randIntn func(int) int, deadline time.Time) (Label, Action, error) {
9498
var xpath Selector
9599
var found bool
96100
var err error
97101
matches := make([]Target, 0)
98102
for _, tgt := range defaultTargets {
99-
xpath, found, err = randMatch(ctx, tgt.ClickOn, randIntn)
103+
xpath, found, err = randMatch(ctx, log, tgt.ClickOn, randIntn, deadline)
100104
if err != nil {
101105
return "", nil, xerrors.Errorf("find matches for %q: %w", tgt.ClickOn, err)
102106
}
@@ -111,14 +115,20 @@ func ClickRandomElement(ctx context.Context, randIntn func(int) int) (Label, Act
111115
}
112116

113117
if len(matches) == 0 {
118+
log.Debug(ctx, "no matches found this time")
114119
return "", nil, xerrors.Errorf("no matches found")
115120
}
116121
match := pick(matches, randIntn)
117-
// rely on map iteration order being random
118122
act := func(actx context.Context) error {
119-
if err := clickAndWait(actx, match.ClickOn, match.WaitFor); err != nil {
123+
log.Debug(ctx, "clicking", slog.F("label", match.Label), slog.F("xpath", match.ClickOn))
124+
if err := runWithDeadline(ctx, deadline, chromedp.Click(match.ClickOn, chromedp.NodeReady)); err != nil {
125+
log.Error(ctx, "click failed", slog.F("label", match.Label), slog.F("xpath", match.ClickOn), slog.Error(err))
120126
return xerrors.Errorf("click %q: %w", match.ClickOn, err)
121127
}
128+
if err := runWithDeadline(ctx, deadline, chromedp.WaitReady(match.WaitFor)); err != nil {
129+
log.Error(ctx, "wait failed", slog.F("label", match.Label), slog.F("xpath", match.WaitFor), slog.Error(err))
130+
return xerrors.Errorf("wait for %q: %w", match.WaitFor, err)
131+
}
122132
return nil
123133
}
124134
return match.Label, act, nil
@@ -128,26 +138,32 @@ func ClickRandomElement(ctx context.Context, randIntn func(int) int) (Label, Act
128138
// The returned selector is the full XPath of the matched node.
129139
// If no matches are found, an error is returned.
130140
// If multiple matches are found, one is chosen at random.
131-
func randMatch(ctx context.Context, s Selector, randIntn func(int) int) (Selector, bool, error) {
141+
func randMatch(ctx context.Context, log slog.Logger, s Selector, randIntn func(int) int, deadline time.Time) (Selector, bool, error) {
132142
var nodes []*cdp.Node
133-
err := chromedp.Run(ctx, chromedp.Nodes(s, &nodes, chromedp.NodeVisible, chromedp.AtLeast(0)))
134-
if err != nil {
143+
log.Debug(ctx, "getting nodes for selector", slog.F("selector", s))
144+
if err := runWithDeadline(ctx, deadline, chromedp.Nodes(s, &nodes, chromedp.NodeReady, chromedp.AtLeast(0))); err != nil {
145+
log.Debug(ctx, "failed to get nodes for selector", slog.F("selector", s), slog.Error(err))
135146
return "", false, xerrors.Errorf("get nodes for selector %q: %w", s, err)
136147
}
137148
if len(nodes) == 0 {
149+
log.Debug(ctx, "no nodes found for selector", slog.F("selector", s))
138150
return "", false, nil
139151
}
140152
n := pick(nodes, randIntn)
153+
log.Debug(ctx, "found node", slog.F("node", n.FullXPath()))
141154
return Selector(n.FullXPath()), true, nil
142155
}
143156

144-
// clickAndWait clicks the given selector and waits for the page to finish loading.
145-
// The page is considered loaded when the network event "LoadingFinished" is received.
146-
func clickAndWait(ctx context.Context, clickOn, waitFor Selector) error {
147-
return chromedp.Run(ctx, chromedp.Tasks{
148-
chromedp.Click(clickOn, chromedp.NodeVisible),
149-
chromedp.WaitVisible(waitFor, chromedp.NodeVisible),
150-
})
157+
func waitForWorkspacesPageLoaded(ctx context.Context, deadline time.Time) error {
158+
return runWithDeadline(ctx, deadline, chromedp.WaitReady(`tbody.MuiTableBody-root`))
159+
}
160+
161+
func runWithDeadline(ctx context.Context, deadline time.Time, acts ...chromedp.Action) error {
162+
deadlineCtx, deadlineCancel := context.WithDeadline(ctx, deadline)
163+
defer deadlineCancel()
164+
c := chromedp.FromContext(ctx)
165+
tasks := chromedp.Tasks(acts)
166+
return tasks.Do(cdp.WithExecutor(deadlineCtx, c.Target))
151167
}
152168

153169
// initChromeDPCtx initializes a chromedp context with the given session token cookie
@@ -178,6 +194,13 @@ func initChromeDPCtx(ctx context.Context, log slog.Logger, u *url.URL, sessionTo
178194
}
179195
}
180196

197+
// force a viewport size of 1024x768 so we don't go into mobile mode
198+
if err := chromedp.Run(cdpCtx, chromedp.EmulateViewport(1024, 768)); err != nil {
199+
cancelFunc()
200+
allocCtxCancel()
201+
return nil, nil, xerrors.Errorf("set viewport size: %w", err)
202+
}
203+
181204
// set cookies
182205
if err := setSessionTokenCookie(cdpCtx, sessionToken, u.Host); err != nil {
183206
cancelFunc()
@@ -209,6 +232,34 @@ func visitMainPage(ctx context.Context, u *url.URL) error {
209232
return chromedp.Run(ctx, chromedp.Navigate(u.String()))
210233
}
211234

235+
func Screenshot(ctx context.Context, name string) (string, error) {
236+
var buf []byte
237+
if err := chromedp.Run(ctx, chromedp.CaptureScreenshot(&buf)); err != nil {
238+
return "", xerrors.Errorf("capture screenshot: %w", err)
239+
}
240+
randExt, err := cryptorand.String(4)
241+
if err != nil {
242+
// this should never happen
243+
return "", xerrors.Errorf("generate random string: %w", err)
244+
}
245+
fname := fmt.Sprintf("scaletest-dashboard-%s-%s-%s.png", name, time.Now().Format("20060102-150405"), randExt)
246+
pwd, err := os.Getwd()
247+
if err != nil {
248+
return "", xerrors.Errorf("get working directory: %w", err)
249+
}
250+
fpath := filepath.Join(pwd, fname)
251+
f, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0o644)
252+
if err != nil {
253+
return "", xerrors.Errorf("open file: %w", err)
254+
}
255+
defer f.Close()
256+
if _, err := f.Write(buf); err != nil {
257+
return "", xerrors.Errorf("write file: %w", err)
258+
}
259+
260+
return fpath, nil
261+
}
262+
212263
// pick chooses a random element from a slice.
213264
// If the slice is empty, it returns the zero value of the type.
214265
func pick[T any](s []T, randIntn func(int) int) T {

‎scaletest/dashboard/config.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,11 @@ type Config struct {
2121
// Headless controls headless mode for chromedp.
2222
Headless bool `json:"headless"`
2323
// ActionFunc is a function that returns an action to run.
24-
ActionFunc func(ctx context.Context, randIntn func(int) int) (Label, Action, error) `json:"-"`
24+
ActionFunc func(ctx context.Context, log slog.Logger, randIntn func(int) int, deadline time.Time) (Label, Action, error) `json:"-"`
25+
// WaitLoaded is a function that waits for the page to be loaded.
26+
WaitLoaded func(ctx context.Context, deadline time.Time) error
27+
// Screenshot is a function that takes a screenshot.
28+
Screenshot func(ctx context.Context, filename string) (string, error)
2529
// RandIntn is a function that returns a random number between 0 and n-1.
2630
RandIntn func(int) int `json:"-"`
2731
}

‎scaletest/dashboard/run.go

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package dashboard
22

33
import (
44
"context"
5+
"errors"
56
"io"
67
"time"
78

@@ -25,6 +26,15 @@ var (
2526

2627
func NewRunner(client *codersdk.Client, metrics Metrics, cfg Config) *Runner {
2728
client.Trace = cfg.Trace
29+
if cfg.WaitLoaded == nil {
30+
cfg.WaitLoaded = waitForWorkspacesPageLoaded
31+
}
32+
if cfg.ActionFunc == nil {
33+
cfg.ActionFunc = clickRandomElement
34+
}
35+
if cfg.Screenshot == nil {
36+
cfg.Screenshot = Screenshot
37+
}
2838
return &Runner{
2939
client: client,
3040
cfg: cfg,
@@ -33,6 +43,16 @@ func NewRunner(client *codersdk.Client, metrics Metrics, cfg Config) *Runner {
3343
}
3444

3545
func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error {
46+
err := r.runUntilDeadlineExceeded(ctx)
47+
// If the context deadline exceeded, don't return an error.
48+
// This just means the test finished.
49+
if err == nil || errors.Is(err, context.DeadlineExceeded) {
50+
return nil
51+
}
52+
return err
53+
}
54+
55+
func (r *Runner) runUntilDeadlineExceeded(ctx context.Context) error {
3656
if r.client == nil {
3757
return xerrors.Errorf("client is nil")
3858
}
@@ -53,6 +73,11 @@ func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error {
5373
defer cdpCancel()
5474
t := time.NewTicker(1) // First one should be immediate
5575
defer t.Stop()
76+
r.cfg.Logger.Info(ctx, "waiting for workspaces page to load")
77+
loadWorkspacePageDeadline := time.Now().Add(r.cfg.Interval)
78+
if err := r.cfg.WaitLoaded(cdpCtx, loadWorkspacePageDeadline); err != nil {
79+
return xerrors.Errorf("wait for workspaces page to load: %w", err)
80+
}
5681
for {
5782
select {
5883
case <-cdpCtx.Done():
@@ -63,10 +88,16 @@ func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error {
6388
offset = time.Duration(r.cfg.RandIntn(int(2*r.cfg.Jitter)) - int(r.cfg.Jitter))
6489
}
6590
wait := r.cfg.Interval + offset
91+
actionCompleteByDeadline := time.Now().Add(wait)
6692
t.Reset(wait)
67-
l, act, err := r.cfg.ActionFunc(cdpCtx, r.cfg.RandIntn)
93+
l, act, err := r.cfg.ActionFunc(cdpCtx, r.cfg.Logger, r.cfg.RandIntn, actionCompleteByDeadline)
6894
if err != nil {
6995
r.cfg.Logger.Error(ctx, "calling ActionFunc", slog.Error(err))
96+
sPath, sErr := r.cfg.Screenshot(cdpCtx, me.Username)
97+
if sErr != nil {
98+
r.cfg.Logger.Error(ctx, "screenshot failed", slog.Error(sErr))
99+
}
100+
r.cfg.Logger.Info(ctx, "screenshot saved", slog.F("path", sPath))
70101
continue
71102
}
72103
start := time.Now()
@@ -77,6 +108,11 @@ func (r *Runner) Run(ctx context.Context, _ string, _ io.Writer) error {
77108
r.metrics.IncErrors(string(l))
78109
//nolint:gocritic
79110
r.cfg.Logger.Error(ctx, "action failed", slog.F("label", l), slog.Error(err))
111+
sPath, sErr := r.cfg.Screenshot(cdpCtx, me.Username+"-"+string(l))
112+
if sErr != nil {
113+
r.cfg.Logger.Error(ctx, "screenshot failed", slog.Error(sErr))
114+
}
115+
r.cfg.Logger.Info(ctx, "screenshot saved", slog.F("path", sPath))
80116
} else {
81117
//nolint:gocritic
82118
r.cfg.Logger.Info(ctx, "action success", slog.F("label", l))

‎scaletest/dashboard/run_test.go

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,14 @@ import (
55
"math/rand"
66
"runtime"
77
"sync"
8+
"sync/atomic"
89
"testing"
910
"time"
1011

1112
"github.com/stretchr/testify/assert"
1213
"github.com/stretchr/testify/require"
1314

15+
"cdr.dev/slog"
1416
"cdr.dev/slog/sloggers/slogtest"
1517
"github.com/coder/coder/v2/coderd/coderdtest"
1618
"github.com/coder/coder/v2/scaletest/dashboard"
@@ -46,17 +48,29 @@ func Test_Run(t *testing.T) {
4648
IgnoreErrors: true,
4749
})
4850
m := &testMetrics{}
51+
var (
52+
waitLoadedCalled atomic.Bool
53+
screenshotCalled atomic.Bool
54+
)
4955
cfg := dashboard.Config{
5056
Interval: 500 * time.Millisecond,
5157
Jitter: 100 * time.Millisecond,
5258
Logger: log,
5359
Headless: true,
54-
ActionFunc: func(_ context.Context, rnd func(int) int) (dashboard.Label, dashboard.Action, error) {
60+
WaitLoaded: func(_ context.Context, _ time.Time) error {
61+
waitLoadedCalled.Store(true)
62+
return nil
63+
},
64+
ActionFunc: func(_ context.Context, _ slog.Logger, rnd func(int) int, _ time.Time) (dashboard.Label, dashboard.Action, error) {
5565
if rnd(2) == 0 {
5666
return "fails", failAction, nil
5767
}
5868
return "succeeds", successAction, nil
5969
},
70+
Screenshot: func(_ context.Context, name string) (string, error) {
71+
screenshotCalled.Store(true)
72+
return "/fake/path/to/" + name + ".png", nil
73+
},
6074
RandIntn: rg.Intn,
6175
}
6276
r := dashboard.NewRunner(client, m, cfg)

0 commit comments

Comments
 (0)