Skip to content

Commit 3561f63

Browse files
deansheatherdeansheather
committed
progress
1 parent be01c61 commit 3561f63

File tree

3 files changed

+287
-22
lines changed

3 files changed

+287
-22
lines changed

‎cli/scaletest.go

Lines changed: 156 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,14 @@ func (s *scaletestStrategyFlags) toStrategy() harness.ExecutionStrategy {
136136
return strategy
137137
}
138138

139+
func (s *scaletestStrategyFlags) toContext(ctx context.Context) (context.Context, context.CancelFunc) {
140+
if s.timeout > 0 {
141+
return context.WithTimeout(ctx, s.timeout)
142+
}
143+
144+
return context.WithCancel(ctx)
145+
}
146+
139147
type scaleTestOutputFormat string
140148

141149
const (
@@ -272,11 +280,39 @@ func requireAdmin(ctx context.Context, client *codersdk.Client) (codersdk.User,
272280
return me, nil
273281
}
274282

283+
// userCleanupRunner is a runner that deletes a user in the Run phase.
284+
type userCleanupRunner struct {
285+
client *codersdk.Client
286+
userID uuid.UUID
287+
}
288+
289+
var _ harness.Runnable = &userCleanupRunner{}
290+
291+
// Run implements Runnable.
292+
func (r *userCleanupRunner) Run(ctx context.Context, _ string, _ io.Writer) error {
293+
if r.userID == uuid.Nil {
294+
return nil
295+
}
296+
ctx, span := tracing.StartSpan(ctx)
297+
defer span.End()
298+
299+
err := r.client.DeleteUser(ctx, r.userID)
300+
if err != nil {
301+
return xerrors.Errorf("delete user %q: %w", r.userID, err)
302+
}
303+
304+
return nil
305+
}
306+
275307
func scaletestCleanup() *cobra.Command {
308+
var (
309+
cleanupStrategy = &scaletestStrategyFlags{cleanup: true}
310+
)
311+
276312
cmd := &cobra.Command{
277313
Use: "cleanup",
278314
Short: "Cleanup any orphaned scaletest resources",
279-
Long: "Cleanup scaletest workspaces.",
315+
Long: "Cleanup scaletest workspaces, then cleanup scaletest users. The strategy flags will apply to each stage of the cleanup process.",
280316
RunE: func(cmd *cobra.Command, args []string) error {
281317
ctx := cmd.Context()
282318
client, err := CreateClient(cmd)
@@ -291,16 +327,126 @@ func scaletestCleanup() *cobra.Command {
291327

292328
client.BypassRatelimits = true
293329

294-
// TODO: cleanup workspaces
295-
_ = isScaleTestWorkspace
330+
cmd.PrintErrln("Fetching scaletest workspaces...")
331+
var (
332+
pageNumber = 0
333+
limit = 100
334+
workspaces []codersdk.Workspace
335+
)
336+
for {
337+
page, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{
338+
Name: "scaletest-",
339+
Offset: pageNumber * limit,
340+
Limit: limit,
341+
})
342+
if err != nil {
343+
return xerrors.Errorf("fetch scaletest workspaces page %d: %w", pageNumber, err)
344+
}
345+
346+
if len(page.Workspaces) == 0 {
347+
break
348+
}
349+
350+
pageWorkspaces := make([]codersdk.Workspace, 0, len(page.Workspaces))
351+
for _, w := range page.Workspaces {
352+
if isScaleTestWorkspace(w) {
353+
pageWorkspaces = append(pageWorkspaces, w)
354+
}
355+
}
356+
workspaces = append(workspaces, pageWorkspaces...)
357+
}
358+
359+
cmd.PrintErrf("Found %d scaletest workspaces\n", len(workspaces))
360+
if len(workspaces) != 0 {
361+
cmd.Println("Deleting scaletest workspaces...")
362+
harness := harness.NewTestHarness(cleanupStrategy.toStrategy(), harness.ConcurrentExecutionStrategy{})
363+
364+
for i, w := range workspaces {
365+
const testName = "cleanup-workspace"
366+
r := workspacebuild.NewCleanupRunner(client, w.ID)
367+
harness.AddRun(testName, strconv.Itoa(i), r)
368+
}
369+
370+
ctx, cancel := cleanupStrategy.toContext(ctx)
371+
defer cancel()
372+
err := harness.Run(ctx)
373+
if err != nil {
374+
return xerrors.Errorf("run test harness to delete workspaces (harness failure, not a test failure): %w", err)
375+
}
296376

297-
// TODO: cleanup users
298-
_ = isScaleTestUser
377+
cmd.Println("Done deleting scaletest workspaces:")
378+
res := harness.Results()
379+
res.PrintText(cmd.ErrOrStderr())
380+
381+
if res.TotalFail > 0 {
382+
return xerrors.Errorf("failed to delete scaletest workspaces")
383+
}
384+
}
385+
386+
cmd.PrintErrln("Fetching scaletest users...")
387+
pageNumber = 0
388+
limit = 100
389+
var users []codersdk.User
390+
for {
391+
page, err := client.Users(ctx, codersdk.UsersRequest{
392+
Search: "scaletest-",
393+
Pagination: codersdk.Pagination{
394+
Offset: pageNumber * limit,
395+
Limit: limit,
396+
},
397+
})
398+
if err != nil {
399+
return xerrors.Errorf("fetch scaletest users page %d: %w", pageNumber, err)
400+
}
401+
402+
if len(page.Users) == 0 {
403+
break
404+
}
405+
406+
pageUsers := make([]codersdk.User, 0, len(page.Users))
407+
for _, u := range page.Users {
408+
if isScaleTestUser(u) {
409+
pageUsers = append(pageUsers, u)
410+
}
411+
}
412+
users = append(users, pageUsers...)
413+
}
414+
415+
cmd.PrintErrf("Found %d scaletest users\n", len(users))
416+
if len(workspaces) != 0 {
417+
cmd.Println("Deleting scaletest users...")
418+
harness := harness.NewTestHarness(cleanupStrategy.toStrategy(), harness.ConcurrentExecutionStrategy{})
419+
420+
for i, u := range users {
421+
const testName = "cleanup-users"
422+
r := &userCleanupRunner{
423+
client: client,
424+
userID: u.ID,
425+
}
426+
harness.AddRun(testName, strconv.Itoa(i), r)
427+
}
428+
429+
ctx, cancel := cleanupStrategy.toContext(ctx)
430+
defer cancel()
431+
err := harness.Run(ctx)
432+
if err != nil {
433+
return xerrors.Errorf("run test harness to delete users (harness failure, not a test failure): %w", err)
434+
}
435+
436+
cmd.Println("Done deleting scaletest users:")
437+
res := harness.Results()
438+
res.PrintText(cmd.ErrOrStderr())
439+
440+
if res.TotalFail > 0 {
441+
return xerrors.Errorf("failed to delete scaletest users")
442+
}
443+
}
299444

300445
return nil
301446
},
302447
}
303448

449+
cleanupStrategy.attach(cmd)
304450
return cmd
305451
}
306452

@@ -494,7 +640,7 @@ func scaletestCreateWorkspaces() *cobra.Command {
494640

495641
config := createworkspaces.Config{
496642
User: createworkspaces.UserConfig{
497-
// TODO: configuration org
643+
// TODO: configurable org
498644
OrganizationID: me.OrganizationIDs[0],
499645
Username: username,
500646
Email: email,
@@ -563,12 +709,8 @@ func scaletestCreateWorkspaces() *cobra.Command {
563709

564710
// TODO: live progress output
565711
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "Running load test...")
566-
testCtx := ctx
567-
if strategy.timeout > 0 {
568-
var testCancel func()
569-
testCtx, testCancel = context.WithTimeout(testCtx, strategy.timeout)
570-
defer testCancel()
571-
}
712+
testCtx, testCancel := strategy.toContext(ctx)
713+
defer testCancel()
572714
err = th.Run(testCtx)
573715
if err != nil {
574716
return xerrors.Errorf("run test harness (harness failure, not a test failure): %w", err)
@@ -583,12 +725,8 @@ func scaletestCreateWorkspaces() *cobra.Command {
583725
}
584726

585727
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\nCleaning up...")
586-
cleanupCtx := ctx
587-
if cleanupStrategy.timeout > 0 {
588-
var cleanupCancel context.CancelFunc
589-
cleanupCtx, cleanupCancel = context.WithTimeout(ctx, cleanupStrategy.timeout)
590-
defer cleanupCancel()
591-
}
728+
cleanupCtx, cleanupCancel := cleanupStrategy.toContext(ctx)
729+
defer cleanupCancel()
592730
err = th.Cleanup(cleanupCtx)
593731
if err != nil {
594732
return xerrors.Errorf("cleanup tests: %w", err)

‎cli/scaletest_test.go

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
package cli_test
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"os"
7+
"path/filepath"
8+
"testing"
9+
10+
"github.com/stretchr/testify/assert"
11+
"github.com/stretchr/testify/require"
12+
13+
"github.com/coder/coder/cli/clitest"
14+
"github.com/coder/coder/coderd/coderdtest"
15+
"github.com/coder/coder/pty/ptytest"
16+
"github.com/coder/coder/scaletest/harness"
17+
"github.com/coder/coder/testutil"
18+
)
19+
20+
func TestScaleTest(t *testing.T) {
21+
// t.Skipf("This test is flakey. See https://github.com/coder/coder/issues/4942")
22+
t.Parallel()
23+
24+
t.Run("WorkspaceBuild", func(t *testing.T) {
25+
t.Parallel()
26+
27+
client := coderdtest.New(t, nil)
28+
user := coderdtest.CreateFirstUser(t, client)
29+
30+
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
31+
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
32+
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
33+
34+
// Write a parameters file.
35+
tDir := t.TempDir()
36+
paramsFile := filepath.Join(tDir, "params.yaml")
37+
outputFile := filepath.Join(tDir, "output.json")
38+
39+
f, err := os.Create(paramsFile)
40+
require.NoError(t, err)
41+
defer f.Close()
42+
_, err = f.WriteString(`---
43+
param1: foo
44+
param2: true
45+
param3: 1
46+
`)
47+
require.NoError(t, err)
48+
err = f.Close()
49+
require.NoError(t, err)
50+
51+
cmd, root := clitest.New(t, "scaletest", "create-workspaces",
52+
"--count", "2",
53+
"--template", template.Name,
54+
"--parameters-file", paramsFile,
55+
"--parameter", "param1=bar",
56+
"--parameter", "param4=baz",
57+
// This flag is important for tests because agents will never be
58+
// started.
59+
"--no-wait-for-agents",
60+
// Run and connect flags cannot be tested because they require an
61+
// agent.
62+
"--concurrency", "2",
63+
"--timeout", "30s",
64+
"--job-timeout", "15s",
65+
"--cleanup-concurrency", "1",
66+
"--cleanup-timeout", "30s",
67+
"--cleanup-job-timeout", "15s",
68+
"--output text",
69+
"--output json:"+outputFile,
70+
)
71+
clitest.SetupConfig(t, client, root)
72+
pty := ptytest.New(t)
73+
cmd.SetOut(pty.Output())
74+
cmd.SetErr(pty.Output())
75+
76+
ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitLong)
77+
defer cancelFunc()
78+
79+
done := make(chan any)
80+
go func() {
81+
err := cmd.ExecuteContext(ctx)
82+
assert.NoError(t, err)
83+
close(done)
84+
}()
85+
pty.ExpectMatch("Test results:")
86+
pty.ExpectMatch("Pass: 2")
87+
select {
88+
case <-done:
89+
case <-ctx.Done():
90+
}
91+
cancelFunc()
92+
<-done
93+
94+
// Verify the output file.
95+
f, err = os.Open(outputFile)
96+
require.NoError(t, err)
97+
defer f.Close()
98+
var res harness.Results
99+
err = json.NewDecoder(f).Decode(&res)
100+
require.NoError(t, err)
101+
102+
require.EqualValues(t, 2, res.TotalRuns)
103+
require.EqualValues(t, 2, res.TotalPass)
104+
})
105+
}

‎scaletest/workspacebuild/run.go

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,23 @@ func (r *Runner) WorkspaceID() (uuid.UUID, error) {
8787
return r.workspaceID, nil
8888
}
8989

90-
// Cleanup implements Cleanable.
91-
func (r *Runner) Cleanup(ctx context.Context, _ string) error {
90+
// CleanupRunner is a runner that deletes a workspace in the Run phase.
91+
type CleanupRunner struct {
92+
client *codersdk.Client
93+
workspaceID uuid.UUID
94+
}
95+
96+
var _ harness.Runnable = &CleanupRunner{}
97+
98+
func NewCleanupRunner(client *codersdk.Client, workspaceID uuid.UUID) *CleanupRunner {
99+
return &CleanupRunner{
100+
client: client,
101+
workspaceID: workspaceID,
102+
}
103+
}
104+
105+
// Run implements Runnable.
106+
func (r *CleanupRunner) Run(ctx context.Context, _ string, logs io.Writer) error {
92107
if r.workspaceID == uuid.Nil {
93108
return nil
94109
}
@@ -102,8 +117,6 @@ func (r *Runner) Cleanup(ctx context.Context, _ string) error {
102117
return xerrors.Errorf("delete workspace: %w", err)
103118
}
104119

105-
// TODO: capture these logs
106-
logs := io.Discard
107120
err = waitForBuild(ctx, logs, r.client, build.ID)
108121
if err != nil {
109122
return xerrors.Errorf("wait for build: %w", err)
@@ -112,6 +125,15 @@ func (r *Runner) Cleanup(ctx context.Context, _ string) error {
112125
return nil
113126
}
114127

128+
// Cleanup implements Cleanable by wrapping CleanupRunner.
129+
func (r *Runner) Cleanup(ctx context.Context, id string) error {
130+
// TODO: capture these logs
131+
return (&CleanupRunner{
132+
client: r.client,
133+
workspaceID: r.workspaceID,
134+
}).Run(ctx, id, io.Discard)
135+
}
136+
115137
func waitForBuild(ctx context.Context, w io.Writer, client *codersdk.Client, buildID uuid.UUID) error {
116138
ctx, span := tracing.StartSpan(ctx)
117139
defer span.End()

0 commit comments

Comments
 (0)