Skip to content

Commit 2ff62ec

Browse files
deansheatherdeansheather
committed
tests
1 parent ae606c6 commit 2ff62ec

File tree

8 files changed

+159
-64
lines changed

8 files changed

+159
-64
lines changed

‎cli/scaletest.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,7 @@ func scaletestCleanup() *cobra.Command {
343343
return xerrors.Errorf("fetch scaletest workspaces page %d: %w", pageNumber, err)
344344
}
345345

346+
pageNumber++
346347
if len(page.Workspaces) == 0 {
347348
break
348349
}
@@ -399,6 +400,7 @@ func scaletestCleanup() *cobra.Command {
399400
return xerrors.Errorf("fetch scaletest users page %d: %w", pageNumber, err)
400401
}
401402

403+
pageNumber++
402404
if len(page.Users) == 0 {
403405
break
404406
}
@@ -513,7 +515,7 @@ func scaletestCreateWorkspaces() *cobra.Command {
513515
if template == "" {
514516
return xerrors.Errorf("--template is required")
515517
}
516-
if id, err := uuid.Parse(template); err != nil {
518+
if id, err := uuid.Parse(template); err == nil && id != uuid.Nil {
517519
tpl, err = client.Template(ctx, id)
518520
if err != nil {
519521
return xerrors.Errorf("get template by ID %q: %w", template, err)
@@ -754,7 +756,7 @@ func scaletestCreateWorkspaces() *cobra.Command {
754756
cliflag.IntVarP(cmd.Flags(), &count, "count", "c", "CODER_LOADTEST_COUNT", 1, "Required: Number of workspaces to create.")
755757
cliflag.StringVarP(cmd.Flags(), &template, "template", "t", "CODER_LOADTEST_TEMPLATE", "", "Required: Name or ID of the template to use for workspaces.")
756758
cliflag.StringVarP(cmd.Flags(), &parametersFile, "parameters-file", "", "CODER_LOADTEST_PARAMETERS_FILE", "", "Path to a YAML file containing the parameters to use for each workspace.")
757-
cliflag.StringArrayVarP(cmd.Flags(), &parameters, "parameters", "", "CODER_LOADTEST_PARAMETERS", []string{}, "Parameters to use for each workspace. Can be specified multiple times. Overrides any existing parameters with the same name from --parameters-file. Format: key=value")
759+
cliflag.StringArrayVarP(cmd.Flags(), &parameters, "parameter", "", "CODER_LOADTEST_PARAMETERS", []string{}, "Parameters to use for each workspace. Can be specified multiple times. Overrides any existing parameters with the same name from --parameters-file. Format: key=value")
758760

759761
cliflag.BoolVarP(cmd.Flags(), &noPlan, "no-plan", "", "CODER_LOADTEST_NO_PLAN", false, "Skip the dry-run step to plan the workspace creation. This step ensures that the given parameters are valid for the given template.")
760762
cliflag.BoolVarP(cmd.Flags(), &noCleanup, "no-cleanup", "", "CODER_LOADTEST_NO_CLEANUP", false, "Do not clean up resources after the test completes. You can cleanup manually using `coder scaletest cleanup`.")
@@ -768,7 +770,7 @@ func scaletestCreateWorkspaces() *cobra.Command {
768770
cliflag.BoolVarP(cmd.Flags(), &runLogOutput, "run-log-output", "", "CODER_LOADTEST_RUN_LOG_OUTPUT", false, "Log the output of the command to the test logs. This should be left off unless you expect small amounts of output. Large amounts of output will cause high memory usage.")
769771

770772
cliflag.StringVarP(cmd.Flags(), &connectURL, "connect-url", "", "CODER_LOADTEST_CONNECT_URL", "", "URL to connect to inside the the workspace over WireGuard. If not specified, no connections will be made over WireGuard.")
771-
cliflag.StringVarP(cmd.Flags(), &connectMode, "connect-mode", "derp", "CODER_LOADTEST_CONNECT_MODE", "derp", "Mode to use for connecting to the workspace. Can be 'derp' or 'direct'.")
773+
cliflag.StringVarP(cmd.Flags(), &connectMode, "connect-mode", "", "CODER_LOADTEST_CONNECT_MODE", "derp", "Mode to use for connecting to the workspace. Can be 'derp' or 'direct'.")
772774
cliflag.DurationVarP(cmd.Flags(), &connectHold, "connect-hold", "", "CODER_LOADTEST_CONNECT_HOLD", 30*time.Second, "How long to hold the WireGuard connection open for.")
773775
cliflag.DurationVarP(cmd.Flags(), &connectInterval, "connect-interval", "", "CODER_LOADTEST_CONNECT_INTERVAL", time.Second, "How long to wait between making requests to the --connect-url once the connection is established.")
774776
cliflag.DurationVarP(cmd.Flags(), &connectTimeout, "connect-timeout", "", "CODER_LOADTEST_CONNECT_TIMEOUT", 5*time.Second, "Timeout for each request to the --connect-url.")

‎cli/scaletest_test.go

Lines changed: 103 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,25 @@ import (
1212

1313
"github.com/coder/coder/cli/clitest"
1414
"github.com/coder/coder/coderd/coderdtest"
15+
"github.com/coder/coder/codersdk"
1516
"github.com/coder/coder/pty/ptytest"
1617
"github.com/coder/coder/scaletest/harness"
1718
"github.com/coder/coder/testutil"
1819
)
1920

2021
func TestScaleTest(t *testing.T) {
21-
// t.Skipf("This test is flakey. See https://github.com/coder/coder/issues/4942")
22+
t.Skipf("This test is flakey. See https://github.com/coder/coder/issues/4942")
2223
t.Parallel()
2324

24-
t.Run("WorkspaceBuild", func(t *testing.T) {
25+
// This test does a create-workspaces scale test with --no-cleanup, checks
26+
// that the created resources are OK, and then runs a cleanup.
27+
t.Run("WorkspaceBuildNoCleanup", func(t *testing.T) {
2528
t.Parallel()
2629

27-
client := coderdtest.New(t, nil)
30+
ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitLong)
31+
defer cancelFunc()
32+
33+
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
2834
user := coderdtest.CreateFirstUser(t, client)
2935

3036
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
@@ -54,6 +60,7 @@ param3: 1
5460
"--parameters-file", paramsFile,
5561
"--parameter", "param1=bar",
5662
"--parameter", "param4=baz",
63+
"--no-cleanup",
5764
// This flag is important for tests because agents will never be
5865
// started.
5966
"--no-wait-for-agents",
@@ -65,17 +72,14 @@ param3: 1
6572
"--cleanup-concurrency", "1",
6673
"--cleanup-timeout", "30s",
6774
"--cleanup-job-timeout", "15s",
68-
"--output text",
69-
"--output json:"+outputFile,
75+
"--output", "text",
76+
"--output", "json:"+outputFile,
7077
)
7178
clitest.SetupConfig(t, client, root)
7279
pty := ptytest.New(t)
7380
cmd.SetOut(pty.Output())
7481
cmd.SetErr(pty.Output())
7582

76-
ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitLong)
77-
defer cancelFunc()
78-
7983
done := make(chan any)
8084
go func() {
8185
err := cmd.ExecuteContext(ctx)
@@ -91,6 +95,10 @@ param3: 1
9195
cancelFunc()
9296
<-done
9397

98+
// Recreate the context.
99+
ctx, cancelFunc = context.WithTimeout(context.Background(), testutil.WaitLong)
100+
defer cancelFunc()
101+
94102
// Verify the output file.
95103
f, err = os.Open(outputFile)
96104
require.NoError(t, err)
@@ -101,5 +109,92 @@ param3: 1
101109

102110
require.EqualValues(t, 2, res.TotalRuns)
103111
require.EqualValues(t, 2, res.TotalPass)
112+
113+
// Find the workspaces and users and check that they are what we expect.
114+
workspaces, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{
115+
Offset: 0,
116+
Limit: 100,
117+
})
118+
require.NoError(t, err)
119+
require.Len(t, workspaces.Workspaces, 2)
120+
121+
seenUsers := map[string]struct{}{}
122+
for _, w := range workspaces.Workspaces {
123+
// Sadly we can't verify params as the API doesn't seem to return
124+
// them.
125+
126+
// Verify that the user is a unique scaletest user.
127+
u, err := client.User(ctx, w.OwnerID.String())
128+
require.NoError(t, err)
129+
130+
_, ok := seenUsers[u.ID.String()]
131+
require.False(t, ok, "user has more than one workspace")
132+
seenUsers[u.ID.String()] = struct{}{}
133+
134+
require.Contains(t, u.Username, "scaletest-")
135+
require.Contains(t, u.Email, "scaletest")
136+
}
137+
138+
require.Len(t, seenUsers, len(workspaces.Workspaces))
139+
140+
// Check that there are exactly 3 users.
141+
users, err := client.Users(ctx, codersdk.UsersRequest{
142+
Pagination: codersdk.Pagination{
143+
Offset: 0,
144+
Limit: 100,
145+
},
146+
})
147+
require.NoError(t, err)
148+
require.Len(t, users.Users, len(seenUsers)+1)
149+
150+
// Cleanup.
151+
cmd, root = clitest.New(t, "scaletest", "cleanup",
152+
"--cleanup-concurrency", "1",
153+
"--cleanup-timeout", "30s",
154+
"--cleanup-job-timeout", "15s",
155+
)
156+
clitest.SetupConfig(t, client, root)
157+
pty = ptytest.New(t)
158+
cmd.SetOut(pty.Output())
159+
cmd.SetErr(pty.Output())
160+
161+
done = make(chan any)
162+
go func() {
163+
err := cmd.ExecuteContext(ctx)
164+
assert.NoError(t, err)
165+
close(done)
166+
}()
167+
pty.ExpectMatch("Test results:")
168+
pty.ExpectMatch("Pass: 2")
169+
pty.ExpectMatch("Test results:")
170+
pty.ExpectMatch("Pass: 2")
171+
select {
172+
case <-done:
173+
case <-ctx.Done():
174+
}
175+
cancelFunc()
176+
<-done
177+
178+
// Recreate the context (again).
179+
ctx, cancelFunc = context.WithTimeout(context.Background(), testutil.WaitLong)
180+
defer cancelFunc()
181+
182+
// Verify that the workspaces are gone.
183+
workspaces, err = client.Workspaces(ctx, codersdk.WorkspaceFilter{
184+
Offset: 0,
185+
Limit: 100,
186+
})
187+
require.NoError(t, err)
188+
require.Len(t, workspaces.Workspaces, 0)
189+
190+
// Verify that the users are gone.
191+
users, err = client.Users(ctx, codersdk.UsersRequest{
192+
Pagination: codersdk.Pagination{
193+
Offset: 0,
194+
Limit: 100,
195+
},
196+
})
197+
require.NoError(t, err)
198+
require.Len(t, users.Users, 1)
104199
})
105200
}

‎coderd/database/databasefake/databasefake.go

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -448,7 +448,7 @@ func (q *fakeQuerier) GetUserByEmailOrUsername(_ context.Context, arg database.G
448448
defer q.mutex.RUnlock()
449449

450450
for _, user := range q.users {
451-
if (strings.EqualFold(user.Email, arg.Email) || strings.EqualFold(user.Username, arg.Username)) && user.Deleted == arg.Deleted {
451+
if strings.EqualFold(user.Email, arg.Email) || strings.EqualFold(user.Username, arg.Username) {
452452
return user, nil
453453
}
454454
}
@@ -513,15 +513,14 @@ func (q *fakeQuerier) GetAuthorizedUserCount(ctx context.Context, params databas
513513
users = append(users, user)
514514
}
515515

516-
if params.Deleted {
517-
tmp := make([]database.User, 0, len(users))
518-
for _, user := range users {
519-
if user.Deleted {
520-
tmp = append(tmp, user)
521-
}
516+
// Filter out deleted since they should never be returned..
517+
tmp := make([]database.User, 0, len(users))
518+
for _, user := range users {
519+
if !user.Deleted {
520+
tmp = append(tmp, user)
522521
}
523-
users = tmp
524522
}
523+
users = tmp
525524

526525
if params.Search != "" {
527526
tmp := make([]database.User, 0, len(users))
@@ -593,15 +592,14 @@ func (q *fakeQuerier) GetUsers(_ context.Context, params database.GetUsersParams
593592
return a.CreatedAt.Before(b.CreatedAt)
594593
})
595594

596-
if params.Deleted {
597-
tmp := make([]database.User, 0, len(users))
598-
for _, user := range users {
599-
if user.Deleted {
600-
tmp = append(tmp, user)
601-
}
595+
// Filter out deleted since they should never be returned..
596+
tmp := make([]database.User, 0, len(users))
597+
for _, user := range users {
598+
if !user.Deleted {
599+
tmp = append(tmp, user)
602600
}
603-
users = tmp
604601
}
602+
users = tmp
605603

606604
if params.AfterID != uuid.Nil {
607605
found := false

‎coderd/database/modelqueries.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,6 @@ func (q *sqlQuerier) GetAuthorizedUserCount(ctx context.Context, arg GetFiltered
261261

262262
query := fmt.Sprintf("-- name: GetAuthorizedUserCount :one\n%s", filtered)
263263
row := q.db.QueryRowContext(ctx, query,
264-
arg.Deleted,
265264
arg.Search,
266265
pq.Array(arg.Status),
267266
pq.Array(arg.RbacRole),

0 commit comments

Comments
 (0)