Skip to content

feat: remove loadtest cmd, add new scaletest cmd #5310

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Dec 15, 2022
Merged

feat: remove loadtest cmd, add new scaletest cmd #5310

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
be01c61
feat: remove loadtest cmd, add new scaletest cmd
deansheather Dec 6, 2022
3561f63
progress
deansheather Dec 12, 2022
ae606c6
Merge branch 'main' into dean/scaletest
deansheather Dec 12, 2022
2ff62ec
tests
deansheather Dec 12, 2022
a6a36f7
chore: add createworkspaces tests
deansheather Dec 13, 2022
de1577d
Merge branch 'main' into dean/scaletest
deansheather Dec 15, 2022
03a5834
fixup! Merge branch 'main' into dean/scaletest
deansheather Dec 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
tests
  • Loading branch information
@deansheather
deansheather committed Dec 12, 2022
commit 2ff62ec5f85177590c7b3f48c35865e5651f6e6d
8 changes: 5 additions & 3 deletions cli/scaletest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,7 @@ func scaletestCleanup() *cobra.Command {
return xerrors.Errorf("fetch scaletest workspaces page %d: %w", pageNumber, err)
}

pageNumber++
if len(page.Workspaces) == 0 {
break
}
Expand Down Expand Up @@ -399,6 +400,7 @@ func scaletestCleanup() *cobra.Command {
return xerrors.Errorf("fetch scaletest users page %d: %w", pageNumber, err)
}

pageNumber++
if len(page.Users) == 0 {
break
}
Expand Down Expand Up @@ -513,7 +515,7 @@ func scaletestCreateWorkspaces() *cobra.Command {
if template == "" {
return xerrors.Errorf("--template is required")
}
if id, err := uuid.Parse(template); err != nil {
if id, err := uuid.Parse(template); err == nil && id != uuid.Nil {
tpl, err = client.Template(ctx, id)
if err != nil {
return xerrors.Errorf("get template by ID %q: %w", template, err)
Expand Down Expand Up @@ -754,7 +756,7 @@ func scaletestCreateWorkspaces() *cobra.Command {
cliflag.IntVarP(cmd.Flags(), &count, "count", "c", "CODER_LOADTEST_COUNT", 1, "Required: Number of workspaces to create.")
cliflag.StringVarP(cmd.Flags(), &template, "template", "t", "CODER_LOADTEST_TEMPLATE", "", "Required: Name or ID of the template to use for workspaces.")
cliflag.StringVarP(cmd.Flags(), &parametersFile, "parameters-file", "", "CODER_LOADTEST_PARAMETERS_FILE", "", "Path to a YAML file containing the parameters to use for each workspace.")
cliflag.StringArrayVarP(cmd.Flags(), &parameters, "parameters", "", "CODER_LOADTEST_PARAMETERS", []string{}, "Parameters to use for each workspace. Can be specified multiple times. Overrides any existing parameters with the same name from --parameters-file. Format: key=value")
cliflag.StringArrayVarP(cmd.Flags(), &parameters, "parameter", "", "CODER_LOADTEST_PARAMETERS", []string{}, "Parameters to use for each workspace. Can be specified multiple times. Overrides any existing parameters with the same name from --parameters-file. Format: key=value")

cliflag.BoolVarP(cmd.Flags(), &noPlan, "no-plan", "", "CODER_LOADTEST_NO_PLAN", false, "Skip the dry-run step to plan the workspace creation. This step ensures that the given parameters are valid for the given template.")
cliflag.BoolVarP(cmd.Flags(), &noCleanup, "no-cleanup", "", "CODER_LOADTEST_NO_CLEANUP", false, "Do not clean up resources after the test completes. You can cleanup manually using `coder scaletest cleanup`.")
Expand All @@ -768,7 +770,7 @@ func scaletestCreateWorkspaces() *cobra.Command {
cliflag.BoolVarP(cmd.Flags(), &runLogOutput, "run-log-output", "", "CODER_LOADTEST_RUN_LOG_OUTPUT", false, "Log the output of the command to the test logs. This should be left off unless you expect small amounts of output. Large amounts of output will cause high memory usage.")

cliflag.StringVarP(cmd.Flags(), &connectURL, "connect-url", "", "CODER_LOADTEST_CONNECT_URL", "", "URL to connect to inside the the workspace over WireGuard. If not specified, no connections will be made over WireGuard.")
cliflag.StringVarP(cmd.Flags(), &connectMode, "connect-mode", "derp", "CODER_LOADTEST_CONNECT_MODE", "derp", "Mode to use for connecting to the workspace. Can be 'derp' or 'direct'.")
cliflag.StringVarP(cmd.Flags(), &connectMode, "connect-mode", "", "CODER_LOADTEST_CONNECT_MODE", "derp", "Mode to use for connecting to the workspace. Can be 'derp' or 'direct'.")
cliflag.DurationVarP(cmd.Flags(), &connectHold, "connect-hold", "", "CODER_LOADTEST_CONNECT_HOLD", 30*time.Second, "How long to hold the WireGuard connection open for.")
cliflag.DurationVarP(cmd.Flags(), &connectInterval, "connect-interval", "", "CODER_LOADTEST_CONNECT_INTERVAL", time.Second, "How long to wait between making requests to the --connect-url once the connection is established.")
cliflag.DurationVarP(cmd.Flags(), &connectTimeout, "connect-timeout", "", "CODER_LOADTEST_CONNECT_TIMEOUT", 5*time.Second, "Timeout for each request to the --connect-url.")
Expand Down
111 changes: 103 additions & 8 deletions cli/scaletest_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,25 @@ import (

"github.com/coder/coder/cli/clitest"
"github.com/coder/coder/coderd/coderdtest"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/pty/ptytest"
"github.com/coder/coder/scaletest/harness"
"github.com/coder/coder/testutil"
)

func TestScaleTest(t *testing.T) {
// t.Skipf("This test is flakey. See https://github.com/coder/coder/issues/4942")
t.Skipf("This test is flakey. See https://github.com/coder/coder/issues/4942")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this still flakey?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Probably, it calls the flakey workspace build scale test :/

t.Parallel()

t.Run("WorkspaceBuild", func(t *testing.T) {
// This test does a create-workspaces scale test with --no-cleanup, checks
// that the created resources are OK, and then runs a cleanup.
t.Run("WorkspaceBuildNoCleanup", func(t *testing.T) {
t.Parallel()

client := coderdtest.New(t, nil)
ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancelFunc()

client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
user := coderdtest.CreateFirstUser(t, client)

version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
Expand Down Expand Up @@ -54,6 +60,7 @@ param3: 1
"--parameters-file", paramsFile,
"--parameter", "param1=bar",
"--parameter", "param4=baz",
"--no-cleanup",
// This flag is important for tests because agents will never be
// started.
"--no-wait-for-agents",
Expand All @@ -65,17 +72,14 @@ param3: 1
"--cleanup-concurrency", "1",
"--cleanup-timeout", "30s",
"--cleanup-job-timeout", "15s",
"--output text",
"--output json:"+outputFile,
"--output", "text",
"--output", "json:"+outputFile,
)
clitest.SetupConfig(t, client, root)
pty := ptytest.New(t)
cmd.SetOut(pty.Output())
cmd.SetErr(pty.Output())

ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancelFunc()

done := make(chan any)
go func() {
err := cmd.ExecuteContext(ctx)
Expand All @@ -91,6 +95,10 @@ param3: 1
cancelFunc()
<-done

// Recreate the context.
ctx, cancelFunc = context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancelFunc()

// Verify the output file.
f, err = os.Open(outputFile)
require.NoError(t, err)
Expand All @@ -101,5 +109,92 @@ param3: 1

require.EqualValues(t, 2, res.TotalRuns)
require.EqualValues(t, 2, res.TotalPass)

// Find the workspaces and users and check that they are what we expect.
workspaces, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{
Offset: 0,
Limit: 100,
})
require.NoError(t, err)
require.Len(t, workspaces.Workspaces, 2)

seenUsers := map[string]struct{}{}
for _, w := range workspaces.Workspaces {
// Sadly we can't verify params as the API doesn't seem to return
// them.

// Verify that the user is a unique scaletest user.
u, err := client.User(ctx, w.OwnerID.String())
require.NoError(t, err)

_, ok := seenUsers[u.ID.String()]
require.False(t, ok, "user has more than one workspace")
seenUsers[u.ID.String()] = struct{}{}

require.Contains(t, u.Username, "scaletest-")
require.Contains(t, u.Email, "scaletest")
}

require.Len(t, seenUsers, len(workspaces.Workspaces))

// Check that there are exactly 3 users.
users, err := client.Users(ctx, codersdk.UsersRequest{
Pagination: codersdk.Pagination{
Offset: 0,
Limit: 100,
},
})
require.NoError(t, err)
require.Len(t, users.Users, len(seenUsers)+1)

// Cleanup.
cmd, root = clitest.New(t, "scaletest", "cleanup",
"--cleanup-concurrency", "1",
"--cleanup-timeout", "30s",
"--cleanup-job-timeout", "15s",
)
clitest.SetupConfig(t, client, root)
pty = ptytest.New(t)
cmd.SetOut(pty.Output())
cmd.SetErr(pty.Output())

done = make(chan any)
go func() {
err := cmd.ExecuteContext(ctx)
assert.NoError(t, err)
close(done)
}()
pty.ExpectMatch("Test results:")
pty.ExpectMatch("Pass: 2")
pty.ExpectMatch("Test results:")
pty.ExpectMatch("Pass: 2")
select {
case <-done:
case <-ctx.Done():
}
cancelFunc()
<-done

// Recreate the context (again).
ctx, cancelFunc = context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancelFunc()

// Verify that the workspaces are gone.
workspaces, err = client.Workspaces(ctx, codersdk.WorkspaceFilter{
Offset: 0,
Limit: 100,
})
require.NoError(t, err)
require.Len(t, workspaces.Workspaces, 0)

// Verify that the users are gone.
users, err = client.Users(ctx, codersdk.UsersRequest{
Pagination: codersdk.Pagination{
Offset: 0,
Limit: 100,
},
})
require.NoError(t, err)
require.Len(t, users.Users, 1)
})
}
28 changes: 13 additions & 15 deletions coderd/database/databasefake/databasefake.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,7 @@ func (q *fakeQuerier) GetUserByEmailOrUsername(_ context.Context, arg database.G
defer q.mutex.RUnlock()

for _, user := range q.users {
if (strings.EqualFold(user.Email, arg.Email) || strings.EqualFold(user.Username, arg.Username)) && user.Deleted == arg.Deleted {
if strings.EqualFold(user.Email, arg.Email) || strings.EqualFold(user.Username, arg.Username) {
return user, nil
}
}
Expand Down Expand Up @@ -513,15 +513,14 @@ func (q *fakeQuerier) GetAuthorizedUserCount(ctx context.Context, params databas
users = append(users, user)
}

if params.Deleted {
tmp := make([]database.User, 0, len(users))
for _, user := range users {
if user.Deleted {
tmp = append(tmp, user)
}
// Filter out deleted since they should never be returned..
tmp := make([]database.User, 0, len(users))
for _, user := range users {
if !user.Deleted {
tmp = append(tmp, user)
}
users = tmp
}
users = tmp

if params.Search != "" {
tmp := make([]database.User, 0, len(users))
Expand Down Expand Up @@ -593,15 +592,14 @@ func (q *fakeQuerier) GetUsers(_ context.Context, params database.GetUsersParams
return a.CreatedAt.Before(b.CreatedAt)
})

if params.Deleted {
tmp := make([]database.User, 0, len(users))
for _, user := range users {
if user.Deleted {
tmp = append(tmp, user)
}
// Filter out deleted since they should never be returned..
tmp := make([]database.User, 0, len(users))
for _, user := range users {
if !user.Deleted {
tmp = append(tmp, user)
}
users = tmp
}
users = tmp

if params.AfterID != uuid.Nil {
found := false
Expand Down
1 change: 0 additions & 1 deletion coderd/database/modelqueries.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,6 @@ func (q *sqlQuerier) GetAuthorizedUserCount(ctx context.Context, arg GetFiltered

query := fmt.Sprintf("-- name: GetAuthorizedUserCount :one\n%s", filtered)
row := q.db.QueryRowContext(ctx, query,
arg.Deleted,
arg.Search,
pq.Array(arg.Status),
pq.Array(arg.RbacRole),
Expand Down
Loading