Skip to content

Commit e7dd3f9

Browse files
deansheatherdeansheather
authored
feat: add load testing harness, coder loadtest command (#4853)
1 parent b1c400a commit e7dd3f9

23 files changed

+2641
-6
lines changed

‎cli/loadtest.go

+179
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
package cli
2+
3+
import (
4+
"bufio"
5+
"bytes"
6+
"context"
7+
"encoding/json"
8+
"fmt"
9+
"io"
10+
"os"
11+
"strconv"
12+
"time"
13+
14+
"github.com/spf13/cobra"
15+
"golang.org/x/xerrors"
16+
17+
"github.com/coder/coder/cli/cliflag"
18+
"github.com/coder/coder/codersdk"
19+
"github.com/coder/coder/loadtest/harness"
20+
)
21+
22+
func loadtest() *cobra.Command {
23+
var (
24+
configPath string
25+
)
26+
cmd := &cobra.Command{
27+
Use: "loadtest --config <path>",
28+
Short: "Load test the Coder API",
29+
// TODO: documentation and a JSON scheme file
30+
Long: "Perform load tests against the Coder server. The load tests " +
31+
"configurable via a JSON file.",
32+
Hidden: true,
33+
Args: cobra.ExactArgs(0),
34+
RunE: func(cmd *cobra.Command, args []string) error {
35+
if configPath == "" {
36+
return xerrors.New("config is required")
37+
}
38+
39+
var (
40+
configReader io.ReadCloser
41+
)
42+
if configPath == "-" {
43+
configReader = io.NopCloser(cmd.InOrStdin())
44+
} else {
45+
f, err := os.Open(configPath)
46+
if err != nil {
47+
return xerrors.Errorf("open config file %q: %w", configPath, err)
48+
}
49+
configReader = f
50+
}
51+
52+
var config LoadTestConfig
53+
err := json.NewDecoder(configReader).Decode(&config)
54+
_ = configReader.Close()
55+
if err != nil {
56+
return xerrors.Errorf("read config file %q: %w", configPath, err)
57+
}
58+
59+
err = config.Validate()
60+
if err != nil {
61+
return xerrors.Errorf("validate config: %w", err)
62+
}
63+
64+
client, err := CreateClient(cmd)
65+
if err != nil {
66+
return err
67+
}
68+
69+
me, err := client.User(cmd.Context(), codersdk.Me)
70+
if err != nil {
71+
return xerrors.Errorf("fetch current user: %w", err)
72+
}
73+
74+
// Only owners can do loadtests. This isn't a very strong check but
75+
// there's not much else we can do. Ratelimits are enforced for
76+
// non-owners so hopefully that limits the damage if someone
77+
// disables this check and runs it against a non-owner account.
78+
ok := false
79+
for _, role := range me.Roles {
80+
if role.Name == "owner" {
81+
ok = true
82+
break
83+
}
84+
}
85+
if !ok {
86+
return xerrors.Errorf("Not logged in as site owner. Load testing is only available to site owners.")
87+
}
88+
89+
// Disable ratelimits for future requests.
90+
client.BypassRatelimits = true
91+
92+
// Prepare the test.
93+
strategy := config.Strategy.ExecutionStrategy()
94+
th := harness.NewTestHarness(strategy)
95+
96+
for i, t := range config.Tests {
97+
name := fmt.Sprintf("%s-%d", t.Type, i)
98+
99+
for j := 0; j < t.Count; j++ {
100+
id := strconv.Itoa(j)
101+
runner, err := t.NewRunner(client)
102+
if err != nil {
103+
return xerrors.Errorf("create %q runner for %s/%s: %w", t.Type, name, id, err)
104+
}
105+
106+
th.AddRun(name, id, runner)
107+
}
108+
}
109+
110+
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "Running load test...")
111+
112+
testCtx := cmd.Context()
113+
if config.Timeout > 0 {
114+
var cancel func()
115+
testCtx, cancel = context.WithTimeout(testCtx, time.Duration(config.Timeout))
116+
defer cancel()
117+
}
118+
119+
// TODO: live progress output
120+
start := time.Now()
121+
err = th.Run(testCtx)
122+
if err != nil {
123+
return xerrors.Errorf("run test harness (harness failure, not a test failure): %w", err)
124+
}
125+
elapsed := time.Since(start)
126+
127+
// Print the results.
128+
// TODO: better result printing
129+
// TODO: move result printing to the loadtest package, add multiple
130+
// output formats (like HTML, JSON)
131+
res := th.Results()
132+
var totalDuration time.Duration
133+
for _, run := range res.Runs {
134+
totalDuration += run.Duration
135+
if run.Error == nil {
136+
continue
137+
}
138+
139+
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\n== FAIL: %s\n\n", run.FullID)
140+
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tError: %s\n\n", run.Error)
141+
142+
// Print log lines indented.
143+
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tLog:\n")
144+
rd := bufio.NewReader(bytes.NewBuffer(run.Logs))
145+
for {
146+
line, err := rd.ReadBytes('\n')
147+
if err == io.EOF {
148+
break
149+
}
150+
if err != nil {
151+
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\n\tLOG PRINT ERROR: %+v\n", err)
152+
}
153+
154+
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\t\t%s", line)
155+
}
156+
}
157+
158+
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\n\nTest results:")
159+
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tPass: %d\n", res.TotalPass)
160+
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tFail: %d\n", res.TotalFail)
161+
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tTotal: %d\n", res.TotalRuns)
162+
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "")
163+
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tTotal duration: %s\n", elapsed)
164+
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tAvg. duration: %s\n", totalDuration/time.Duration(res.TotalRuns))
165+
166+
// Cleanup.
167+
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\nCleaning up...")
168+
err = th.Cleanup(cmd.Context())
169+
if err != nil {
170+
return xerrors.Errorf("cleanup tests: %w", err)
171+
}
172+
173+
return nil
174+
},
175+
}
176+
177+
cliflag.StringVarP(cmd.Flags(), &configPath, "config", "", "CODER_LOADTEST_CONFIG_PATH", "", "Path to the load test configuration file, or - to read from stdin.")
178+
return cmd
179+
}

‎cli/loadtest_test.go

+136
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
package cli_test
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"encoding/json"
7+
"os"
8+
"path/filepath"
9+
"testing"
10+
"time"
11+
12+
"github.com/stretchr/testify/assert"
13+
"github.com/stretchr/testify/require"
14+
15+
"github.com/coder/coder/cli"
16+
"github.com/coder/coder/cli/clitest"
17+
"github.com/coder/coder/coderd/coderdtest"
18+
"github.com/coder/coder/coderd/httpapi"
19+
"github.com/coder/coder/codersdk"
20+
"github.com/coder/coder/loadtest/placebo"
21+
"github.com/coder/coder/loadtest/workspacebuild"
22+
"github.com/coder/coder/pty/ptytest"
23+
"github.com/coder/coder/testutil"
24+
)
25+
26+
func TestLoadTest(t *testing.T) {
27+
t.Parallel()
28+
29+
t.Run("PlaceboFromStdin", func(t *testing.T) {
30+
t.Parallel()
31+
32+
client := coderdtest.New(t, nil)
33+
_ = coderdtest.CreateFirstUser(t, client)
34+
35+
config := cli.LoadTestConfig{
36+
Strategy: cli.LoadTestStrategy{
37+
Type: cli.LoadTestStrategyTypeLinear,
38+
},
39+
Tests: []cli.LoadTest{
40+
{
41+
Type: cli.LoadTestTypePlacebo,
42+
Count: 10,
43+
Placebo: &placebo.Config{
44+
Sleep: httpapi.Duration(10 * time.Millisecond),
45+
},
46+
},
47+
},
48+
Timeout: httpapi.Duration(testutil.WaitShort),
49+
}
50+
51+
configBytes, err := json.Marshal(config)
52+
require.NoError(t, err)
53+
54+
cmd, root := clitest.New(t, "loadtest", "--config", "-")
55+
clitest.SetupConfig(t, client, root)
56+
pty := ptytest.New(t)
57+
cmd.SetIn(bytes.NewReader(configBytes))
58+
cmd.SetOut(pty.Output())
59+
cmd.SetErr(pty.Output())
60+
61+
ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitLong)
62+
defer cancelFunc()
63+
64+
done := make(chan any)
65+
go func() {
66+
errC := cmd.ExecuteContext(ctx)
67+
assert.NoError(t, errC)
68+
close(done)
69+
}()
70+
pty.ExpectMatch("Test results:")
71+
pty.ExpectMatch("Pass: 10")
72+
cancelFunc()
73+
<-done
74+
})
75+
76+
t.Run("WorkspaceBuildFromFile", func(t *testing.T) {
77+
t.Parallel()
78+
79+
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
80+
user := coderdtest.CreateFirstUser(t, client)
81+
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
82+
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
83+
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
84+
85+
config := cli.LoadTestConfig{
86+
Strategy: cli.LoadTestStrategy{
87+
Type: cli.LoadTestStrategyTypeConcurrent,
88+
ConcurrencyLimit: 2,
89+
},
90+
Tests: []cli.LoadTest{
91+
{
92+
Type: cli.LoadTestTypeWorkspaceBuild,
93+
Count: 2,
94+
WorkspaceBuild: &workspacebuild.Config{
95+
OrganizationID: user.OrganizationID,
96+
UserID: user.UserID.String(),
97+
Request: codersdk.CreateWorkspaceRequest{
98+
TemplateID: template.ID,
99+
},
100+
},
101+
},
102+
},
103+
Timeout: httpapi.Duration(testutil.WaitLong),
104+
}
105+
106+
d := t.TempDir()
107+
configPath := filepath.Join(d, "/config.loadtest.json")
108+
f, err := os.Create(configPath)
109+
require.NoError(t, err)
110+
defer f.Close()
111+
err = json.NewEncoder(f).Encode(config)
112+
require.NoError(t, err)
113+
_ = f.Close()
114+
115+
cmd, root := clitest.New(t, "loadtest", "--config", configPath)
116+
clitest.SetupConfig(t, client, root)
117+
pty := ptytest.New(t)
118+
cmd.SetIn(pty.Input())
119+
cmd.SetOut(pty.Output())
120+
cmd.SetErr(pty.Output())
121+
122+
ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitLong)
123+
defer cancelFunc()
124+
125+
done := make(chan any)
126+
go func() {
127+
errC := cmd.ExecuteContext(ctx)
128+
assert.NoError(t, errC)
129+
close(done)
130+
}()
131+
pty.ExpectMatch("Test results:")
132+
pty.ExpectMatch("Pass: 2")
133+
<-done
134+
cancelFunc()
135+
})
136+
}

0 commit comments

Comments
 (0)