Skip to content

Commit 5f099ea

Browse files
deansheatherdeansheather
authored
feat: loadtest output formats (#4928)
1 parent f918977 commit 5f099ea

File tree

4 files changed

+394
-81
lines changed

4 files changed

+394
-81
lines changed

‎cli/loadtest.go

Lines changed: 156 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,13 @@
11
package cli
22

33
import (
4-
"bufio"
5-
"bytes"
64
"context"
75
"encoding/json"
86
"fmt"
97
"io"
108
"os"
119
"strconv"
10+
"strings"
1211
"time"
1312

1413
"github.com/spf13/cobra"
@@ -21,44 +20,46 @@ import (
2120

2221
func loadtest() *cobra.Command {
2322
var (
24-
configPath string
23+
configPath string
24+
outputSpecs []string
2525
)
2626
cmd := &cobra.Command{
27-
Use: "loadtest --config <path>",
27+
Use: "loadtest --config <path> [--output json[:path]] [--output text[:path]]]",
2828
Short: "Load test the Coder API",
29-
// TODO: documentation and a JSON scheme file
30-
Long: "Perform load tests against the Coder server. The load tests " +
31-
"configurable via a JSON file.",
29+
// TODO: documentation and a JSON schema file
30+
Long: "Perform load tests against the Coder server. The load tests are configurable via a JSON file.",
31+
Example: formatExamples(
32+
example{
33+
Description: "Run a loadtest with the given configuration file",
34+
Command: "coder loadtest --config path/to/config.json",
35+
},
36+
example{
37+
Description: "Run a loadtest, reading the configuration from stdin",
38+
Command: "cat path/to/config.json | coder loadtest --config -",
39+
},
40+
example{
41+
Description: "Run a loadtest outputting JSON results instead",
42+
Command: "coder loadtest --config path/to/config.json --output json",
43+
},
44+
example{
45+
Description: "Run a loadtest outputting JSON results to a file",
46+
Command: "coder loadtest --config path/to/config.json --output json:path/to/results.json",
47+
},
48+
example{
49+
Description: "Run a loadtest outputting text results to stdout and JSON results to a file",
50+
Command: "coder loadtest --config path/to/config.json --output text --output json:path/to/results.json",
51+
},
52+
),
3253
Hidden: true,
3354
Args: cobra.ExactArgs(0),
3455
RunE: func(cmd *cobra.Command, args []string) error {
35-
if configPath == "" {
36-
return xerrors.New("config is required")
37-
}
38-
39-
var (
40-
configReader io.ReadCloser
41-
)
42-
if configPath == "-" {
43-
configReader = io.NopCloser(cmd.InOrStdin())
44-
} else {
45-
f, err := os.Open(configPath)
46-
if err != nil {
47-
return xerrors.Errorf("open config file %q: %w", configPath, err)
48-
}
49-
configReader = f
50-
}
51-
52-
var config LoadTestConfig
53-
err := json.NewDecoder(configReader).Decode(&config)
54-
_ = configReader.Close()
56+
config, err := loadLoadTestConfigFile(configPath, cmd.InOrStdin())
5557
if err != nil {
56-
return xerrors.Errorf("read config file %q: %w", configPath, err)
58+
return err
5759
}
58-
59-
err = config.Validate()
60+
outputs, err := parseLoadTestOutputs(outputSpecs)
6061
if err != nil {
61-
return xerrors.Errorf("validate config: %w", err)
62+
return err
6263
}
6364

6465
client, err := CreateClient(cmd)
@@ -117,63 +118,156 @@ func loadtest() *cobra.Command {
117118
}
118119

119120
// TODO: live progress output
120-
start := time.Now()
121121
err = th.Run(testCtx)
122122
if err != nil {
123123
return xerrors.Errorf("run test harness (harness failure, not a test failure): %w", err)
124124
}
125-
elapsed := time.Since(start)
126125

127126
// Print the results.
128-
// TODO: better result printing
129-
// TODO: move result printing to the loadtest package, add multiple
130-
// output formats (like HTML, JSON)
131127
res := th.Results()
132-
var totalDuration time.Duration
133-
for _, run := range res.Runs {
134-
totalDuration += run.Duration
135-
if run.Error == nil {
136-
continue
128+
for _, output := range outputs {
129+
var (
130+
w = cmd.OutOrStdout()
131+
c io.Closer
132+
)
133+
if output.path != "-" {
134+
f, err := os.Create(output.path)
135+
if err != nil {
136+
return xerrors.Errorf("create output file: %w", err)
137+
}
138+
w, c = f, f
137139
}
138140

139-
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\n== FAIL: %s\n\n", run.FullID)
140-
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tError: %s\n\n", run.Error)
141-
142-
// Print log lines indented.
143-
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tLog:\n")
144-
rd := bufio.NewReader(bytes.NewBuffer(run.Logs))
145-
for {
146-
line, err := rd.ReadBytes('\n')
147-
if err == io.EOF {
148-
break
149-
}
141+
switch output.format {
142+
case loadTestOutputFormatText:
143+
res.PrintText(w)
144+
case loadTestOutputFormatJSON:
145+
err = json.NewEncoder(w).Encode(res)
150146
if err != nil {
151-
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\n\tLOG PRINT ERROR: %+v\n", err)
147+
return xerrors.Errorf("encode JSON: %w", err)
152148
}
149+
}
153150

154-
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\t\t%s", line)
151+
if c != nil {
152+
err = c.Close()
153+
if err != nil {
154+
return xerrors.Errorf("close output file: %w", err)
155+
}
155156
}
156157
}
157158

158-
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\n\nTest results:")
159-
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tPass: %d\n", res.TotalPass)
160-
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tFail: %d\n", res.TotalFail)
161-
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tTotal: %d\n", res.TotalRuns)
162-
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "")
163-
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tTotal duration: %s\n", elapsed)
164-
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tAvg. duration: %s\n", totalDuration/time.Duration(res.TotalRuns))
165-
166159
// Cleanup.
167160
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\nCleaning up...")
168161
err = th.Cleanup(cmd.Context())
169162
if err != nil {
170163
return xerrors.Errorf("cleanup tests: %w", err)
171164
}
172165

166+
if res.TotalFail > 0 {
167+
return xerrors.New("load test failed, see above for more details")
168+
}
169+
173170
return nil
174171
},
175172
}
176173

177174
cliflag.StringVarP(cmd.Flags(), &configPath, "config", "", "CODER_LOADTEST_CONFIG_PATH", "", "Path to the load test configuration file, or - to read from stdin.")
175+
cliflag.StringArrayVarP(cmd.Flags(), &outputSpecs, "output", "", "CODER_LOADTEST_OUTPUTS", []string{"text"}, "Output formats, see usage for more information.")
178176
return cmd
179177
}
178+
179+
func loadLoadTestConfigFile(configPath string, stdin io.Reader) (LoadTestConfig, error) {
180+
if configPath == "" {
181+
return LoadTestConfig{}, xerrors.New("config is required")
182+
}
183+
184+
var (
185+
configReader io.ReadCloser
186+
)
187+
if configPath == "-" {
188+
configReader = io.NopCloser(stdin)
189+
} else {
190+
f, err := os.Open(configPath)
191+
if err != nil {
192+
return LoadTestConfig{}, xerrors.Errorf("open config file %q: %w", configPath, err)
193+
}
194+
configReader = f
195+
}
196+
197+
var config LoadTestConfig
198+
err := json.NewDecoder(configReader).Decode(&config)
199+
_ = configReader.Close()
200+
if err != nil {
201+
return LoadTestConfig{}, xerrors.Errorf("read config file %q: %w", configPath, err)
202+
}
203+
204+
err = config.Validate()
205+
if err != nil {
206+
return LoadTestConfig{}, xerrors.Errorf("validate config: %w", err)
207+
}
208+
209+
return config, nil
210+
}
211+
212+
type loadTestOutputFormat string
213+
214+
const (
215+
loadTestOutputFormatText loadTestOutputFormat = "text"
216+
loadTestOutputFormatJSON loadTestOutputFormat = "json"
217+
// TODO: html format
218+
)
219+
220+
type loadTestOutput struct {
221+
format loadTestOutputFormat
222+
// Up to one path (the first path) will have the value "-" which signifies
223+
// stdout.
224+
path string
225+
}
226+
227+
func parseLoadTestOutputs(outputs []string) ([]loadTestOutput, error) {
228+
var stdoutFormat loadTestOutputFormat
229+
230+
validFormats := map[loadTestOutputFormat]struct{}{
231+
loadTestOutputFormatText: {},
232+
loadTestOutputFormatJSON: {},
233+
}
234+
235+
var out []loadTestOutput
236+
for i, o := range outputs {
237+
parts := strings.SplitN(o, ":", 2)
238+
format := loadTestOutputFormat(parts[0])
239+
if _, ok := validFormats[format]; !ok {
240+
return nil, xerrors.Errorf("invalid output format %q in output flag %d", parts[0], i)
241+
}
242+
243+
if len(parts) == 1 {
244+
if stdoutFormat != "" {
245+
return nil, xerrors.Errorf("multiple output flags specified for stdout")
246+
}
247+
stdoutFormat = format
248+
continue
249+
}
250+
if len(parts) != 2 {
251+
return nil, xerrors.Errorf("invalid output flag %d: %q", i, o)
252+
}
253+
254+
out = append(out, loadTestOutput{
255+
format: format,
256+
path: parts[1],
257+
})
258+
}
259+
260+
// Default to --output text
261+
if stdoutFormat == "" && len(out) == 0 {
262+
stdoutFormat = loadTestOutputFormatText
263+
}
264+
265+
if stdoutFormat != "" {
266+
out = append([]loadTestOutput{{
267+
format: stdoutFormat,
268+
path: "-",
269+
}}, out...)
270+
}
271+
272+
return out, nil
273+
}

0 commit comments

Comments
 (0)