From ae8df1378e33e8917a2da96538ef2b9223192ec9 Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Mon, 7 Nov 2022 15:37:56 +0000 Subject: [PATCH] feat: loadtest output formats --- cli/loadtest.go | 218 ++++++++++++++++++++++++++---------- cli/loadtest_test.go | 159 ++++++++++++++++++++++++++ loadtest/harness/harness.go | 9 ++ loadtest/harness/results.go | 89 +++++++++++---- 4 files changed, 394 insertions(+), 81 deletions(-) diff --git a/cli/loadtest.go b/cli/loadtest.go index ac811366a1152..fcec6019e8c91 100644 --- a/cli/loadtest.go +++ b/cli/loadtest.go @@ -1,14 +1,13 @@ package cli import ( - "bufio" - "bytes" "context" "encoding/json" "fmt" "io" "os" "strconv" + "strings" "time" "github.com/spf13/cobra" @@ -21,44 +20,46 @@ import ( func loadtest() *cobra.Command { var ( - configPath string + configPath string + outputSpecs []string ) cmd := &cobra.Command{ - Use: "loadtest --config ", + Use: "loadtest --config [--output json[:path]] [--output text[:path]]]", Short: "Load test the Coder API", - // TODO: documentation and a JSON scheme file - Long: "Perform load tests against the Coder server. The load tests " + - "configurable via a JSON file.", + // TODO: documentation and a JSON schema file + Long: "Perform load tests against the Coder server. The load tests are configurable via a JSON file.", + Example: formatExamples( + example{ + Description: "Run a loadtest with the given configuration file", + Command: "coder loadtest --config path/to/config.json", + }, + example{ + Description: "Run a loadtest, reading the configuration from stdin", + Command: "cat path/to/config.json | coder loadtest --config -", + }, + example{ + Description: "Run a loadtest outputting JSON results instead", + Command: "coder loadtest --config path/to/config.json --output json", + }, + example{ + Description: "Run a loadtest outputting JSON results to a file", + Command: "coder loadtest --config path/to/config.json --output json:path/to/results.json", + }, + example{ + Description: "Run a loadtest outputting text results to stdout and JSON results to a file", + Command: "coder loadtest --config path/to/config.json --output text --output json:path/to/results.json", + }, + ), Hidden: true, Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, args []string) error { - if configPath == "" { - return xerrors.New("config is required") - } - - var ( - configReader io.ReadCloser - ) - if configPath == "-" { - configReader = io.NopCloser(cmd.InOrStdin()) - } else { - f, err := os.Open(configPath) - if err != nil { - return xerrors.Errorf("open config file %q: %w", configPath, err) - } - configReader = f - } - - var config LoadTestConfig - err := json.NewDecoder(configReader).Decode(&config) - _ = configReader.Close() + config, err := loadLoadTestConfigFile(configPath, cmd.InOrStdin()) if err != nil { - return xerrors.Errorf("read config file %q: %w", configPath, err) + return err } - - err = config.Validate() + outputs, err := parseLoadTestOutputs(outputSpecs) if err != nil { - return xerrors.Errorf("validate config: %w", err) + return err } client, err := CreateClient(cmd) @@ -117,52 +118,44 @@ func loadtest() *cobra.Command { } // TODO: live progress output - start := time.Now() err = th.Run(testCtx) if err != nil { return xerrors.Errorf("run test harness (harness failure, not a test failure): %w", err) } - elapsed := time.Since(start) // Print the results. - // TODO: better result printing - // TODO: move result printing to the loadtest package, add multiple - // output formats (like HTML, JSON) res := th.Results() - var totalDuration time.Duration - for _, run := range res.Runs { - totalDuration += run.Duration - if run.Error == nil { - continue + for _, output := range outputs { + var ( + w = cmd.OutOrStdout() + c io.Closer + ) + if output.path != "-" { + f, err := os.Create(output.path) + if err != nil { + return xerrors.Errorf("create output file: %w", err) + } + w, c = f, f } - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\n== FAIL: %s\n\n", run.FullID) - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tError: %s\n\n", run.Error) - - // Print log lines indented. - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tLog:\n") - rd := bufio.NewReader(bytes.NewBuffer(run.Logs)) - for { - line, err := rd.ReadBytes('\n') - if err == io.EOF { - break - } + switch output.format { + case loadTestOutputFormatText: + res.PrintText(w) + case loadTestOutputFormatJSON: + err = json.NewEncoder(w).Encode(res) if err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\n\tLOG PRINT ERROR: %+v\n", err) + return xerrors.Errorf("encode JSON: %w", err) } + } - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\t\t%s", line) + if c != nil { + err = c.Close() + if err != nil { + return xerrors.Errorf("close output file: %w", err) + } } } - _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\n\nTest results:") - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tPass: %d\n", res.TotalPass) - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tFail: %d\n", res.TotalFail) - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tTotal: %d\n", res.TotalRuns) - _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "") - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tTotal duration: %s\n", elapsed) - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tAvg. duration: %s\n", totalDuration/time.Duration(res.TotalRuns)) - // Cleanup. _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\nCleaning up...") err = th.Cleanup(cmd.Context()) @@ -170,10 +163,111 @@ func loadtest() *cobra.Command { return xerrors.Errorf("cleanup tests: %w", err) } + if res.TotalFail > 0 { + return xerrors.New("load test failed, see above for more details") + } + return nil }, } cliflag.StringVarP(cmd.Flags(), &configPath, "config", "", "CODER_LOADTEST_CONFIG_PATH", "", "Path to the load test configuration file, or - to read from stdin.") + cliflag.StringArrayVarP(cmd.Flags(), &outputSpecs, "output", "", "CODER_LOADTEST_OUTPUTS", []string{"text"}, "Output formats, see usage for more information.") return cmd } + +func loadLoadTestConfigFile(configPath string, stdin io.Reader) (LoadTestConfig, error) { + if configPath == "" { + return LoadTestConfig{}, xerrors.New("config is required") + } + + var ( + configReader io.ReadCloser + ) + if configPath == "-" { + configReader = io.NopCloser(stdin) + } else { + f, err := os.Open(configPath) + if err != nil { + return LoadTestConfig{}, xerrors.Errorf("open config file %q: %w", configPath, err) + } + configReader = f + } + + var config LoadTestConfig + err := json.NewDecoder(configReader).Decode(&config) + _ = configReader.Close() + if err != nil { + return LoadTestConfig{}, xerrors.Errorf("read config file %q: %w", configPath, err) + } + + err = config.Validate() + if err != nil { + return LoadTestConfig{}, xerrors.Errorf("validate config: %w", err) + } + + return config, nil +} + +type loadTestOutputFormat string + +const ( + loadTestOutputFormatText loadTestOutputFormat = "text" + loadTestOutputFormatJSON loadTestOutputFormat = "json" + // TODO: html format +) + +type loadTestOutput struct { + format loadTestOutputFormat + // Up to one path (the first path) will have the value "-" which signifies + // stdout. + path string +} + +func parseLoadTestOutputs(outputs []string) ([]loadTestOutput, error) { + var stdoutFormat loadTestOutputFormat + + validFormats := map[loadTestOutputFormat]struct{}{ + loadTestOutputFormatText: {}, + loadTestOutputFormatJSON: {}, + } + + var out []loadTestOutput + for i, o := range outputs { + parts := strings.SplitN(o, ":", 2) + format := loadTestOutputFormat(parts[0]) + if _, ok := validFormats[format]; !ok { + return nil, xerrors.Errorf("invalid output format %q in output flag %d", parts[0], i) + } + + if len(parts) == 1 { + if stdoutFormat != "" { + return nil, xerrors.Errorf("multiple output flags specified for stdout") + } + stdoutFormat = format + continue + } + if len(parts) != 2 { + return nil, xerrors.Errorf("invalid output flag %d: %q", i, o) + } + + out = append(out, loadTestOutput{ + format: format, + path: parts[1], + }) + } + + // Default to --output text + if stdoutFormat == "" && len(out) == 0 { + stdoutFormat = loadTestOutputFormatText + } + + if stdoutFormat != "" { + out = append([]loadTestOutput{{ + format: stdoutFormat, + path: "-", + }}, out...) + } + + return out, nil +} diff --git a/cli/loadtest_test.go b/cli/loadtest_test.go index 44a0cd6b69309..544031e92f3e2 100644 --- a/cli/loadtest_test.go +++ b/cli/loadtest_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "os" "path/filepath" "testing" @@ -17,6 +18,7 @@ import ( "github.com/coder/coder/coderd/coderdtest" "github.com/coder/coder/coderd/httpapi" "github.com/coder/coder/codersdk" + "github.com/coder/coder/loadtest/harness" "github.com/coder/coder/loadtest/placebo" "github.com/coder/coder/loadtest/workspacebuild" "github.com/coder/coder/pty/ptytest" @@ -133,4 +135,161 @@ func TestLoadTest(t *testing.T) { <-done cancelFunc() }) + + t.Run("OutputFormats", func(t *testing.T) { + t.Parallel() + + type outputFlag struct { + format string + path string + } + + dir := t.TempDir() + + cases := []struct { + name string + outputs []outputFlag + errContains string + }{ + { + name: "Default", + outputs: []outputFlag{}, + }, + { + name: "ExplicitText", + outputs: []outputFlag{{format: "text"}}, + }, + { + name: "JSON", + outputs: []outputFlag{ + { + format: "json", + path: filepath.Join(dir, "results.json"), + }, + }, + }, + { + name: "TextAndJSON", + outputs: []outputFlag{ + { + format: "text", + }, + { + format: "json", + path: filepath.Join(dir, "results.json"), + }, + }, + }, + { + name: "TextAndJSON2", + outputs: []outputFlag{ + { + format: "text", + }, + { + format: "text", + path: filepath.Join(dir, "results.txt"), + }, + { + format: "json", + path: filepath.Join(dir, "results.json"), + }, + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + t.Parallel() + + client := coderdtest.New(t, nil) + _ = coderdtest.CreateFirstUser(t, client) + + config := cli.LoadTestConfig{ + Strategy: cli.LoadTestStrategy{ + Type: cli.LoadTestStrategyTypeLinear, + }, + Tests: []cli.LoadTest{ + { + Type: cli.LoadTestTypePlacebo, + Count: 10, + Placebo: &placebo.Config{ + Sleep: httpapi.Duration(10 * time.Millisecond), + }, + }, + }, + Timeout: httpapi.Duration(testutil.WaitShort), + } + + configBytes, err := json.Marshal(config) + require.NoError(t, err) + + args := []string{"loadtest", "--config", "-"} + for _, output := range c.outputs { + flag := output.format + if output.path != "" { + flag += ":" + output.path + } + args = append(args, "--output", flag) + } + + cmd, root := clitest.New(t, args...) + clitest.SetupConfig(t, client, root) + cmd.SetIn(bytes.NewReader(configBytes)) + out := bytes.NewBuffer(nil) + cmd.SetOut(out) + pty := ptytest.New(t) + cmd.SetErr(pty.Output()) + + ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitLong) + defer cancelFunc() + + done := make(chan any) + go func() { + errC := cmd.ExecuteContext(ctx) + if c.errContains != "" { + assert.Error(t, errC) + assert.Contains(t, errC.Error(), c.errContains) + } else { + assert.NoError(t, errC) + } + close(done) + }() + + <-done + + if c.errContains != "" { + return + } + if len(c.outputs) == 0 { + // This is the default output format when no flags are + // specified. + c.outputs = []outputFlag{{format: "text"}} + } + for i, output := range c.outputs { + msg := fmt.Sprintf("flag %d", i) + var b []byte + if output.path == "" { + b = out.Bytes() + } else { + b, err = os.ReadFile(output.path) + require.NoError(t, err, msg) + } + + switch output.format { + case "text": + require.Contains(t, string(b), "Test results:", msg) + require.Contains(t, string(b), "Pass: 10", msg) + case "json": + var res harness.Results + err = json.Unmarshal(b, &res) + require.NoError(t, err, msg) + require.Equal(t, 10, res.TotalRuns, msg) + require.Equal(t, 10, res.TotalPass, msg) + require.Len(t, res.Runs, 10, msg) + } + } + }) + } + }) } diff --git a/loadtest/harness/harness.go b/loadtest/harness/harness.go index 435a7406b8e61..c9d200a7d8f19 100644 --- a/loadtest/harness/harness.go +++ b/loadtest/harness/harness.go @@ -3,6 +3,7 @@ package harness import ( "context" "sync" + "time" "github.com/hashicorp/go-multierror" "golang.org/x/xerrors" @@ -27,6 +28,7 @@ type TestHarness struct { runs []*TestRun started bool done chan struct{} + elapsed time.Duration } // NewTestHarness creates a new TestHarness with the given ExecutionStrategy. @@ -63,6 +65,13 @@ func (h *TestHarness) Run(ctx context.Context) (err error) { } }() + start := time.Now() + defer func() { + h.mut.Lock() + defer h.mut.Unlock() + h.elapsed = time.Since(start) + }() + err = h.strategy.Execute(ctx, h.runs) //nolint:revive // we use named returns because we mutate it in a defer return diff --git a/loadtest/harness/results.go b/loadtest/harness/results.go index 615b6861634a6..3bb1c02596688 100644 --- a/loadtest/harness/results.go +++ b/loadtest/harness/results.go @@ -1,25 +1,36 @@ package harness -import "time" +import ( + "bufio" + "fmt" + "io" + "strings" + "time" + + "github.com/coder/coder/coderd/httpapi" +) // Results is the full compiled results for a set of test runs. type Results struct { - TotalRuns int - TotalPass int - TotalFail int + TotalRuns int `json:"total_runs"` + TotalPass int `json:"total_pass"` + TotalFail int `json:"total_fail"` + Elapsed httpapi.Duration `json:"elapsed"` + ElapsedMS int64 `json:"elapsed_ms"` - Runs map[string]RunResult + Runs map[string]RunResult `json:"runs"` } // RunResult is the result of a single test run. type RunResult struct { - FullID string - TestName string - ID string - Logs []byte - Error error - StartedAt time.Time - Duration time.Duration + FullID string `json:"full_id"` + TestName string `json:"test_name"` + ID string `json:"id"` + Logs string `json:"logs"` + Error error `json:"error"` + StartedAt time.Time `json:"started_at"` + Duration httpapi.Duration `json:"duration"` + DurationMS int64 `json:"duration_ms"` } // Results returns the results of the test run. Panics if the test run is not @@ -32,13 +43,14 @@ func (r *TestRun) Result() RunResult { } return RunResult{ - FullID: r.FullID(), - TestName: r.testName, - ID: r.id, - Logs: r.logs.Bytes(), - Error: r.err, - StartedAt: r.started, - Duration: r.duration, + FullID: r.FullID(), + TestName: r.testName, + ID: r.id, + Logs: r.logs.String(), + Error: r.err, + StartedAt: r.started, + Duration: httpapi.Duration(r.duration), + DurationMS: r.duration.Milliseconds(), } } @@ -56,6 +68,8 @@ func (h *TestHarness) Results() Results { results := Results{ TotalRuns: len(h.runs), Runs: make(map[string]RunResult, len(h.runs)), + Elapsed: httpapi.Duration(h.elapsed), + ElapsedMS: h.elapsed.Milliseconds(), } for _, run := range h.runs { runRes := run.Result() @@ -70,3 +84,40 @@ func (h *TestHarness) Results() Results { return results } + +// PrintText prints the results as human-readable text to the given writer. +func (r *Results) PrintText(w io.Writer) { + var totalDuration time.Duration + for _, run := range r.Runs { + totalDuration += time.Duration(run.Duration) + if run.Error == nil { + continue + } + + _, _ = fmt.Fprintf(w, "\n== FAIL: %s\n\n", run.FullID) + _, _ = fmt.Fprintf(w, "\tError: %s\n\n", run.Error) + + // Print log lines indented. + _, _ = fmt.Fprintf(w, "\tLog:\n") + rd := bufio.NewReader(strings.NewReader(run.Logs)) + for { + line, err := rd.ReadBytes('\n') + if err == io.EOF { + break + } + if err != nil { + _, _ = fmt.Fprintf(w, "\n\tLOG PRINT ERROR: %+v\n", err) + } + + _, _ = fmt.Fprintf(w, "\t\t%s", line) + } + } + + _, _ = fmt.Fprintln(w, "\n\nTest results:") + _, _ = fmt.Fprintf(w, "\tPass: %d\n", r.TotalPass) + _, _ = fmt.Fprintf(w, "\tFail: %d\n", r.TotalFail) + _, _ = fmt.Fprintf(w, "\tTotal: %d\n", r.TotalRuns) + _, _ = fmt.Fprintln(w, "") + _, _ = fmt.Fprintf(w, "\tTotal duration: %s\n", time.Duration(r.Elapsed)) + _, _ = fmt.Fprintf(w, "\tAvg. duration: %s\n", totalDuration/time.Duration(r.TotalRuns)) +}