From 6c767324e9708600888954ce8a315316c8e5a5da Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 Oct 2023 12:13:12 +0000 Subject: [PATCH 1/3] fix(scaletest): output error and trace instead of {} for json output --- scaletest/harness/results.go | 13 +++++ scaletest/harness/results_test.go | 87 +++++++++++++++++++++++++++++-- 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/scaletest/harness/results.go b/scaletest/harness/results.go index a290f807b8097..f9d626e3960aa 100644 --- a/scaletest/harness/results.go +++ b/scaletest/harness/results.go @@ -2,6 +2,7 @@ package harness import ( "bufio" + "encoding/json" "fmt" "io" "strings" @@ -33,6 +34,18 @@ type RunResult struct { DurationMS int64 `json:"duration_ms"` } +// MarshalJSON implements json.Marhshaler for RunResult. +func (r RunResult) MarshalJSON() ([]byte, error) { + type alias RunResult + return json.Marshal(&struct { + alias + Error string `json:"error"` + }{ + alias: alias(r), + Error: fmt.Sprintf("%+v", r.Error), + }) +} + // Results returns the results of the test run. Panics if the test run is not // done yet. func (r *TestRun) Result() RunResult { diff --git a/scaletest/harness/results_test.go b/scaletest/harness/results_test.go index d16bb694b7889..31ec80b1393e2 100644 --- a/scaletest/harness/results_test.go +++ b/scaletest/harness/results_test.go @@ -2,9 +2,12 @@ package harness_test import ( "bytes" + "encoding/json" "testing" "time" + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/xerrors" @@ -12,9 +15,18 @@ import ( "github.com/coder/coder/v2/scaletest/harness" ) +type testError struct { + hidden error +} + +func (e testError) Error() string { + return e.hidden.Error() +} + func Test_Results(t *testing.T) { t.Parallel() + now := time.Date(2023, 10, 5, 12, 3, 56, 395813665, time.UTC) results := harness.Results{ TotalRuns: 10, TotalPass: 8, @@ -26,7 +38,7 @@ func Test_Results(t *testing.T) { ID: "0", Logs: "test-0/0 log line 1\ntest-0/0 log line 2", Error: xerrors.New("test-0/0 error"), - StartedAt: time.Now(), + StartedAt: now, Duration: httpapi.Duration(time.Second), DurationMS: 1000, }, @@ -36,7 +48,17 @@ func Test_Results(t *testing.T) { ID: "1", Logs: "test-0/1 log line 1\ntest-0/1 log line 2", Error: nil, - StartedAt: time.Now(), + StartedAt: now.Add(333 * time.Millisecond), + Duration: httpapi.Duration(time.Second), + DurationMS: 1000, + }, + "test-0/2": { + FullID: "test-0/2", + TestName: "test-0", + ID: "2", + Logs: "test-0/2 log line 1\ntest-0/2 log line 2", + Error: testError{hidden: xerrors.New("test-0/2 error")}, + StartedAt: now.Add(666 * time.Millisecond), Duration: httpapi.Duration(time.Second), DurationMS: 1000, }, @@ -45,7 +67,7 @@ func Test_Results(t *testing.T) { ElapsedMS: 1000, } - expected := ` + wantText := ` == FAIL: test-0/0 Error: test-0/0 error @@ -53,6 +75,13 @@ func Test_Results(t *testing.T) { Log: test-0/0 log line 1 +== FAIL: test-0/2 + + Error: test-0/2 error + + Log: + test-0/2 log line 1 + Test results: Pass: 8 @@ -60,11 +89,59 @@ Test results: Total: 10 Total duration: 1s - Avg. duration: 200ms + Avg. duration: 300ms +` + wantJSON := `{ + "total_runs": 10, + "total_pass": 8, + "total_fail": 2, + "elapsed": "1s", + "elapsed_ms": 1000, + "runs": { + "test-0/0": { + "full_id": "test-0/0", + "test_name": "test-0", + "id": "0", + "logs": "test-0/0 log line 1\ntest-0/0 log line 2", + "started_at": "2023-10-05T12:03:56.395813665Z", + "duration": "1s", + "duration_ms": 1000, + "error": "test-0/0 error:\n github.com/coder/coder/v2/scaletest/harness_test.Test_Results\n /home/coder/src/coder/coder/scaletest/harness/results_test.go:40" + }, + "test-0/1": { + "full_id": "test-0/1", + "test_name": "test-0", + "id": "1", + "logs": "test-0/1 log line 1\ntest-0/1 log line 2", + "started_at": "2023-10-05T12:03:56.728813665Z", + "duration": "1s", + "duration_ms": 1000, + "error": "\u003cnil\u003e" + }, + "test-0/2": { + "full_id": "test-0/2", + "test_name": "test-0", + "id": "2", + "logs": "test-0/2 log line 1\ntest-0/2 log line 2", + "started_at": "2023-10-05T12:03:57.061813665Z", + "duration": "1s", + "duration_ms": 1000, + "error": "test-0/2 error" + } + } +} ` out := bytes.NewBuffer(nil) results.PrintText(out) - require.Equal(t, expected, out.String()) + assert.Empty(t, cmp.Diff(wantText, out.String()), "text result does not match (-want +got)") + + out.Reset() + enc := json.NewEncoder(out) + enc.SetIndent("", "\t") + err := enc.Encode(results) + require.NoError(t, err) + + assert.Empty(t, cmp.Diff(wantJSON, out.String()), "JSON result does not match (-want +got)") } From bc6278c6c532e11db6c6c18edb7f95babef6490a Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 Oct 2023 12:37:21 +0000 Subject: [PATCH 2/3] deterministic output --- scaletest/harness/results.go | 7 ++++++- scaletest/harness/results_test.go | 10 ++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/scaletest/harness/results.go b/scaletest/harness/results.go index f9d626e3960aa..494102fb53022 100644 --- a/scaletest/harness/results.go +++ b/scaletest/harness/results.go @@ -5,10 +5,12 @@ import ( "encoding/json" "fmt" "io" + "sort" "strings" "time" "github.com/coder/coder/v2/coderd/httpapi" + "golang.org/x/exp/maps" ) // Results is the full compiled results for a set of test runs. @@ -101,7 +103,10 @@ func (h *TestHarness) Results() Results { // PrintText prints the results as human-readable text to the given writer. func (r *Results) PrintText(w io.Writer) { var totalDuration time.Duration - for _, run := range r.Runs { + keys := maps.Keys(r.Runs) + sort.Strings(keys) + for _, key := range keys { + run := r.Runs[key] totalDuration += time.Duration(run.Duration) if run.Error == nil { continue diff --git a/scaletest/harness/results_test.go b/scaletest/harness/results_test.go index 31ec80b1393e2..a7d025036923e 100644 --- a/scaletest/harness/results_test.go +++ b/scaletest/harness/results_test.go @@ -3,6 +3,9 @@ package harness_test import ( "bytes" "encoding/json" + "os" + "path/filepath" + "strings" "testing" "time" @@ -106,7 +109,7 @@ Test results: "started_at": "2023-10-05T12:03:56.395813665Z", "duration": "1s", "duration_ms": 1000, - "error": "test-0/0 error:\n github.com/coder/coder/v2/scaletest/harness_test.Test_Results\n /home/coder/src/coder/coder/scaletest/harness/results_test.go:40" + "error": "test-0/0 error:\n github.com/coder/coder/v2/scaletest/harness_test.Test_Results\n [working_directory]results_test.go:43" }, "test-0/1": { "full_id": "test-0/1", @@ -131,6 +134,9 @@ Test results: } } ` + wd, err := os.Getwd() + require.NoError(t, err) + wantJSON = strings.Replace(wantJSON, "[working_directory]", wd+string(filepath.Separator), 1) out := bytes.NewBuffer(nil) results.PrintText(out) @@ -140,7 +146,7 @@ Test results: out.Reset() enc := json.NewEncoder(out) enc.SetIndent("", "\t") - err := enc.Encode(results) + err = enc.Encode(results) require.NoError(t, err) assert.Empty(t, cmp.Diff(wantJSON, out.String()), "JSON result does not match (-want +got)") From 345e25aa2ec8bb20a1bc1da322cf08bd53ff6f30 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 Oct 2023 13:15:10 +0000 Subject: [PATCH 3/3] hello windows --- scaletest/harness/results.go | 3 ++- scaletest/harness/results_test.go | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scaletest/harness/results.go b/scaletest/harness/results.go index 494102fb53022..a96212f9feb51 100644 --- a/scaletest/harness/results.go +++ b/scaletest/harness/results.go @@ -9,8 +9,9 @@ import ( "strings" "time" - "github.com/coder/coder/v2/coderd/httpapi" "golang.org/x/exp/maps" + + "github.com/coder/coder/v2/coderd/httpapi" ) // Results is the full compiled results for a set of test runs. diff --git a/scaletest/harness/results_test.go b/scaletest/harness/results_test.go index a7d025036923e..65eea6c2c44f9 100644 --- a/scaletest/harness/results_test.go +++ b/scaletest/harness/results_test.go @@ -109,7 +109,7 @@ Test results: "started_at": "2023-10-05T12:03:56.395813665Z", "duration": "1s", "duration_ms": 1000, - "error": "test-0/0 error:\n github.com/coder/coder/v2/scaletest/harness_test.Test_Results\n [working_directory]results_test.go:43" + "error": "test-0/0 error:\n github.com/coder/coder/v2/scaletest/harness_test.Test_Results\n [working_directory]/results_test.go:43" }, "test-0/1": { "full_id": "test-0/1", @@ -136,7 +136,8 @@ Test results: ` wd, err := os.Getwd() require.NoError(t, err) - wantJSON = strings.Replace(wantJSON, "[working_directory]", wd+string(filepath.Separator), 1) + wd = filepath.ToSlash(wd) // Hello there Windows, my friend... + wantJSON = strings.Replace(wantJSON, "[working_directory]", wd, 1) out := bytes.NewBuffer(nil) results.PrintText(out)