Skip to content

Commit 5d5a7da

Browse files
mafredrimafredri
authored
fix(scaletest): output error and trace instead of {} for json output (#10075)
1 parent ab9276b commit 5d5a7da

File tree

2 files changed

+109
-6
lines changed

2 files changed

+109
-6
lines changed

‎scaletest/harness/results.go

+20-1
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,15 @@ package harness
22

33
import (
44
"bufio"
5+
"encoding/json"
56
"fmt"
67
"io"
8+
"sort"
79
"strings"
810
"time"
911

12+
"golang.org/x/exp/maps"
13+
1014
"github.com/coder/coder/v2/coderd/httpapi"
1115
)
1216

@@ -33,6 +37,18 @@ type RunResult struct {
3337
DurationMS int64 `json:"duration_ms"`
3438
}
3539

40+
// MarshalJSON implements json.Marhshaler for RunResult.
41+
func (r RunResult) MarshalJSON() ([]byte, error) {
42+
type alias RunResult
43+
return json.Marshal(&struct {
44+
alias
45+
Error string `json:"error"`
46+
}{
47+
alias: alias(r),
48+
Error: fmt.Sprintf("%+v", r.Error),
49+
})
50+
}
51+
3652
// Results returns the results of the test run. Panics if the test run is not
3753
// done yet.
3854
func (r *TestRun) Result() RunResult {
@@ -88,7 +104,10 @@ func (h *TestHarness) Results() Results {
88104
// PrintText prints the results as human-readable text to the given writer.
89105
func (r *Results) PrintText(w io.Writer) {
90106
var totalDuration time.Duration
91-
for _, run := range r.Runs {
107+
keys := maps.Keys(r.Runs)
108+
sort.Strings(keys)
109+
for _, key := range keys {
110+
run := r.Runs[key]
92111
totalDuration += time.Duration(run.Duration)
93112
if run.Error == nil {
94113
continue

‎scaletest/harness/results_test.go

+89-5
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,34 @@ package harness_test
22

33
import (
44
"bytes"
5+
"encoding/json"
6+
"os"
7+
"path/filepath"
8+
"strings"
59
"testing"
610
"time"
711

12+
"github.com/google/go-cmp/cmp"
13+
"github.com/stretchr/testify/assert"
814
"github.com/stretchr/testify/require"
915
"golang.org/x/xerrors"
1016

1117
"github.com/coder/coder/v2/coderd/httpapi"
1218
"github.com/coder/coder/v2/scaletest/harness"
1319
)
1420

21+
type testError struct {
22+
hidden error
23+
}
24+
25+
func (e testError) Error() string {
26+
return e.hidden.Error()
27+
}
28+
1529
func Test_Results(t *testing.T) {
1630
t.Parallel()
1731

32+
now := time.Date(2023, 10, 5, 12, 3, 56, 395813665, time.UTC)
1833
results := harness.Results{
1934
TotalRuns: 10,
2035
TotalPass: 8,
@@ -26,7 +41,7 @@ func Test_Results(t *testing.T) {
2641
ID: "0",
2742
Logs: "test-0/0 log line 1\ntest-0/0 log line 2",
2843
Error: xerrors.New("test-0/0 error"),
29-
StartedAt: time.Now(),
44+
StartedAt: now,
3045
Duration: httpapi.Duration(time.Second),
3146
DurationMS: 1000,
3247
},
@@ -36,7 +51,17 @@ func Test_Results(t *testing.T) {
3651
ID: "1",
3752
Logs: "test-0/1 log line 1\ntest-0/1 log line 2",
3853
Error: nil,
39-
StartedAt: time.Now(),
54+
StartedAt: now.Add(333 * time.Millisecond),
55+
Duration: httpapi.Duration(time.Second),
56+
DurationMS: 1000,
57+
},
58+
"test-0/2": {
59+
FullID: "test-0/2",
60+
TestName: "test-0",
61+
ID: "2",
62+
Logs: "test-0/2 log line 1\ntest-0/2 log line 2",
63+
Error: testError{hidden: xerrors.New("test-0/2 error")},
64+
StartedAt: now.Add(666 * time.Millisecond),
4065
Duration: httpapi.Duration(time.Second),
4166
DurationMS: 1000,
4267
},
@@ -45,26 +70,85 @@ func Test_Results(t *testing.T) {
4570
ElapsedMS: 1000,
4671
}
4772

48-
expected := `
73+
wantText := `
4974
== FAIL: test-0/0
5075
5176
Error: test-0/0 error
5277
5378
Log:
5479
test-0/0 log line 1
5580
81+
== FAIL: test-0/2
82+
83+
Error: test-0/2 error
84+
85+
Log:
86+
test-0/2 log line 1
87+
5688
5789
Test results:
5890
Pass: 8
5991
Fail: 2
6092
Total: 10
6193
6294
Total duration: 1s
63-
Avg. duration: 200ms
95+
Avg. duration: 300ms
6496
`
97+
wantJSON := `{
98+
"total_runs": 10,
99+
"total_pass": 8,
100+
"total_fail": 2,
101+
"elapsed": "1s",
102+
"elapsed_ms": 1000,
103+
"runs": {
104+
"test-0/0": {
105+
"full_id": "test-0/0",
106+
"test_name": "test-0",
107+
"id": "0",
108+
"logs": "test-0/0 log line 1\ntest-0/0 log line 2",
109+
"started_at": "2023-10-05T12:03:56.395813665Z",
110+
"duration": "1s",
111+
"duration_ms": 1000,
112+
"error": "test-0/0 error:\n github.com/coder/coder/v2/scaletest/harness_test.Test_Results\n [working_directory]/results_test.go:43"
113+
},
114+
"test-0/1": {
115+
"full_id": "test-0/1",
116+
"test_name": "test-0",
117+
"id": "1",
118+
"logs": "test-0/1 log line 1\ntest-0/1 log line 2",
119+
"started_at": "2023-10-05T12:03:56.728813665Z",
120+
"duration": "1s",
121+
"duration_ms": 1000,
122+
"error": "\u003cnil\u003e"
123+
},
124+
"test-0/2": {
125+
"full_id": "test-0/2",
126+
"test_name": "test-0",
127+
"id": "2",
128+
"logs": "test-0/2 log line 1\ntest-0/2 log line 2",
129+
"started_at": "2023-10-05T12:03:57.061813665Z",
130+
"duration": "1s",
131+
"duration_ms": 1000,
132+
"error": "test-0/2 error"
133+
}
134+
}
135+
}
136+
`
137+
wd, err := os.Getwd()
138+
require.NoError(t, err)
139+
wd = filepath.ToSlash(wd) // Hello there Windows, my friend...
140+
wantJSON = strings.Replace(wantJSON, "[working_directory]", wd, 1)
65141

66142
out := bytes.NewBuffer(nil)
67143
results.PrintText(out)
68144

69-
require.Equal(t, expected, out.String())
145+
assert.Empty(t, cmp.Diff(wantText, out.String()), "text result does not match (-want +got)")
146+
147+
out.Reset()
148+
enc := json.NewEncoder(out)
149+
enc.SetIndent("", "\t")
150+
err = enc.Encode(results)
151+
require.NoError(t, err)
152+
153+
assert.Empty(t, cmp.Diff(wantJSON, out.String()), "JSON result does not match (-want +got)")
70154
}

0 commit comments

Comments
 (0)