Skip to content

Commit 068f9a0

Browse files
cstyanjoobisb
cstyan
and
joobisb
authored
feat: include read/write byte stats in scaletests JSON report (#17777)
PR to fix #12157 --------- Signed-off-by: Callum Styan <callumstyan@gmail.com> Co-authored-by: joobisb <joobisb@gmail.com>
1 parent 4bd5609 commit 068f9a0

File tree

7 files changed

+136
-49
lines changed

7 files changed

+136
-49
lines changed

‎scaletest/harness/results.go

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,16 @@ type Results struct {
2727

2828
// RunResult is the result of a single test run.
2929
type RunResult struct {
30-
FullID string `json:"full_id"`
31-
TestName string `json:"test_name"`
32-
ID string `json:"id"`
33-
Logs string `json:"logs"`
34-
Error error `json:"error"`
35-
StartedAt time.Time `json:"started_at"`
36-
Duration httpapi.Duration `json:"duration"`
37-
DurationMS int64 `json:"duration_ms"`
30+
FullID string `json:"full_id"`
31+
TestName string `json:"test_name"`
32+
ID string `json:"id"`
33+
Logs string `json:"logs"`
34+
Error error `json:"error"`
35+
StartedAt time.Time `json:"started_at"`
36+
Duration httpapi.Duration `json:"duration"`
37+
DurationMS int64 `json:"duration_ms"`
38+
TotalBytesRead int64 `json:"total_bytes_read"`
39+
TotalBytesWritten int64 `json:"total_bytes_written"`
3840
}
3941

4042
// MarshalJSON implements json.Marhshaler for RunResult.
@@ -59,14 +61,16 @@ func (r *TestRun) Result() RunResult {
5961
}
6062

6163
return RunResult{
62-
FullID: r.FullID(),
63-
TestName: r.testName,
64-
ID: r.id,
65-
Logs: r.logs.String(),
66-
Error: r.err,
67-
StartedAt: r.started,
68-
Duration: httpapi.Duration(r.duration),
69-
DurationMS: r.duration.Milliseconds(),
64+
FullID: r.FullID(),
65+
TestName: r.testName,
66+
ID: r.id,
67+
Logs: r.logs.String(),
68+
Error: r.err,
69+
StartedAt: r.started,
70+
Duration: httpapi.Duration(r.duration),
71+
DurationMS: r.duration.Milliseconds(),
72+
TotalBytesRead: r.bytesRead,
73+
TotalBytesWritten: r.bytesWritten,
7074
}
7175
}
7276

‎scaletest/harness/results_test.go

Lines changed: 36 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -36,34 +36,40 @@ func Test_Results(t *testing.T) {
3636
TotalFail: 2,
3737
Runs: map[string]harness.RunResult{
3838
"test-0/0": {
39-
FullID: "test-0/0",
40-
TestName: "test-0",
41-
ID: "0",
42-
Logs: "test-0/0 log line 1\ntest-0/0 log line 2",
43-
Error: xerrors.New("test-0/0 error"),
44-
StartedAt: now,
45-
Duration: httpapi.Duration(time.Second),
46-
DurationMS: 1000,
39+
FullID: "test-0/0",
40+
TestName: "test-0",
41+
ID: "0",
42+
Logs: "test-0/0 log line 1\ntest-0/0 log line 2",
43+
Error: xerrors.New("test-0/0 error"),
44+
StartedAt: now,
45+
Duration: httpapi.Duration(time.Second),
46+
DurationMS: 1000,
47+
TotalBytesRead: 1024,
48+
TotalBytesWritten: 2048,
4749
},
4850
"test-0/1": {
49-
FullID: "test-0/1",
50-
TestName: "test-0",
51-
ID: "1",
52-
Logs: "test-0/1 log line 1\ntest-0/1 log line 2",
53-
Error: nil,
54-
StartedAt: now.Add(333 * time.Millisecond),
55-
Duration: httpapi.Duration(time.Second),
56-
DurationMS: 1000,
51+
FullID: "test-0/1",
52+
TestName: "test-0",
53+
ID: "1",
54+
Logs: "test-0/1 log line 1\ntest-0/1 log line 2",
55+
Error: nil,
56+
StartedAt: now.Add(333 * time.Millisecond),
57+
Duration: httpapi.Duration(time.Second),
58+
DurationMS: 1000,
59+
TotalBytesRead: 512,
60+
TotalBytesWritten: 1024,
5761
},
5862
"test-0/2": {
59-
FullID: "test-0/2",
60-
TestName: "test-0",
61-
ID: "2",
62-
Logs: "test-0/2 log line 1\ntest-0/2 log line 2",
63-
Error: testError{hidden: xerrors.New("test-0/2 error")},
64-
StartedAt: now.Add(666 * time.Millisecond),
65-
Duration: httpapi.Duration(time.Second),
66-
DurationMS: 1000,
63+
FullID: "test-0/2",
64+
TestName: "test-0",
65+
ID: "2",
66+
Logs: "test-0/2 log line 1\ntest-0/2 log line 2",
67+
Error: testError{hidden: xerrors.New("test-0/2 error")},
68+
StartedAt: now.Add(666 * time.Millisecond),
69+
Duration: httpapi.Duration(time.Second),
70+
DurationMS: 1000,
71+
TotalBytesRead: 2048,
72+
TotalBytesWritten: 4096,
6773
},
6874
},
6975
Elapsed: httpapi.Duration(time.Second),
@@ -109,6 +115,8 @@ Test results:
109115
"started_at": "2023-10-05T12:03:56.395813665Z",
110116
"duration": "1s",
111117
"duration_ms": 1000,
118+
"total_bytes_read": 1024,
119+
"total_bytes_written": 2048,
112120
"error": "test-0/0 error:\n github.com/coder/coder/v2/scaletest/harness_test.Test_Results\n [working_directory]/results_test.go:43"
113121
},
114122
"test-0/1": {
@@ -119,6 +127,8 @@ Test results:
119127
"started_at": "2023-10-05T12:03:56.728813665Z",
120128
"duration": "1s",
121129
"duration_ms": 1000,
130+
"total_bytes_read": 512,
131+
"total_bytes_written": 1024,
122132
"error": "\u003cnil\u003e"
123133
},
124134
"test-0/2": {
@@ -129,6 +139,8 @@ Test results:
129139
"started_at": "2023-10-05T12:03:57.061813665Z",
130140
"duration": "1s",
131141
"duration_ms": 1000,
142+
"total_bytes_read": 2048,
143+
"total_bytes_written": 4096,
132144
"error": "test-0/2 error"
133145
}
134146
}

‎scaletest/harness/run.go

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,13 @@ type Cleanable interface {
3131
Cleanup(ctx context.Context, id string, logs io.Writer) error
3232
}
3333

34+
// Collectable is an optional extension to Runnable that allows to get metrics from the runner.
35+
type Collectable interface {
36+
Runnable
37+
// Gets the bytes transferred
38+
GetBytesTransferred() (int64, int64)
39+
}
40+
3441
// AddRun creates a new *TestRun with the given name, ID and Runnable, adds it
3542
// to the harness and returns it. Panics if the harness has been started, or a
3643
// test with the given run.FullID() is already registered.
@@ -66,11 +73,13 @@ type TestRun struct {
6673
id string
6774
runner Runnable
6875

69-
logs *syncBuffer
70-
done chan struct{}
71-
started time.Time
72-
duration time.Duration
73-
err error
76+
logs *syncBuffer
77+
done chan struct{}
78+
started time.Time
79+
duration time.Duration
80+
err error
81+
bytesRead int64
82+
bytesWritten int64
7483
}
7584

7685
func NewTestRun(testName string, id string, runner Runnable) *TestRun {
@@ -98,6 +107,11 @@ func (r *TestRun) Run(ctx context.Context) (err error) {
98107
defer func() {
99108
r.duration = time.Since(r.started)
100109
r.err = err
110+
c, ok := r.runner.(Collectable)
111+
if !ok {
112+
return
113+
}
114+
r.bytesRead, r.bytesWritten = c.GetBytesTransferred()
101115
}()
102116
defer func() {
103117
e := recover()
@@ -107,6 +121,7 @@ func (r *TestRun) Run(ctx context.Context) (err error) {
107121
}()
108122

109123
err = r.runner.Run(ctx, r.id, r.logs)
124+
110125
//nolint:revive // we use named returns because we mutate it in a defer
111126
return
112127
}

‎scaletest/harness/run_test.go

Lines changed: 38 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,24 @@ type testFns struct {
1717
RunFn func(ctx context.Context, id string, logs io.Writer) error
1818
// CleanupFn is optional if no cleanup is required.
1919
CleanupFn func(ctx context.Context, id string, logs io.Writer) error
20+
// getBytesTransferred is optional if byte transfer tracking is required.
21+
getBytesTransferred func() (int64, int64)
2022
}
2123

2224
// Run implements Runnable.
2325
func (fns testFns) Run(ctx context.Context, id string, logs io.Writer) error {
2426
return fns.RunFn(ctx, id, logs)
2527
}
2628

29+
// GetBytesTransferred implements Collectable.
30+
func (fns testFns) GetBytesTransferred() (bytesRead int64, bytesWritten int64) {
31+
if fns.getBytesTransferred == nil {
32+
return 0, 0
33+
}
34+
35+
return fns.getBytesTransferred()
36+
}
37+
2738
// Cleanup implements Cleanable.
2839
func (fns testFns) Cleanup(ctx context.Context, id string, logs io.Writer) error {
2940
if fns.CleanupFn == nil {
@@ -40,9 +51,10 @@ func Test_TestRun(t *testing.T) {
4051
t.Parallel()
4152

4253
var (
43-
name, id = "test", "1"
44-
runCalled int64
45-
cleanupCalled int64
54+
name, id = "test", "1"
55+
runCalled int64
56+
cleanupCalled int64
57+
collectableCalled int64
4658

4759
testFns = testFns{
4860
RunFn: func(ctx context.Context, id string, logs io.Writer) error {
@@ -53,6 +65,10 @@ func Test_TestRun(t *testing.T) {
5365
atomic.AddInt64(&cleanupCalled, 1)
5466
return nil
5567
},
68+
getBytesTransferred: func() (int64, int64) {
69+
atomic.AddInt64(&collectableCalled, 1)
70+
return 0, 0
71+
},
5672
}
5773
)
5874

@@ -62,6 +78,7 @@ func Test_TestRun(t *testing.T) {
6278
err := run.Run(context.Background())
6379
require.NoError(t, err)
6480
require.EqualValues(t, 1, atomic.LoadInt64(&runCalled))
81+
require.EqualValues(t, 1, atomic.LoadInt64(&collectableCalled))
6582

6683
err = run.Cleanup(context.Background())
6784
require.NoError(t, err)
@@ -105,6 +122,24 @@ func Test_TestRun(t *testing.T) {
105122
})
106123
})
107124

125+
t.Run("Collectable", func(t *testing.T) {
126+
t.Parallel()
127+
128+
t.Run("NoFn", func(t *testing.T) {
129+
t.Parallel()
130+
131+
run := harness.NewTestRun("test", "1", testFns{
132+
RunFn: func(ctx context.Context, id string, logs io.Writer) error {
133+
return nil
134+
},
135+
getBytesTransferred: nil,
136+
})
137+
138+
err := run.Run(context.Background())
139+
require.NoError(t, err)
140+
})
141+
})
142+
108143
t.Run("CatchesRunPanic", func(t *testing.T) {
109144
t.Parallel()
110145

‎scaletest/workspacetraffic/metrics.go

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
package workspacetraffic
22

3-
import "github.com/prometheus/client_golang/prometheus"
3+
import (
4+
"sync/atomic"
5+
6+
"github.com/prometheus/client_golang/prometheus"
7+
)
48

59
type Metrics struct {
610
BytesReadTotal prometheus.CounterVec
@@ -75,12 +79,14 @@ type ConnMetrics interface {
7579
AddError(float64)
7680
ObserveLatency(float64)
7781
AddTotal(float64)
82+
GetTotalBytes() int64
7883
}
7984

8085
type connMetrics struct {
8186
addError func(float64)
8287
observeLatency func(float64)
8388
addTotal func(float64)
89+
total int64
8490
}
8591

8692
func (c *connMetrics) AddError(f float64) {
@@ -92,5 +98,10 @@ func (c *connMetrics) ObserveLatency(f float64) {
9298
}
9399

94100
func (c *connMetrics) AddTotal(f float64) {
101+
atomic.AddInt64(&c.total, int64(f))
95102
c.addTotal(f)
96103
}
104+
105+
func (c *connMetrics) GetTotalBytes() int64 {
106+
return c.total
107+
}

‎scaletest/workspacetraffic/run.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,12 @@ func (r *Runner) Run(ctx context.Context, _ string, logs io.Writer) (err error)
210210
}
211211
}
212212

213+
func (r *Runner) GetBytesTransferred() (bytesRead, bytesWritten int64) {
214+
bytesRead = r.cfg.ReadMetrics.GetTotalBytes()
215+
bytesWritten = r.cfg.WriteMetrics.GetTotalBytes()
216+
return bytesRead, bytesWritten
217+
}
218+
213219
// Cleanup does nothing, successfully.
214220
func (*Runner) Cleanup(context.Context, string, io.Writer) error {
215221
return nil

‎scaletest/workspacetraffic/run_test.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -422,3 +422,7 @@ func (m *testMetrics) Latencies() []float64 {
422422
defer m.Unlock()
423423
return m.latencies
424424
}
425+
426+
func (m *testMetrics) GetTotalBytes() int64 {
427+
return int64(m.total)
428+
}

0 commit comments

Comments
 (0)