Skip to content

Commit ec96a00

Browse files
johnstcnjohnstcn
committed
ignore certain errors such as websocket closing or context cancellation
1 parent 399e506 commit ec96a00

File tree

1 file changed

+45
-22
lines changed
  • scaletest/workspacetraffic

1 file changed

+45
-22
lines changed

‎scaletest/workspacetraffic/run.go

Lines changed: 45 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ package workspacetraffic
33
import (
44
"context"
55
"encoding/json"
6+
"errors"
67
"io"
7-
"sync/atomic"
88
"time"
99

1010
"github.com/google/uuid"
@@ -19,6 +19,8 @@ import (
1919
"github.com/coder/coder/cryptorand"
2020
"github.com/coder/coder/scaletest/harness"
2121
"github.com/coder/coder/scaletest/loadtestutil"
22+
23+
promtest "github.com/prometheus/client_golang/prometheus/testutil"
2224
)
2325

2426
type Runner struct {
@@ -49,6 +51,16 @@ func (r *Runner) Run(ctx context.Context, _ string, logs io.Writer) error {
4951
r.client.Logger = logger
5052
r.client.LogBodies = true
5153

54+
// Initialize our metrics eagerly. This is mainly so that we can test for the
55+
// presence of a zero-valued metric as opposed to the absence of a metric.
56+
lvs := []string{r.cfg.WorkspaceOwner, r.cfg.WorkspaceName, r.cfg.AgentName}
57+
r.metrics.BytesReadTotal.WithLabelValues(lvs...).Add(0)
58+
r.metrics.BytesWrittenTotal.WithLabelValues(lvs...).Add(0)
59+
r.metrics.ReadErrorsTotal.WithLabelValues(lvs...).Add(0)
60+
r.metrics.WriteErrorsTotal.WithLabelValues(lvs...).Add(0)
61+
r.metrics.ReadLatencySeconds.WithLabelValues(lvs...).Observe(0)
62+
r.metrics.WriteLatencySeconds.WithLabelValues(lvs...).Observe(0)
63+
5264
var (
5365
agentID = r.cfg.AgentID
5466
reconnect = uuid.New()
@@ -92,7 +104,7 @@ func (r *Runner) Run(ctx context.Context, _ string, logs io.Writer) error {
92104
}()
93105

94106
// Wrap the conn in a countReadWriter so we can monitor bytes sent/rcvd.
95-
crw := countReadWriter{ReadWriter: conn, metrics: r.metrics, labels: []string{r.cfg.WorkspaceOwner, r.cfg.WorkspaceName, r.cfg.AgentName}}
107+
crw := countReadWriter{ReadWriter: conn, metrics: r.metrics, labels: lvs}
96108

97109
// Create a ticker for sending data to the PTY.
98110
tick := time.NewTicker(tickInterval)
@@ -133,11 +145,12 @@ func (r *Runner) Run(ctx context.Context, _ string, logs io.Writer) error {
133145
}
134146

135147
duration := time.Since(start)
136-
137-
logger.Info(ctx, "results",
148+
logger.Info(ctx, "Test Results",
138149
slog.F("duration", duration),
139-
slog.F("sent", crw.BytesWritten()),
140-
slog.F("rcvd", crw.BytesRead()),
150+
slog.F("bytes_read_total", promtest.ToFloat64(r.metrics.BytesReadTotal)),
151+
slog.F("bytes_written_total", promtest.ToFloat64(r.metrics.BytesWrittenTotal)),
152+
slog.F("read_errors_total", promtest.ToFloat64(r.metrics.ReadErrorsTotal)),
153+
slog.F("write_errors_total", promtest.ToFloat64(r.metrics.WriteErrorsTotal)),
141154
)
142155

143156
return nil
@@ -186,42 +199,36 @@ func writeRandomData(dst io.Writer, size int64, tick <-chan time.Time) error {
186199
// countReadWriter wraps an io.ReadWriter and counts the number of bytes read and written.
187200
type countReadWriter struct {
188201
io.ReadWriter
189-
bytesRead atomic.Int64
190-
bytesWritten atomic.Int64
191-
metrics *Metrics
192-
labels []string
202+
metrics *Metrics
203+
labels []string
193204
}
194205

195206
func (w *countReadWriter) Read(p []byte) (int, error) {
196207
start := time.Now()
197208
n, err := w.ReadWriter.Read(p)
209+
if reportableErr(err) {
210+
w.metrics.ReadErrorsTotal.WithLabelValues(w.labels...).Inc()
211+
}
198212
w.metrics.ReadLatencySeconds.WithLabelValues(w.labels...).Observe(time.Since(start).Seconds())
199213
if n > 0 {
200-
w.bytesRead.Add(int64(n))
201-
w.metrics.BytesRead.WithLabelValues(w.labels...).Add(float64(n))
214+
w.metrics.BytesReadTotal.WithLabelValues(w.labels...).Add(float64(n))
202215
}
203216
return n, err
204217
}
205218

206219
func (w *countReadWriter) Write(p []byte) (int, error) {
207220
start := time.Now()
208221
n, err := w.ReadWriter.Write(p)
222+
if reportableErr(err) {
223+
w.metrics.WriteErrorsTotal.WithLabelValues(w.labels...).Inc()
224+
}
209225
w.metrics.WriteLatencySeconds.WithLabelValues(w.labels...).Observe(time.Since(start).Seconds())
210226
if n > 0 {
211-
w.bytesWritten.Add(int64(n))
212-
w.metrics.BytesWritten.WithLabelValues(w.labels...).Add(float64(n))
227+
w.metrics.BytesWrittenTotal.WithLabelValues(w.labels...).Add(float64(n))
213228
}
214229
return n, err
215230
}
216231

217-
func (w *countReadWriter) BytesRead() int64 {
218-
return w.bytesRead.Load()
219-
}
220-
221-
func (w *countReadWriter) BytesWritten() int64 {
222-
return w.bytesWritten.Load()
223-
}
224-
225232
func mustRandStr(l int64) string {
226233
if l < 1 {
227234
l = 1
@@ -232,3 +239,19 @@ func mustRandStr(l int64) string {
232239
}
233240
return randStr
234241
}
242+
243+
// some errors we want to report in metrics; others we want to ignore
244+
// such as websocket.StatusNormalClosure or context.Canceled
245+
func reportableErr(err error) bool {
246+
if err == nil {
247+
return false
248+
}
249+
if xerrors.Is(err, context.Canceled) {
250+
return false
251+
}
252+
var wsErr websocket.CloseError
253+
if errors.As(err, &wsErr) {
254+
return wsErr.Code != websocket.StatusNormalClosure
255+
}
256+
return false
257+
}

0 commit comments

Comments
 (0)