Skip to content

Commit d5a98cc

Browse files
authored
fix: avoid race in TestPGPubsub_Metrics by using Eventually (#11973)
Annoyingly, prometheus Registry collects metrics async, which is causing our test to be racy. They also don't export enough from the Metric interface for us to replicate a synchronous collect, so we have to use Eventually to test.
1 parent 5a359d5 commit d5a98cc

File tree

1 file changed

+33
-31
lines changed

1 file changed

+33
-31
lines changed

coderd/database/pubsub/pubsub_test.go

Lines changed: 33 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ func TestPGPubsub_Metrics(t *testing.T) {
4343

4444
metrics, err := registry.Gather()
4545
require.NoError(t, err)
46-
requireGaugeValue(t, metrics, 0, "coder_pubsub_current_events")
47-
requireGaugeValue(t, metrics, 0, "coder_pubsub_current_subscribers")
46+
require.True(t, gaugeHasValue(t, metrics, 0, "coder_pubsub_current_events"))
47+
require.True(t, gaugeHasValue(t, metrics, 0, "coder_pubsub_current_subscribers"))
4848

4949
event := "test"
5050
data := "testing"
@@ -60,16 +60,18 @@ func TestPGPubsub_Metrics(t *testing.T) {
6060
}()
6161
_ = testutil.RequireRecvCtx(ctx, t, messageChannel)
6262

63-
metrics, err = registry.Gather()
64-
require.NoError(t, err)
65-
requireGaugeValue(t, metrics, 1, "coder_pubsub_current_events")
66-
requireGaugeValue(t, metrics, 1, "coder_pubsub_current_subscribers")
67-
requireGaugeValue(t, metrics, 1, "coder_pubsub_connected")
68-
requireCounterValue(t, metrics, 1, "coder_pubsub_publishes_total", "true")
69-
requireCounterValue(t, metrics, 1, "coder_pubsub_subscribes_total", "true")
70-
requireCounterValue(t, metrics, 1, "coder_pubsub_messages_total", "normal")
71-
requireCounterValue(t, metrics, 7, "coder_pubsub_received_bytes_total")
72-
requireCounterValue(t, metrics, 7, "coder_pubsub_published_bytes_total")
63+
require.Eventually(t, func() bool {
64+
metrics, err = registry.Gather()
65+
assert.NoError(t, err)
66+
return gaugeHasValue(t, metrics, 1, "coder_pubsub_current_events") &&
67+
gaugeHasValue(t, metrics, 1, "coder_pubsub_current_subscribers") &&
68+
gaugeHasValue(t, metrics, 1, "coder_pubsub_connected") &&
69+
counterHasValue(t, metrics, 1, "coder_pubsub_publishes_total", "true") &&
70+
counterHasValue(t, metrics, 1, "coder_pubsub_subscribes_total", "true") &&
71+
counterHasValue(t, metrics, 1, "coder_pubsub_messages_total", "normal") &&
72+
counterHasValue(t, metrics, 7, "coder_pubsub_received_bytes_total") &&
73+
counterHasValue(t, metrics, 7, "coder_pubsub_published_bytes_total")
74+
}, testutil.WaitShort, testutil.IntervalFast)
7375

7476
colossalData := make([]byte, 7600)
7577
for i := range colossalData {
@@ -88,20 +90,22 @@ func TestPGPubsub_Metrics(t *testing.T) {
8890
_ = testutil.RequireRecvCtx(ctx, t, messageChannel)
8991
_ = testutil.RequireRecvCtx(ctx, t, messageChannel)
9092

91-
metrics, err = registry.Gather()
92-
require.NoError(t, err)
93-
requireGaugeValue(t, metrics, 1, "coder_pubsub_current_events")
94-
requireGaugeValue(t, metrics, 2, "coder_pubsub_current_subscribers")
95-
requireGaugeValue(t, metrics, 1, "coder_pubsub_connected")
96-
requireCounterValue(t, metrics, 2, "coder_pubsub_publishes_total", "true")
97-
requireCounterValue(t, metrics, 2, "coder_pubsub_subscribes_total", "true")
98-
requireCounterValue(t, metrics, 1, "coder_pubsub_messages_total", "normal")
99-
requireCounterValue(t, metrics, 1, "coder_pubsub_messages_total", "colossal")
100-
requireCounterValue(t, metrics, 7607, "coder_pubsub_received_bytes_total")
101-
requireCounterValue(t, metrics, 7607, "coder_pubsub_published_bytes_total")
93+
require.Eventually(t, func() bool {
94+
metrics, err = registry.Gather()
95+
assert.NoError(t, err)
96+
return gaugeHasValue(t, metrics, 1, "coder_pubsub_current_events") &&
97+
gaugeHasValue(t, metrics, 2, "coder_pubsub_current_subscribers") &&
98+
gaugeHasValue(t, metrics, 1, "coder_pubsub_connected") &&
99+
counterHasValue(t, metrics, 2, "coder_pubsub_publishes_total", "true") &&
100+
counterHasValue(t, metrics, 2, "coder_pubsub_subscribes_total", "true") &&
101+
counterHasValue(t, metrics, 1, "coder_pubsub_messages_total", "normal") &&
102+
counterHasValue(t, metrics, 1, "coder_pubsub_messages_total", "colossal") &&
103+
counterHasValue(t, metrics, 7607, "coder_pubsub_received_bytes_total") &&
104+
counterHasValue(t, metrics, 7607, "coder_pubsub_published_bytes_total")
105+
}, testutil.WaitShort, testutil.IntervalFast)
102106
}
103107

104-
func requireGaugeValue(t testing.TB, metrics []*dto.MetricFamily, value float64, name string, label ...string) {
108+
func gaugeHasValue(t testing.TB, metrics []*dto.MetricFamily, value float64, name string, label ...string) bool {
105109
t.Helper()
106110
for _, family := range metrics {
107111
if family.GetName() != name {
@@ -115,14 +119,13 @@ func requireGaugeValue(t testing.TB, metrics []*dto.MetricFamily, value float64,
115119
continue
116120
}
117121
}
118-
require.Equal(t, value, m.GetGauge().GetValue())
119-
return
122+
return value == m.GetGauge().GetValue()
120123
}
121124
}
122-
t.Fatal("didn't find metric")
125+
return false
123126
}
124127

125-
func requireCounterValue(t testing.TB, metrics []*dto.MetricFamily, value float64, name string, label ...string) {
128+
func counterHasValue(t testing.TB, metrics []*dto.MetricFamily, value float64, name string, label ...string) bool {
126129
t.Helper()
127130
for _, family := range metrics {
128131
if family.GetName() != name {
@@ -136,9 +139,8 @@ func requireCounterValue(t testing.TB, metrics []*dto.MetricFamily, value float6
136139
continue
137140
}
138141
}
139-
require.Equal(t, value, m.GetCounter().GetValue())
140-
return
142+
return value == m.GetCounter().GetValue()
141143
}
142144
}
143-
t.Fatal("didn't find metric")
145+
return false
144146
}

0 commit comments

Comments
 (0)