Skip to content

Commit 67a3215

Browse files
SasSwartSasSwart
committed
assert for label values in prebuild metrics tests
1 parent f814510 commit 67a3215

File tree

6 files changed

+155
-38
lines changed

6 files changed

+155
-38
lines changed

‎coderd/database/queries.sql.go

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/queries/prebuilds.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ WHERE (b.transition = 'start'::workspace_transition
2222

2323
-- name: GetTemplatePresetsWithPrebuilds :many
2424
SELECT t.id AS template_id,
25+
t.name AS template_name,
2526
tv.id AS template_version_id,
2627
tv.id = t.active_version_id AS using_active_version,
2728
tvpp.preset_id,

‎enterprise/coderd/coderd.go

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -674,7 +674,7 @@ func (api *API) Close() error {
674674
}
675675

676676
if api.PrebuildsReconciler != nil {
677-
ctx, giveUp := context.WithTimeoutCause(context.Background(), time.Second*30, errors.New("gave up waiting for reconciler to stop"))
677+
ctx, giveUp := context.WithTimeoutCause(context.Background(), time.Second*30, xerrors.New("gave up waiting for reconciler to stop"))
678678
defer giveUp()
679679
api.PrebuildsReconciler.Stop(ctx, xerrors.New("api closed")) // TODO: determine root cause (requires changes up the stack, though).
680680
}
@@ -884,9 +884,9 @@ func (api *API) updateEntitlements(ctx context.Context) error {
884884
if initial, changed, enabled := featureChanged(codersdk.FeatureWorkspacePrebuilds); shouldUpdate(initial, changed, enabled) || api.PrebuildsReconciler == nil {
885885
reconciler, claimer, metrics := api.setupPrebuilds(enabled)
886886
if api.PrebuildsReconciler != nil {
887-
stopCtx, giveUp := context.WithTimeoutCause(context.Background(), time.Second*30, errors.New("gave up waiting for reconciler to stop"))
887+
stopCtx, giveUp := context.WithTimeoutCause(context.Background(), time.Second*30, xerrors.New("gave up waiting for reconciler to stop"))
888888
defer giveUp()
889-
api.PrebuildsReconciler.Stop(stopCtx, errors.New("entitlements change"))
889+
api.PrebuildsReconciler.Stop(stopCtx, xerrors.New("entitlements change"))
890890
}
891891

892892
// Only register metrics once.
@@ -1178,16 +1178,18 @@ func (api *API) setupPrebuilds(entitled bool) (agplprebuilds.ReconciliationOrche
11781178
return agplprebuilds.NewNoopReconciler(), agplprebuilds.DefaultClaimer, nil
11791179
}
11801180

1181+
reconciler := prebuilds.NewStoreReconciler(api.Database, api.Pubsub, api.DeploymentValues.Prebuilds,
1182+
api.Logger.Named("prebuilds"), quartz.NewReal())
1183+
11811184
logger := api.Logger.Named("prebuilds.metrics")
1182-
collector := prebuilds.NewMetricsCollector(api.Database, logger)
1185+
collector := prebuilds.NewMetricsCollector(api.Database, logger, reconciler)
11831186
err := api.PrometheusRegistry.Register(collector)
11841187
if err != nil {
11851188
logger.Error(context.Background(), "failed to register prebuilds metrics collector", slog.F("error", err))
11861189
collector = nil
11871190
}
11881191

1189-
return prebuilds.NewStoreReconciler(api.Database, api.Pubsub, api.DeploymentValues.Prebuilds,
1190-
api.Logger.Named("prebuilds"), quartz.NewReal()),
1192+
return reconciler,
11911193
prebuilds.EnterpriseClaimer{},
11921194
collector
11931195
}

‎enterprise/coderd/prebuilds/metricscollector.go

Lines changed: 35 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010

1111
"github.com/coder/coder/v2/coderd/database"
1212
"github.com/coder/coder/v2/coderd/database/dbauthz"
13+
"github.com/coder/coder/v2/coderd/prebuilds"
1314
)
1415

1516
var (
@@ -24,18 +25,18 @@ var (
2425
)
2526

2627
type MetricsCollector struct {
27-
database database.Store
28-
logger slog.Logger
29-
// reconciler *prebuilds.Reconciler
28+
database database.Store
29+
logger slog.Logger
30+
reconciler prebuilds.Reconciler
3031
}
3132

3233
var _ prometheus.Collector = new(MetricsCollector)
3334

34-
func NewMetricsCollector(db database.Store, logger slog.Logger) *MetricsCollector {
35+
func NewMetricsCollector(db database.Store, logger slog.Logger, reconciler prebuilds.Reconciler) *MetricsCollector {
3536
return &MetricsCollector{
36-
database: db,
37-
logger: logger.Named("prebuilds_metrics_collector"),
38-
// reconciler: reconciler,
37+
database: db,
38+
logger: logger.Named("prebuilds_metrics_collector"),
39+
reconciler: reconciler,
3940
}
4041
}
4142

@@ -66,5 +67,31 @@ func (mc *MetricsCollector) Collect(metricsCh chan<- prometheus.Metric) {
6667
metricsCh <- prometheus.MustNewConstMetric(FailedPrebuildsDesc, prometheus.CounterValue, float64(metric.FailedCount), metric.TemplateName, metric.PresetName)
6768
metricsCh <- prometheus.MustNewConstMetric(AssignedPrebuildsDesc, prometheus.CounterValue, float64(metric.ClaimedCount), metric.TemplateName, metric.PresetName)
6869
}
69-
// TODO (sasswart): read gauges from controller
70+
71+
state, err := mc.reconciler.SnapshotState(dbauthz.AsSystemRestricted(ctx), mc.database)
72+
if err != nil {
73+
mc.logger.Error(ctx, "failed to get latest prebuild state", slog.Error(err))
74+
return
75+
}
76+
77+
for _, preset := range state.Presets {
78+
if !preset.UsingActiveVersion {
79+
continue
80+
}
81+
82+
presetState, err := state.FilterByPreset(preset.PresetID)
83+
if err != nil {
84+
mc.logger.Error(ctx, "failed to filter by preset", slog.Error(err))
85+
continue
86+
}
87+
actions, err := mc.reconciler.DetermineActions(ctx, *presetState)
88+
if err != nil {
89+
mc.logger.Error(ctx, "failed to determine actions", slog.Error(err))
90+
continue
91+
}
92+
93+
metricsCh <- prometheus.MustNewConstMetric(DesiredPrebuildsDesc, prometheus.GaugeValue, float64(actions.Desired), preset.TemplateName, preset.Name)
94+
metricsCh <- prometheus.MustNewConstMetric(ActualPrebuildsDesc, prometheus.GaugeValue, float64(actions.Actual), preset.TemplateName, preset.Name)
95+
metricsCh <- prometheus.MustNewConstMetric(EligiblePrebuildsDesc, prometheus.GaugeValue, float64(actions.Eligible), preset.TemplateName, preset.Name)
96+
}
7097
}

‎enterprise/coderd/prebuilds/metricscollector_test.go

Lines changed: 101 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,17 @@ import (
99
"github.com/stretchr/testify/require"
1010
"tailscale.com/types/ptr"
1111

12-
promtestutil "github.com/prometheus/client_golang/prometheus/testutil"
12+
"github.com/prometheus/client_golang/prometheus"
13+
prometheus_client "github.com/prometheus/client_model/go"
1314

1415
"cdr.dev/slog/sloggers/slogtest"
1516
"github.com/coder/coder/v2/coderd/database"
1617
"github.com/coder/coder/v2/coderd/database/dbgen"
1718
"github.com/coder/coder/v2/coderd/database/dbtestutil"
19+
"github.com/coder/coder/v2/codersdk"
1820
"github.com/coder/coder/v2/enterprise/coderd/prebuilds"
1921
"github.com/coder/coder/v2/testutil"
22+
"github.com/coder/quartz"
2023
)
2124

2225
func TestMetricsCollector(t *testing.T) {
@@ -97,6 +100,7 @@ func TestMetricsCollector(t *testing.T) {
97100
}
98101
})
99102
db, pubsub := dbtestutil.NewDB(t)
103+
reconciler := prebuilds.NewStoreReconciler(db, pubsub, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t))
100104
ctx := testutil.Context(t, testutil.WaitLong)
101105

102106
createdUsers := []uuid.UUID{prebuilds.OwnerID}
@@ -109,32 +113,79 @@ func TestMetricsCollector(t *testing.T) {
109113
}
110114
}
111115

112-
collector := prebuilds.NewMetricsCollector(db, logger)
116+
collector := prebuilds.NewMetricsCollector(db, logger, reconciler)
117+
registry := prometheus.NewPedanticRegistry()
118+
registry.Register(collector)
113119

114-
iterations := 3
115-
for i := 0; i < iterations; i++ {
116-
orgID, templateID := setupTestDBTemplate(t, db, createdUsers[0])
117-
templateVersionID := setupTestDBTemplateVersion(t, ctx, db, pubsub, orgID, createdUsers[0], templateID)
118-
presetID := setupTestDBPreset(t, ctx, db, pubsub, templateVersionID)
120+
numTemplates := 2
121+
for i := 0; i < numTemplates; i++ {
122+
orgID, templateID := setupTestDBTemplate(t, db, ownerID)
123+
templateVersionID := setupTestDBTemplateVersion(t, ctx, db, pubsub, orgID, ownerID, templateID)
124+
preset := setupTestDBPreset(t, ctx, db, pubsub, templateVersionID, 1)
119125
setupTestDBPrebuild(
120126
t, ctx, db, pubsub,
121-
transition, jobStatus, orgID, templateID, templateVersionID, presetID, initiatorID, ownerID,
127+
transition, jobStatus, orgID, templateID, templateVersionID, preset.ID, initiatorID, ownerID,
122128
)
123129
}
124130

125-
if test.shouldIncrementPrebuildsCreated != nil {
126-
createdCount := promtestutil.CollectAndCount(collector, "coderd_prebuilds_created")
127-
require.Equal(t, *test.shouldIncrementPrebuildsCreated, createdCount == iterations, "createdCount: %d", createdCount)
128-
}
131+
metricsFamilies, err := registry.Gather()
132+
require.NoError(t, err)
129133

130-
if test.shouldIncrementPrebuildsFailed != nil {
131-
failedCount := promtestutil.CollectAndCount(collector, "coderd_prebuilds_failed")
132-
require.Equal(t, *test.shouldIncrementPrebuildsFailed, failedCount == iterations, "failedCount: %d", failedCount)
133-
}
134+
templates, err := db.GetTemplates(ctx)
135+
require.NoError(t, err)
136+
require.Equal(t, numTemplates, len(templates))
137+
138+
for _, template := range templates {
139+
templateVersions, err := db.GetTemplateVersionsByTemplateID(ctx, database.GetTemplateVersionsByTemplateIDParams{
140+
TemplateID: template.ID,
141+
})
142+
require.NoError(t, err)
143+
require.Equal(t, 1, len(templateVersions))
144+
145+
presets, err := db.GetPresetsByTemplateVersionID(ctx, templateVersions[0].ID)
146+
require.NoError(t, err)
147+
require.Equal(t, 1, len(presets))
148+
149+
for _, preset := range presets {
150+
if test.shouldIncrementPrebuildsCreated != nil {
151+
metric := findMetric(metricsFamilies, "coderd_prebuilds_created", map[string]string{
152+
"template_name": template.Name,
153+
"preset_name": preset.Name,
154+
})
155+
if *test.shouldIncrementPrebuildsCreated {
156+
require.NotNil(t, metric)
157+
require.Equal(t, metric.GetCounter().GetValue(), 1.0)
158+
} else {
159+
require.Nil(t, metric)
160+
}
161+
}
134162

135-
if test.shouldIncrementPrebuildsAssigned != nil {
136-
assignedCount := promtestutil.CollectAndCount(collector, "coderd_prebuilds_assigned")
137-
require.Equal(t, *test.shouldIncrementPrebuildsAssigned, assignedCount == iterations, "assignedCount: %d", assignedCount)
163+
if test.shouldIncrementPrebuildsFailed != nil {
164+
metric := findMetric(metricsFamilies, "coderd_prebuilds_failed", map[string]string{
165+
"template_name": template.Name,
166+
"preset_name": preset.Name,
167+
})
168+
if *test.shouldIncrementPrebuildsFailed {
169+
require.NotNil(t, metric)
170+
require.Equal(t, metric.GetCounter().GetValue(), 1.0)
171+
} else {
172+
require.Nil(t, metric)
173+
}
174+
}
175+
176+
if test.shouldIncrementPrebuildsAssigned != nil {
177+
metric := findMetric(metricsFamilies, "coderd_prebuilds_assigned", map[string]string{
178+
"template_name": template.Name,
179+
"preset_name": preset.Name,
180+
})
181+
if *test.shouldIncrementPrebuildsAssigned {
182+
require.NotNil(t, metric)
183+
require.Equal(t, metric.GetCounter().GetValue(), 1.0)
184+
} else {
185+
require.Nil(t, metric)
186+
}
187+
}
188+
}
138189
}
139190
})
140191
}
@@ -143,3 +194,34 @@ func TestMetricsCollector(t *testing.T) {
143194
}
144195
}
145196
}
197+
198+
func findMetric(metricsFamilies []*prometheus_client.MetricFamily, name string, labels map[string]string) *prometheus_client.Metric {
199+
for _, metricFamily := range metricsFamilies {
200+
if metricFamily.GetName() == name {
201+
for _, metric := range metricFamily.GetMetric() {
202+
matches := true
203+
labelPairs := metric.GetLabel()
204+
205+
// Check if all requested labels match
206+
for wantName, wantValue := range labels {
207+
found := false
208+
for _, label := range labelPairs {
209+
if label.GetName() == wantName && label.GetValue() == wantValue {
210+
found = true
211+
break
212+
}
213+
}
214+
if !found {
215+
matches = false
216+
break
217+
}
218+
}
219+
220+
if matches {
221+
return metric
222+
}
223+
}
224+
}
225+
}
226+
return nil
227+
}

‎enterprise/coderd/prebuilds/reconcile_test.go

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -302,12 +302,13 @@ func TestPrebuildReconciliation(t *testing.T) {
302302
ownerID,
303303
templateID,
304304
)
305-
presetID := setupTestDBPreset(
305+
preset := setupTestDBPreset(
306306
t,
307307
ctx,
308308
db,
309309
pubsub,
310310
templateVersionID,
311+
1,
311312
)
312313
prebuildID := setupTestDBPrebuild(
313314
t,
@@ -319,7 +320,7 @@ func TestPrebuildReconciliation(t *testing.T) {
319320
orgID,
320321
templateID,
321322
templateVersionID,
322-
presetID,
323+
preset.ID,
323324
prebuilds.OwnerID,
324325
prebuilds.OwnerID,
325326
)
@@ -437,7 +438,8 @@ func setupTestDBPreset(
437438
db database.Store,
438439
ps pubsub.Pubsub,
439440
templateVersionID uuid.UUID,
440-
) uuid.UUID {
441+
desiredInstances int32,
442+
) database.TemplateVersionPreset {
441443
t.Helper()
442444
preset := dbgen.Preset(t, db, database.InsertPresetParams{
443445
TemplateVersionID: templateVersionID,
@@ -451,10 +453,10 @@ func setupTestDBPreset(
451453
_, err := db.InsertPresetPrebuild(ctx, database.InsertPresetPrebuildParams{
452454
ID: uuid.New(),
453455
PresetID: preset.ID,
454-
DesiredInstances: 1,
456+
DesiredInstances: desiredInstances,
455457
})
456458
require.NoError(t, err)
457-
return preset.ID
459+
return preset
458460
}
459461

460462
func setupTestDBPrebuild(

0 commit comments

Comments
 (0)