Skip to content

feat: expose agent metrics via Prometheus endpoint #7011

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Apr 7, 2023
Merged

feat: expose agent metrics via Prometheus endpoint #7011

Changes from 1 commit
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
8d4e67d
WIP
mtojek Apr 3, 2023
da729e6
Merge branch 'main' into 6724-metrics
mtojek Apr 4, 2023
9ad09b2
WIP
mtojek Apr 4, 2023
440657c
WIP
mtojek Apr 5, 2023
8764f89
Agents
mtojek Apr 5, 2023
663b5d5
fix
mtojek Apr 5, 2023
63aff5e
1min
mtojek Apr 5, 2023
3905481
fix
mtojek Apr 5, 2023
f8d6f46
WIP
mtojek Apr 5, 2023
d487a77
Test
mtojek Apr 5, 2023
7acbaf0
docs
mtojek Apr 5, 2023
7418779
fmt
mtojek Apr 5, 2023
3a8e4e6
Add timer to measure the metrics collection
mtojek Apr 6, 2023
b5d0581
Use CachedGaugeVec
mtojek Apr 6, 2023
e4d708b
Unit tests
mtojek Apr 6, 2023
e0669f0
Address PR comments
mtojek Apr 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Unit tests
  • Loading branch information
@mtojek
mtojek committed Apr 6, 2023
commit e4d708b8f36214712989605f1589ee0f1b9dc613
140 changes: 140 additions & 0 deletions coderd/prometheusmetrics/collector_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
package prometheusmetrics_test

import (
"sort"
"testing"

"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/coder/coder/coderd/prometheusmetrics"
)

func TestCollector_Add(t *testing.T) {
t.Parallel()

// given
agentsGauge := prometheusmetrics.NewCachedGaugeVec(prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "coderd",
Subsystem: "agents",
Name: "up",
Help: "The number of active agents per workspace.",
}, []string{"username", "workspace_name"}))

// when
agentsGauge.WithLabelValues(prometheusmetrics.VectorOperationAdd, 7, "first user", "my workspace")
agentsGauge.WithLabelValues(prometheusmetrics.VectorOperationAdd, 23, "second user", "your workspace")
agentsGauge.WithLabelValues(prometheusmetrics.VectorOperationAdd, 1, "first user", "my workspace")
agentsGauge.WithLabelValues(prometheusmetrics.VectorOperationAdd, 25, "second user", "your workspace")
agentsGauge.Commit()

// then
ch := make(chan prometheus.Metric, 2)
agentsGauge.Collect(ch)

metrics := collectAndSortMetrics(t, agentsGauge, 2)

assert.Equal(t, "first user", metrics[0].Label[0].GetValue()) // Username
assert.Equal(t, "my workspace", metrics[0].Label[1].GetValue()) // Workspace name
assert.Equal(t, 8, int(metrics[0].Gauge.GetValue())) // Metric value

assert.Equal(t, "second user", metrics[1].Label[0].GetValue()) // Username
assert.Equal(t, "your workspace", metrics[1].Label[1].GetValue()) // Workspace name
assert.Equal(t, 48, int(metrics[1].Gauge.GetValue())) // Metric value
}

func TestCollector_Set(t *testing.T) {
t.Parallel()

// given
agentsGauge := prometheusmetrics.NewCachedGaugeVec(prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "coderd",
Subsystem: "agents",
Name: "up",
Help: "The number of active agents per workspace.",
}, []string{"username", "workspace_name"}))

// when
agentsGauge.WithLabelValues(prometheusmetrics.VectorOperationSet, 3, "first user", "my workspace")
agentsGauge.WithLabelValues(prometheusmetrics.VectorOperationSet, 4, "second user", "your workspace")
agentsGauge.WithLabelValues(prometheusmetrics.VectorOperationSet, 5, "first user", "my workspace")
agentsGauge.WithLabelValues(prometheusmetrics.VectorOperationSet, 6, "second user", "your workspace")
agentsGauge.Commit()

// then
ch := make(chan prometheus.Metric, 2)
agentsGauge.Collect(ch)

metrics := collectAndSortMetrics(t, agentsGauge, 2)

assert.Equal(t, "first user", metrics[0].Label[0].GetValue()) // Username
assert.Equal(t, "my workspace", metrics[0].Label[1].GetValue()) // Workspace name
assert.Equal(t, 5, int(metrics[0].Gauge.GetValue())) // Metric value

assert.Equal(t, "second user", metrics[1].Label[0].GetValue()) // Username
assert.Equal(t, "your workspace", metrics[1].Label[1].GetValue()) // Workspace name
assert.Equal(t, 6, int(metrics[1].Gauge.GetValue())) // Metric value
}

func TestCollector_Set_Add(t *testing.T) {
t.Parallel()

// given
agentsGauge := prometheusmetrics.NewCachedGaugeVec(prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "coderd",
Subsystem: "agents",
Name: "up",
Help: "The number of active agents per workspace.",
}, []string{"username", "workspace_name"}))

// when
agentsGauge.WithLabelValues(prometheusmetrics.VectorOperationAdd, 9, "first user", "my workspace")
agentsGauge.WithLabelValues(prometheusmetrics.VectorOperationAdd, 8, "second user", "your workspace")
agentsGauge.WithLabelValues(prometheusmetrics.VectorOperationAdd, 7, "first user", "my workspace")
agentsGauge.WithLabelValues(prometheusmetrics.VectorOperationAdd, 6, "second user", "your workspace")
agentsGauge.WithLabelValues(prometheusmetrics.VectorOperationSet, 5, "first user", "my workspace")
agentsGauge.WithLabelValues(prometheusmetrics.VectorOperationSet, 4, "second user", "your workspace")
agentsGauge.WithLabelValues(prometheusmetrics.VectorOperationAdd, 3, "first user", "my workspace")
agentsGauge.WithLabelValues(prometheusmetrics.VectorOperationAdd, 2, "second user", "your workspace")
agentsGauge.Commit()

// then
ch := make(chan prometheus.Metric, 2)
agentsGauge.Collect(ch)

metrics := collectAndSortMetrics(t, agentsGauge, 2)

assert.Equal(t, "first user", metrics[0].Label[0].GetValue()) // Username
assert.Equal(t, "my workspace", metrics[0].Label[1].GetValue()) // Workspace name
assert.Equal(t, 8, int(metrics[0].Gauge.GetValue())) // Metric value

assert.Equal(t, "second user", metrics[1].Label[0].GetValue()) // Username
assert.Equal(t, "your workspace", metrics[1].Label[1].GetValue()) // Workspace name
assert.Equal(t, 6, int(metrics[1].Gauge.GetValue())) // Metric value
}

func collectAndSortMetrics(t *testing.T, collector prometheus.Collector, count int) []dto.Metric {
ch := make(chan prometheus.Metric, count)
defer close(ch)

var metrics []dto.Metric

collector.Collect(ch)
for i := 0; i < count; i++ {
m := <-ch

var metric dto.Metric
err := m.Write(&metric)
require.NoError(t, err)

metrics = append(metrics, metric)
}

// Ensure always the same order of metrics
sort.Slice(metrics, func(i, j int) bool {
return sort.StringsAreSorted([]string{metrics[i].Label[0].GetValue(), metrics[j].Label[1].GetValue()})
})
return metrics
}