Skip to content

Commit 1d2af9c

Browse files
gcp-cherry-pick-bot[bot]ibetitsmike
gcp-cherry-pick-bot[bot]
and
ibetitsmike
authored
feat: add path & method labels to prometheus metrics for current requests (cherry-pick #17362) (#17494)
Cherry-picked feat: add path & method labels to prometheus metrics for current requests (#17362) Closes: #17212 Co-authored-by: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com>
1 parent 0a387c5 commit 1d2af9c

File tree

2 files changed

+133
-10
lines changed

2 files changed

+133
-10
lines changed

‎coderd/httpmw/prometheus.go

+42-10
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package httpmw
33
import (
44
"net/http"
55
"strconv"
6+
"strings"
67
"time"
78

89
"github.com/go-chi/chi/v5"
@@ -22,18 +23,18 @@ func Prometheus(register prometheus.Registerer) func(http.Handler) http.Handler
2223
Name: "requests_processed_total",
2324
Help: "The total number of processed API requests",
2425
}, []string{"code", "method", "path"})
25-
requestsConcurrent := factory.NewGauge(prometheus.GaugeOpts{
26+
requestsConcurrent := factory.NewGaugeVec(prometheus.GaugeOpts{
2627
Namespace: "coderd",
2728
Subsystem: "api",
2829
Name: "concurrent_requests",
2930
Help: "The number of concurrent API requests.",
30-
})
31-
websocketsConcurrent := factory.NewGauge(prometheus.GaugeOpts{
31+
}, []string{"method", "path"})
32+
websocketsConcurrent := factory.NewGaugeVec(prometheus.GaugeOpts{
3233
Namespace: "coderd",
3334
Subsystem: "api",
3435
Name: "concurrent_websockets",
3536
Help: "The total number of concurrent API websockets.",
36-
})
37+
}, []string{"path"})
3738
websocketsDist := factory.NewHistogramVec(prometheus.HistogramOpts{
3839
Namespace: "coderd",
3940
Subsystem: "api",
@@ -61,7 +62,6 @@ func Prometheus(register prometheus.Registerer) func(http.Handler) http.Handler
6162
var (
6263
start = time.Now()
6364
method = r.Method
64-
rctx = chi.RouteContext(r.Context())
6565
)
6666

6767
sw, ok := w.(*tracing.StatusWriter)
@@ -72,24 +72,25 @@ func Prometheus(register prometheus.Registerer) func(http.Handler) http.Handler
7272
var (
7373
dist *prometheus.HistogramVec
7474
distOpts []string
75+
path = getRoutePattern(r)
7576
)
77+
7678
// We want to count WebSockets separately.
7779
if httpapi.IsWebsocketUpgrade(r) {
78-
websocketsConcurrent.Inc()
79-
defer websocketsConcurrent.Dec()
80+
websocketsConcurrent.WithLabelValues(path).Inc()
81+
defer websocketsConcurrent.WithLabelValues(path).Dec()
8082

8183
dist = websocketsDist
8284
} else {
83-
requestsConcurrent.Inc()
84-
defer requestsConcurrent.Dec()
85+
requestsConcurrent.WithLabelValues(method, path).Inc()
86+
defer requestsConcurrent.WithLabelValues(method, path).Dec()
8587

8688
dist = requestsDist
8789
distOpts = []string{method}
8890
}
8991

9092
next.ServeHTTP(w, r)
9193

92-
path := rctx.RoutePattern()
9394
distOpts = append(distOpts, path)
9495
statusStr := strconv.Itoa(sw.Status)
9596

@@ -98,3 +99,34 @@ func Prometheus(register prometheus.Registerer) func(http.Handler) http.Handler
9899
})
99100
}
100101
}
102+
103+
func getRoutePattern(r *http.Request) string {
104+
rctx := chi.RouteContext(r.Context())
105+
if rctx == nil {
106+
return ""
107+
}
108+
109+
if pattern := rctx.RoutePattern(); pattern != "" {
110+
// Pattern is already available
111+
return pattern
112+
}
113+
114+
routePath := r.URL.Path
115+
if r.URL.RawPath != "" {
116+
routePath = r.URL.RawPath
117+
}
118+
119+
tctx := chi.NewRouteContext()
120+
routes := rctx.Routes
121+
if routes != nil && !routes.Match(tctx, r.Method, routePath) {
122+
// No matching pattern. /api/* requests will be matched as "UNKNOWN"
123+
// All other ones will be matched as "STATIC".
124+
if strings.HasPrefix(routePath, "/api/") {
125+
return "UNKNOWN"
126+
}
127+
return "STATIC"
128+
}
129+
130+
// tctx has the updated pattern, since Match mutates it
131+
return tctx.RoutePattern()
132+
}

‎coderd/httpmw/prometheus_test.go

+91
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,19 @@ import (
88

99
"github.com/go-chi/chi/v5"
1010
"github.com/prometheus/client_golang/prometheus"
11+
cm "github.com/prometheus/client_model/go"
12+
"github.com/stretchr/testify/assert"
1113
"github.com/stretchr/testify/require"
1214

1315
"github.com/coder/coder/v2/coderd/httpmw"
1416
"github.com/coder/coder/v2/coderd/tracing"
17+
"github.com/coder/coder/v2/testutil"
18+
"github.com/coder/websocket"
1519
)
1620

1721
func TestPrometheus(t *testing.T) {
1822
t.Parallel()
23+
1924
t.Run("All", func(t *testing.T) {
2025
t.Parallel()
2126
req := httptest.NewRequest("GET", "/", nil)
@@ -29,4 +34,90 @@ func TestPrometheus(t *testing.T) {
2934
require.NoError(t, err)
3035
require.Greater(t, len(metrics), 0)
3136
})
37+
38+
t.Run("Concurrent", func(t *testing.T) {
39+
t.Parallel()
40+
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
41+
defer cancel()
42+
43+
reg := prometheus.NewRegistry()
44+
promMW := httpmw.Prometheus(reg)
45+
46+
// Create a test handler to simulate a WebSocket connection
47+
testHandler := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
48+
conn, err := websocket.Accept(rw, r, nil)
49+
if !assert.NoError(t, err, "failed to accept websocket") {
50+
return
51+
}
52+
defer conn.Close(websocket.StatusGoingAway, "")
53+
})
54+
55+
wrappedHandler := promMW(testHandler)
56+
57+
r := chi.NewRouter()
58+
r.Use(tracing.StatusWriterMiddleware, promMW)
59+
r.Get("/api/v2/build/{build}/logs", func(rw http.ResponseWriter, r *http.Request) {
60+
wrappedHandler.ServeHTTP(rw, r)
61+
})
62+
63+
srv := httptest.NewServer(r)
64+
defer srv.Close()
65+
// nolint: bodyclose
66+
conn, _, err := websocket.Dial(ctx, srv.URL+"/api/v2/build/1/logs", nil)
67+
require.NoError(t, err, "failed to dial WebSocket")
68+
defer conn.Close(websocket.StatusNormalClosure, "")
69+
70+
metrics, err := reg.Gather()
71+
require.NoError(t, err)
72+
require.Greater(t, len(metrics), 0)
73+
metricLabels := getMetricLabels(metrics)
74+
75+
concurrentWebsockets, ok := metricLabels["coderd_api_concurrent_websockets"]
76+
require.True(t, ok, "coderd_api_concurrent_websockets metric not found")
77+
require.Equal(t, "/api/v2/build/{build}/logs", concurrentWebsockets["path"])
78+
})
79+
80+
t.Run("UserRoute", func(t *testing.T) {
81+
t.Parallel()
82+
reg := prometheus.NewRegistry()
83+
promMW := httpmw.Prometheus(reg)
84+
85+
r := chi.NewRouter()
86+
r.With(promMW).Get("/api/v2/users/{user}", func(w http.ResponseWriter, r *http.Request) {})
87+
88+
req := httptest.NewRequest("GET", "/api/v2/users/john", nil)
89+
90+
sw := &tracing.StatusWriter{ResponseWriter: httptest.NewRecorder()}
91+
92+
r.ServeHTTP(sw, req)
93+
94+
metrics, err := reg.Gather()
95+
require.NoError(t, err)
96+
require.Greater(t, len(metrics), 0)
97+
metricLabels := getMetricLabels(metrics)
98+
99+
reqProcessed, ok := metricLabels["coderd_api_requests_processed_total"]
100+
require.True(t, ok, "coderd_api_requests_processed_total metric not found")
101+
require.Equal(t, "/api/v2/users/{user}", reqProcessed["path"])
102+
require.Equal(t, "GET", reqProcessed["method"])
103+
104+
concurrentRequests, ok := metricLabels["coderd_api_concurrent_requests"]
105+
require.True(t, ok, "coderd_api_concurrent_requests metric not found")
106+
require.Equal(t, "/api/v2/users/{user}", concurrentRequests["path"])
107+
require.Equal(t, "GET", concurrentRequests["method"])
108+
})
109+
}
110+
111+
func getMetricLabels(metrics []*cm.MetricFamily) map[string]map[string]string {
112+
metricLabels := map[string]map[string]string{}
113+
for _, metricFamily := range metrics {
114+
metricName := metricFamily.GetName()
115+
metricLabels[metricName] = map[string]string{}
116+
for _, metric := range metricFamily.GetMetric() {
117+
for _, labelPair := range metric.GetLabel() {
118+
metricLabels[metricName][labelPair.GetName()] = labelPair.GetValue()
119+
}
120+
}
121+
}
122+
return metricLabels
32123
}

0 commit comments

Comments
 (0)