diff --git a/coderd/httpmw/prometheus.go b/coderd/httpmw/prometheus.go index b96be84e879e3..8b7b33381c74d 100644 --- a/coderd/httpmw/prometheus.go +++ b/coderd/httpmw/prometheus.go @@ -3,6 +3,7 @@ package httpmw import ( "net/http" "strconv" + "strings" "time" "github.com/go-chi/chi/v5" @@ -22,18 +23,18 @@ func Prometheus(register prometheus.Registerer) func(http.Handler) http.Handler Name: "requests_processed_total", Help: "The total number of processed API requests", }, []string{"code", "method", "path"}) - requestsConcurrent := factory.NewGauge(prometheus.GaugeOpts{ + requestsConcurrent := factory.NewGaugeVec(prometheus.GaugeOpts{ Namespace: "coderd", Subsystem: "api", Name: "concurrent_requests", Help: "The number of concurrent API requests.", - }) - websocketsConcurrent := factory.NewGauge(prometheus.GaugeOpts{ + }, []string{"method", "path"}) + websocketsConcurrent := factory.NewGaugeVec(prometheus.GaugeOpts{ Namespace: "coderd", Subsystem: "api", Name: "concurrent_websockets", Help: "The total number of concurrent API websockets.", - }) + }, []string{"path"}) websocketsDist := factory.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "coderd", Subsystem: "api", @@ -61,7 +62,6 @@ func Prometheus(register prometheus.Registerer) func(http.Handler) http.Handler var ( start = time.Now() method = r.Method - rctx = chi.RouteContext(r.Context()) ) sw, ok := w.(*tracing.StatusWriter) @@ -72,16 +72,18 @@ func Prometheus(register prometheus.Registerer) func(http.Handler) http.Handler var ( dist *prometheus.HistogramVec distOpts []string + path = getRoutePattern(r) ) + // We want to count WebSockets separately. if httpapi.IsWebsocketUpgrade(r) { - websocketsConcurrent.Inc() - defer websocketsConcurrent.Dec() + websocketsConcurrent.WithLabelValues(path).Inc() + defer websocketsConcurrent.WithLabelValues(path).Dec() dist = websocketsDist } else { - requestsConcurrent.Inc() - defer requestsConcurrent.Dec() + requestsConcurrent.WithLabelValues(method, path).Inc() + defer requestsConcurrent.WithLabelValues(method, path).Dec() dist = requestsDist distOpts = []string{method} @@ -89,7 +91,6 @@ func Prometheus(register prometheus.Registerer) func(http.Handler) http.Handler next.ServeHTTP(w, r) - path := rctx.RoutePattern() distOpts = append(distOpts, path) statusStr := strconv.Itoa(sw.Status) @@ -98,3 +99,34 @@ func Prometheus(register prometheus.Registerer) func(http.Handler) http.Handler }) } } + +func getRoutePattern(r *http.Request) string { + rctx := chi.RouteContext(r.Context()) + if rctx == nil { + return "" + } + + if pattern := rctx.RoutePattern(); pattern != "" { + // Pattern is already available + return pattern + } + + routePath := r.URL.Path + if r.URL.RawPath != "" { + routePath = r.URL.RawPath + } + + tctx := chi.NewRouteContext() + routes := rctx.Routes + if routes != nil && !routes.Match(tctx, r.Method, routePath) { + // No matching pattern. /api/* requests will be matched as "UNKNOWN" + // All other ones will be matched as "STATIC". + if strings.HasPrefix(routePath, "/api/") { + return "UNKNOWN" + } + return "STATIC" + } + + // tctx has the updated pattern, since Match mutates it + return tctx.RoutePattern() +} diff --git a/coderd/httpmw/prometheus_test.go b/coderd/httpmw/prometheus_test.go index a51eea5d00312..d40558f5ca5e7 100644 --- a/coderd/httpmw/prometheus_test.go +++ b/coderd/httpmw/prometheus_test.go @@ -8,14 +8,19 @@ import ( "github.com/go-chi/chi/v5" "github.com/prometheus/client_golang/prometheus" + cm "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/coder/coder/v2/coderd/httpmw" "github.com/coder/coder/v2/coderd/tracing" + "github.com/coder/coder/v2/testutil" + "github.com/coder/websocket" ) func TestPrometheus(t *testing.T) { t.Parallel() + t.Run("All", func(t *testing.T) { t.Parallel() req := httptest.NewRequest("GET", "/", nil) @@ -29,4 +34,90 @@ func TestPrometheus(t *testing.T) { require.NoError(t, err) require.Greater(t, len(metrics), 0) }) + + t.Run("Concurrent", func(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) + defer cancel() + + reg := prometheus.NewRegistry() + promMW := httpmw.Prometheus(reg) + + // Create a test handler to simulate a WebSocket connection + testHandler := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(rw, r, nil) + if !assert.NoError(t, err, "failed to accept websocket") { + return + } + defer conn.Close(websocket.StatusGoingAway, "") + }) + + wrappedHandler := promMW(testHandler) + + r := chi.NewRouter() + r.Use(tracing.StatusWriterMiddleware, promMW) + r.Get("/api/v2/build/{build}/logs", func(rw http.ResponseWriter, r *http.Request) { + wrappedHandler.ServeHTTP(rw, r) + }) + + srv := httptest.NewServer(r) + defer srv.Close() + // nolint: bodyclose + conn, _, err := websocket.Dial(ctx, srv.URL+"/api/v2/build/1/logs", nil) + require.NoError(t, err, "failed to dial WebSocket") + defer conn.Close(websocket.StatusNormalClosure, "") + + metrics, err := reg.Gather() + require.NoError(t, err) + require.Greater(t, len(metrics), 0) + metricLabels := getMetricLabels(metrics) + + concurrentWebsockets, ok := metricLabels["coderd_api_concurrent_websockets"] + require.True(t, ok, "coderd_api_concurrent_websockets metric not found") + require.Equal(t, "/api/v2/build/{build}/logs", concurrentWebsockets["path"]) + }) + + t.Run("UserRoute", func(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + promMW := httpmw.Prometheus(reg) + + r := chi.NewRouter() + r.With(promMW).Get("/api/v2/users/{user}", func(w http.ResponseWriter, r *http.Request) {}) + + req := httptest.NewRequest("GET", "/api/v2/users/john", nil) + + sw := &tracing.StatusWriter{ResponseWriter: httptest.NewRecorder()} + + r.ServeHTTP(sw, req) + + metrics, err := reg.Gather() + require.NoError(t, err) + require.Greater(t, len(metrics), 0) + metricLabels := getMetricLabels(metrics) + + reqProcessed, ok := metricLabels["coderd_api_requests_processed_total"] + require.True(t, ok, "coderd_api_requests_processed_total metric not found") + require.Equal(t, "/api/v2/users/{user}", reqProcessed["path"]) + require.Equal(t, "GET", reqProcessed["method"]) + + concurrentRequests, ok := metricLabels["coderd_api_concurrent_requests"] + require.True(t, ok, "coderd_api_concurrent_requests metric not found") + require.Equal(t, "/api/v2/users/{user}", concurrentRequests["path"]) + require.Equal(t, "GET", concurrentRequests["method"]) + }) +} + +func getMetricLabels(metrics []*cm.MetricFamily) map[string]map[string]string { + metricLabels := map[string]map[string]string{} + for _, metricFamily := range metrics { + metricName := metricFamily.GetName() + metricLabels[metricName] = map[string]string{} + for _, metric := range metricFamily.GetMetric() { + for _, labelPair := range metric.GetLabel() { + metricLabels[metricName][labelPair.GetName()] = labelPair.GetValue() + } + } + } + return metricLabels }