From 67eee75ab733428c00bf2bbcaa0ea2094039975b Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Mon, 2 May 2022 18:09:31 -0500 Subject: [PATCH 1/3] feat: add pprof and prometheus metrics to `coder server` --- cli/server.go | 37 +++++++++++- coderd/coderd.go | 12 +++- coderd/httpmw/prometheus.go | 109 ++++++++++++++++++++++++++++++++++++ go.mod | 6 ++ 4 files changed, 161 insertions(+), 3 deletions(-) create mode 100644 coderd/httpmw/prometheus.go diff --git a/cli/server.go b/cli/server.go index dd5460a9c83d3..a4dd4b888d6ac 100644 --- a/cli/server.go +++ b/cli/server.go @@ -12,6 +12,7 @@ import ( "log" "net" "net/http" + "net/http/pprof" "net/url" "os" "os/signal" @@ -23,6 +24,7 @@ import ( "github.com/google/go-github/v43/github" "github.com/pion/turn/v2" "github.com/pion/webrtc/v3" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/cobra" "golang.org/x/oauth2" xgithub "golang.org/x/oauth2/github" @@ -55,6 +57,10 @@ func server() *cobra.Command { var ( accessURL string address string + promEnabled bool + promAddress string + pprofEnabled bool + pprofAddress string cacheDir string dev bool devUserEmail string @@ -245,6 +251,15 @@ func server() *cobra.Command { } } + // This prevents the pprof import from being accidentally deleted. + var _ = pprof.Handler + if pprofEnabled { + defer serveHandler(cmd.Context(), logger, nil, pprofAddress, "pprof")() + } + if promEnabled { + defer serveHandler(cmd.Context(), logger, promhttp.Handler(), promAddress, "prometheus")() + } + errCh := make(chan error, 1) provisionerDaemons := make([]*provisionerd.Server, 0) for i := 0; uint8(i) < provisionerDaemonCount; i++ { @@ -400,8 +415,12 @@ func server() *cobra.Command { }, } - cliflag.StringVarP(root.Flags(), &accessURL, "access-url", "", "CODER_ACCESS_URL", "", "Specifies the external URL to access Coder") - cliflag.StringVarP(root.Flags(), &address, "address", "a", "CODER_ADDRESS", "127.0.0.1:3000", "The address to serve the API and dashboard") + cliflag.StringVarP(root.Flags(), &accessURL, "access-url", "", "CODER_ACCESS_URL", "", "Specifies the external URL to access Coder.") + cliflag.StringVarP(root.Flags(), &address, "address", "a", "CODER_ADDRESS", "127.0.0.1:3000", "The address to serve the API and dashboard.") + cliflag.BoolVarP(root.Flags(), &promEnabled, "enable-prometheus", "", "CODER_ENABLE_PROMETHEUS", false, "Enable serving prometheus metrics on the addressdefined by --prometheus-address.") + cliflag.StringVarP(root.Flags(), &promAddress, "prometheus-address", "", "CODER_PROMETHEUS_ADDRESS", "127.0.0.1:2112", "The address to serve prometheus metrics.") + cliflag.BoolVarP(root.Flags(), &promEnabled, "enable-pprof", "", "CODER_ENABLE_PPROF", false, "Enable serving pprof metrics on the address defined by --pprof-address.") + cliflag.StringVarP(root.Flags(), &pprofAddress, "pprof-address", "", "CODER_PPROF_ADDRESS", "127.0.0.1:6060", "The address to serve pprof.") // systemd uses the CACHE_DIRECTORY environment variable! cliflag.StringVarP(root.Flags(), &cacheDir, "cache-dir", "", "CACHE_DIRECTORY", filepath.Join(os.TempDir(), "coder-cache"), "Specifies a directory to cache binaries for provision operations.") cliflag.BoolVarP(root.Flags(), &dev, "dev", "", "CODER_DEV_MODE", false, "Serve Coder in dev mode for tinkering") @@ -661,6 +680,20 @@ func configureGithubOAuth2(accessURL *url.URL, clientID, clientSecret string, al }, nil } +func serveHandler(ctx context.Context, log slog.Logger, handler http.Handler, addr, name string) (closeFunc func()) { + log.Debug(ctx, "http server listening", slog.F("addr", addr), slog.F("name", name)) + + srv := &http.Server{Addr: addr, Handler: handler} + go func() { + err := srv.ListenAndServe() + if err != nil && !xerrors.Is(err, http.ErrServerClosed) { + log.Error(ctx, "http server listen", slog.F("name", name), slog.Error(err)) + } + }() + + return func() { _ = srv.Close() } +} + type datadogLogger struct { logger slog.Logger } diff --git a/coderd/coderd.go b/coderd/coderd.go index 37585097a6269..e15f41ac6f19b 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -10,6 +10,7 @@ import ( "time" "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" "github.com/pion/webrtc/v3" "google.golang.org/api/idtoken" @@ -68,9 +69,18 @@ func New(options *Options) (http.Handler, func()) { }) r := chi.NewRouter() + r.Use( + func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(middleware.NewWrapResponseWriter(w, r.ProtoMajor), r) + }) + }, + httpmw.Prometheus, + chitrace.Middleware(), + ) + r.Route("/api/v2", func(r chi.Router) { r.Use( - chitrace.Middleware(), // Specific routes can specify smaller limits. httpmw.RateLimitPerMinute(options.APIRateLimit), debugLogRequest(api.Logger), diff --git a/coderd/httpmw/prometheus.go b/coderd/httpmw/prometheus.go new file mode 100644 index 0000000000000..bd375efce7202 --- /dev/null +++ b/coderd/httpmw/prometheus.go @@ -0,0 +1,109 @@ +package httpmw + +import ( + "net/http" + "strconv" + "time" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + requestsProcessed = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "coderd", + Subsystem: "api", + Name: "requests_processed_total", + Help: "The total number of processed API requests", + }, []string{"code", "method", "path"}) + requestsConcurrent = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "api", + Name: "concurrent_requests", + Help: "The number of concurrent API requests", + }, []string{"method", "path"}) + websocketsConcurrent = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "api", + Name: "concurrent_websockets", + Help: "The total number of concurrent API websockets", + }, []string{"path"}) + websocketsDist = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "coderd", + Subsystem: "api", + Name: "websocket_durations_ms", + Help: "Websocket duration distribution of requests in milliseconds", + Buckets: []float64{ + durationToFloatMs(01 * time.Millisecond), + durationToFloatMs(01 * time.Second), + durationToFloatMs(01 * time.Minute), + durationToFloatMs(01 * time.Hour), + durationToFloatMs(15 * time.Hour), + durationToFloatMs(30 * time.Hour), + }, + }, []string{"path"}) + requestsDist = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "coderd", + Subsystem: "api", + Name: "request_latencies_ms", + Help: "Latency distribution of requests in milliseconds", + Buckets: []float64{1, 5, 10, 25, 50, 100, 500, 1000, 5000, 10000, 30000}, + }, []string{"method", "path"}) +) + +func durationToFloatMs(d time.Duration) float64 { + return float64(d.Milliseconds()) +} + +func Prometheus(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var ( + start = time.Now() + method = r.Method + rctx = chi.RouteContext(r.Context()) + path = rctx.RoutePattern() + ) + sw, ok := w.(middleware.WrapResponseWriter) + if !ok { + panic("dev error: http.ResponseWriter is not middleware.WrapResponseWriter") + } + + var ( + dist *prometheus.HistogramVec + distOpts []string + ) + // We want to count websockets separately. + if isWebsocketUpgrade(r) { + websocketsConcurrent.WithLabelValues(path).Inc() + defer websocketsConcurrent.WithLabelValues(path).Dec() + + dist = websocketsDist + distOpts = []string{path} + } else { + requestsConcurrent.WithLabelValues(method, path).Inc() + defer requestsConcurrent.WithLabelValues(method, path).Dec() + + dist = requestsDist + distOpts = []string{method, path} + } + + next.ServeHTTP(w, r) + statusStr := strconv.Itoa(sw.Status()) + + requestsProcessed.WithLabelValues(statusStr, method, path).Inc() + dist.WithLabelValues(distOpts...).Observe(float64(time.Since(start).Milliseconds())) + }) +} + +func isWebsocketUpgrade(r *http.Request) bool { + vs := r.Header.Values("Upgrade") + for _, v := range vs { + if v == "websocket" { + return true + } + } + return false +} diff --git a/go.mod b/go.mod index 0b4cf04a078c0..128a705b78d79 100644 --- a/go.mod +++ b/go.mod @@ -90,6 +90,7 @@ require ( github.com/pion/webrtc/v3 v3.1.34 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 github.com/pkg/sftp v1.13.4 + github.com/prometheus/client_golang v1.12.1 github.com/quasilyte/go-ruleguard/dsl v0.3.19 github.com/robfig/cron/v3 v3.0.1 github.com/spf13/cobra v1.4.0 @@ -135,6 +136,7 @@ require ( github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.1.2 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/charmbracelet/bubbles v0.10.3 // indirect @@ -184,6 +186,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect github.com/miekg/dns v1.1.45 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect @@ -213,6 +216,9 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pquerna/cachecontrol v0.1.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/sirupsen/logrus v1.8.1 // indirect From 1c4eb1d1becb334821184ee3992dd6ce4e4bf441 Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Mon, 2 May 2022 18:45:26 -0500 Subject: [PATCH 2/3] rename flags --- cli/server.go | 4 ++-- coderd/httpmw/prometheus.go | 31 ++++++++++++++++--------------- site/src/api/typesGenerated.ts | 7 ++++--- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/cli/server.go b/cli/server.go index a4dd4b888d6ac..92b7b437b7d3e 100644 --- a/cli/server.go +++ b/cli/server.go @@ -417,9 +417,9 @@ func server() *cobra.Command { cliflag.StringVarP(root.Flags(), &accessURL, "access-url", "", "CODER_ACCESS_URL", "", "Specifies the external URL to access Coder.") cliflag.StringVarP(root.Flags(), &address, "address", "a", "CODER_ADDRESS", "127.0.0.1:3000", "The address to serve the API and dashboard.") - cliflag.BoolVarP(root.Flags(), &promEnabled, "enable-prometheus", "", "CODER_ENABLE_PROMETHEUS", false, "Enable serving prometheus metrics on the addressdefined by --prometheus-address.") + cliflag.BoolVarP(root.Flags(), &promEnabled, "prometheus-enable", "", "CODER_PROMETHEUS_ENABLE", false, "Enable serving prometheus metrics on the addressdefined by --prometheus-address.") cliflag.StringVarP(root.Flags(), &promAddress, "prometheus-address", "", "CODER_PROMETHEUS_ADDRESS", "127.0.0.1:2112", "The address to serve prometheus metrics.") - cliflag.BoolVarP(root.Flags(), &promEnabled, "enable-pprof", "", "CODER_ENABLE_PPROF", false, "Enable serving pprof metrics on the address defined by --pprof-address.") + cliflag.BoolVarP(root.Flags(), &promEnabled, "pprof-enable", "", "CODER_PPROF_ENABLE", false, "Enable serving pprof metrics on the address defined by --pprof-address.") cliflag.StringVarP(root.Flags(), &pprofAddress, "pprof-address", "", "CODER_PPROF_ADDRESS", "127.0.0.1:6060", "The address to serve pprof.") // systemd uses the CACHE_DIRECTORY environment variable! cliflag.StringVarP(root.Flags(), &cacheDir, "cache-dir", "", "CACHE_DIRECTORY", filepath.Join(os.TempDir(), "coder-cache"), "Specifies a directory to cache binaries for provision operations.") diff --git a/coderd/httpmw/prometheus.go b/coderd/httpmw/prometheus.go index bd375efce7202..e03966ff9788f 100644 --- a/coderd/httpmw/prometheus.go +++ b/coderd/httpmw/prometheus.go @@ -6,7 +6,7 @@ import ( "time" "github.com/go-chi/chi/v5" - "github.com/go-chi/chi/v5/middleware" + chimw "github.com/go-chi/chi/v5/middleware" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -19,18 +19,18 @@ var ( Name: "requests_processed_total", Help: "The total number of processed API requests", }, []string{"code", "method", "path"}) - requestsConcurrent = promauto.NewGaugeVec(prometheus.GaugeOpts{ + requestsConcurrent = promauto.NewGauge(prometheus.GaugeOpts{ Namespace: "coderd", Subsystem: "api", Name: "concurrent_requests", Help: "The number of concurrent API requests", - }, []string{"method", "path"}) - websocketsConcurrent = promauto.NewGaugeVec(prometheus.GaugeOpts{ + }) + websocketsConcurrent = promauto.NewGauge(prometheus.GaugeOpts{ Namespace: "coderd", Subsystem: "api", Name: "concurrent_websockets", Help: "The total number of concurrent API websockets", - }, []string{"path"}) + }) websocketsDist = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "coderd", Subsystem: "api", @@ -64,11 +64,10 @@ func Prometheus(next http.Handler) http.Handler { start = time.Now() method = r.Method rctx = chi.RouteContext(r.Context()) - path = rctx.RoutePattern() ) - sw, ok := w.(middleware.WrapResponseWriter) + sw, ok := w.(chimw.WrapResponseWriter) if !ok { - panic("dev error: http.ResponseWriter is not middleware.WrapResponseWriter") + panic("dev error: http.ResponseWriter is not chimw.WrapResponseWriter") } var ( @@ -77,24 +76,26 @@ func Prometheus(next http.Handler) http.Handler { ) // We want to count websockets separately. if isWebsocketUpgrade(r) { - websocketsConcurrent.WithLabelValues(path).Inc() - defer websocketsConcurrent.WithLabelValues(path).Dec() + websocketsConcurrent.Inc() + defer websocketsConcurrent.Dec() dist = websocketsDist - distOpts = []string{path} } else { - requestsConcurrent.WithLabelValues(method, path).Inc() - defer requestsConcurrent.WithLabelValues(method, path).Dec() + requestsConcurrent.Inc() + defer requestsConcurrent.Dec() dist = requestsDist - distOpts = []string{method, path} + distOpts = []string{method} } next.ServeHTTP(w, r) + + path := rctx.RoutePattern() + distOpts = append(distOpts, path) statusStr := strconv.Itoa(sw.Status()) requestsProcessed.WithLabelValues(statusStr, method, path).Inc() - dist.WithLabelValues(distOpts...).Observe(float64(time.Since(start).Milliseconds())) + dist.WithLabelValues(distOpts...).Observe(float64(time.Since(start)) / 1e6) }) } diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index da6cdc6925354..ee4add928bb11 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -91,6 +91,7 @@ export interface CreateWorkspaceBuildRequest { // This is likely an enum in an external package ("github.com/coder/coder/coderd/database.WorkspaceTransition") readonly transition: string readonly dry_run: boolean + readonly state: string } // From codersdk/organizations.go:52:6 @@ -265,12 +266,12 @@ export interface UpdateUserProfileRequest { readonly username: string } -// From codersdk/workspaces.go:94:6 +// From codersdk/workspaces.go:95:6 export interface UpdateWorkspaceAutostartRequest { readonly schedule: string } -// From codersdk/workspaces.go:114:6 +// From codersdk/workspaces.go:115:6 export interface UpdateWorkspaceAutostopRequest { readonly schedule: string } @@ -366,7 +367,7 @@ export interface WorkspaceAgentResourceMetadata { readonly cpu_mhz: number } -// From codersdk/workspacebuilds.go:17:6 +// From codersdk/workspacebuilds.go:18:6 export interface WorkspaceBuild { readonly id: string readonly created_at: string From dca1216a89c3caac26a85739b3f413c276fe7dd2 Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Mon, 2 May 2022 18:50:08 -0500 Subject: [PATCH 3/3] fixup! rename flags --- cli/server.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cli/server.go b/cli/server.go index 92b7b437b7d3e..37118753f858d 100644 --- a/cli/server.go +++ b/cli/server.go @@ -254,9 +254,11 @@ func server() *cobra.Command { // This prevents the pprof import from being accidentally deleted. var _ = pprof.Handler if pprofEnabled { + //nolint:revive defer serveHandler(cmd.Context(), logger, nil, pprofAddress, "pprof")() } if promEnabled { + //nolint:revive defer serveHandler(cmd.Context(), logger, promhttp.Handler(), promAddress, "prometheus")() } @@ -680,14 +682,14 @@ func configureGithubOAuth2(accessURL *url.URL, clientID, clientSecret string, al }, nil } -func serveHandler(ctx context.Context, log slog.Logger, handler http.Handler, addr, name string) (closeFunc func()) { - log.Debug(ctx, "http server listening", slog.F("addr", addr), slog.F("name", name)) +func serveHandler(ctx context.Context, logger slog.Logger, handler http.Handler, addr, name string) (closeFunc func()) { + logger.Debug(ctx, "http server listening", slog.F("addr", addr), slog.F("name", name)) srv := &http.Server{Addr: addr, Handler: handler} go func() { err := srv.ListenAndServe() if err != nil && !xerrors.Is(err, http.ErrServerClosed) { - log.Error(ctx, "http server listen", slog.F("name", name), slog.Error(err)) + logger.Error(ctx, "http server listen", slog.F("name", name), slog.Error(err)) } }()