diff --git a/cli/server.go b/cli/server.go index dd5460a9c83d3..37118753f858d 100644 --- a/cli/server.go +++ b/cli/server.go @@ -12,6 +12,7 @@ import ( "log" "net" "net/http" + "net/http/pprof" "net/url" "os" "os/signal" @@ -23,6 +24,7 @@ import ( "github.com/google/go-github/v43/github" "github.com/pion/turn/v2" "github.com/pion/webrtc/v3" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/cobra" "golang.org/x/oauth2" xgithub "golang.org/x/oauth2/github" @@ -55,6 +57,10 @@ func server() *cobra.Command { var ( accessURL string address string + promEnabled bool + promAddress string + pprofEnabled bool + pprofAddress string cacheDir string dev bool devUserEmail string @@ -245,6 +251,17 @@ func server() *cobra.Command { } } + // This prevents the pprof import from being accidentally deleted. + var _ = pprof.Handler + if pprofEnabled { + //nolint:revive + defer serveHandler(cmd.Context(), logger, nil, pprofAddress, "pprof")() + } + if promEnabled { + //nolint:revive + defer serveHandler(cmd.Context(), logger, promhttp.Handler(), promAddress, "prometheus")() + } + errCh := make(chan error, 1) provisionerDaemons := make([]*provisionerd.Server, 0) for i := 0; uint8(i) < provisionerDaemonCount; i++ { @@ -400,8 +417,12 @@ func server() *cobra.Command { }, } - cliflag.StringVarP(root.Flags(), &accessURL, "access-url", "", "CODER_ACCESS_URL", "", "Specifies the external URL to access Coder") - cliflag.StringVarP(root.Flags(), &address, "address", "a", "CODER_ADDRESS", "127.0.0.1:3000", "The address to serve the API and dashboard") + cliflag.StringVarP(root.Flags(), &accessURL, "access-url", "", "CODER_ACCESS_URL", "", "Specifies the external URL to access Coder.") + cliflag.StringVarP(root.Flags(), &address, "address", "a", "CODER_ADDRESS", "127.0.0.1:3000", "The address to serve the API and dashboard.") + cliflag.BoolVarP(root.Flags(), &promEnabled, "prometheus-enable", "", "CODER_PROMETHEUS_ENABLE", false, "Enable serving prometheus metrics on the addressdefined by --prometheus-address.") + cliflag.StringVarP(root.Flags(), &promAddress, "prometheus-address", "", "CODER_PROMETHEUS_ADDRESS", "127.0.0.1:2112", "The address to serve prometheus metrics.") + cliflag.BoolVarP(root.Flags(), &promEnabled, "pprof-enable", "", "CODER_PPROF_ENABLE", false, "Enable serving pprof metrics on the address defined by --pprof-address.") + cliflag.StringVarP(root.Flags(), &pprofAddress, "pprof-address", "", "CODER_PPROF_ADDRESS", "127.0.0.1:6060", "The address to serve pprof.") // systemd uses the CACHE_DIRECTORY environment variable! cliflag.StringVarP(root.Flags(), &cacheDir, "cache-dir", "", "CACHE_DIRECTORY", filepath.Join(os.TempDir(), "coder-cache"), "Specifies a directory to cache binaries for provision operations.") cliflag.BoolVarP(root.Flags(), &dev, "dev", "", "CODER_DEV_MODE", false, "Serve Coder in dev mode for tinkering") @@ -661,6 +682,20 @@ func configureGithubOAuth2(accessURL *url.URL, clientID, clientSecret string, al }, nil } +func serveHandler(ctx context.Context, logger slog.Logger, handler http.Handler, addr, name string) (closeFunc func()) { + logger.Debug(ctx, "http server listening", slog.F("addr", addr), slog.F("name", name)) + + srv := &http.Server{Addr: addr, Handler: handler} + go func() { + err := srv.ListenAndServe() + if err != nil && !xerrors.Is(err, http.ErrServerClosed) { + logger.Error(ctx, "http server listen", slog.F("name", name), slog.Error(err)) + } + }() + + return func() { _ = srv.Close() } +} + type datadogLogger struct { logger slog.Logger } diff --git a/coderd/coderd.go b/coderd/coderd.go index 37585097a6269..e15f41ac6f19b 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -10,6 +10,7 @@ import ( "time" "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" "github.com/pion/webrtc/v3" "google.golang.org/api/idtoken" @@ -68,9 +69,18 @@ func New(options *Options) (http.Handler, func()) { }) r := chi.NewRouter() + r.Use( + func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(middleware.NewWrapResponseWriter(w, r.ProtoMajor), r) + }) + }, + httpmw.Prometheus, + chitrace.Middleware(), + ) + r.Route("/api/v2", func(r chi.Router) { r.Use( - chitrace.Middleware(), // Specific routes can specify smaller limits. httpmw.RateLimitPerMinute(options.APIRateLimit), debugLogRequest(api.Logger), diff --git a/coderd/httpmw/prometheus.go b/coderd/httpmw/prometheus.go new file mode 100644 index 0000000000000..e03966ff9788f --- /dev/null +++ b/coderd/httpmw/prometheus.go @@ -0,0 +1,110 @@ +package httpmw + +import ( + "net/http" + "strconv" + "time" + + "github.com/go-chi/chi/v5" + chimw "github.com/go-chi/chi/v5/middleware" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + requestsProcessed = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "coderd", + Subsystem: "api", + Name: "requests_processed_total", + Help: "The total number of processed API requests", + }, []string{"code", "method", "path"}) + requestsConcurrent = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "api", + Name: "concurrent_requests", + Help: "The number of concurrent API requests", + }) + websocketsConcurrent = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "api", + Name: "concurrent_websockets", + Help: "The total number of concurrent API websockets", + }) + websocketsDist = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "coderd", + Subsystem: "api", + Name: "websocket_durations_ms", + Help: "Websocket duration distribution of requests in milliseconds", + Buckets: []float64{ + durationToFloatMs(01 * time.Millisecond), + durationToFloatMs(01 * time.Second), + durationToFloatMs(01 * time.Minute), + durationToFloatMs(01 * time.Hour), + durationToFloatMs(15 * time.Hour), + durationToFloatMs(30 * time.Hour), + }, + }, []string{"path"}) + requestsDist = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "coderd", + Subsystem: "api", + Name: "request_latencies_ms", + Help: "Latency distribution of requests in milliseconds", + Buckets: []float64{1, 5, 10, 25, 50, 100, 500, 1000, 5000, 10000, 30000}, + }, []string{"method", "path"}) +) + +func durationToFloatMs(d time.Duration) float64 { + return float64(d.Milliseconds()) +} + +func Prometheus(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var ( + start = time.Now() + method = r.Method + rctx = chi.RouteContext(r.Context()) + ) + sw, ok := w.(chimw.WrapResponseWriter) + if !ok { + panic("dev error: http.ResponseWriter is not chimw.WrapResponseWriter") + } + + var ( + dist *prometheus.HistogramVec + distOpts []string + ) + // We want to count websockets separately. + if isWebsocketUpgrade(r) { + websocketsConcurrent.Inc() + defer websocketsConcurrent.Dec() + + dist = websocketsDist + } else { + requestsConcurrent.Inc() + defer requestsConcurrent.Dec() + + dist = requestsDist + distOpts = []string{method} + } + + next.ServeHTTP(w, r) + + path := rctx.RoutePattern() + distOpts = append(distOpts, path) + statusStr := strconv.Itoa(sw.Status()) + + requestsProcessed.WithLabelValues(statusStr, method, path).Inc() + dist.WithLabelValues(distOpts...).Observe(float64(time.Since(start)) / 1e6) + }) +} + +func isWebsocketUpgrade(r *http.Request) bool { + vs := r.Header.Values("Upgrade") + for _, v := range vs { + if v == "websocket" { + return true + } + } + return false +} diff --git a/go.mod b/go.mod index 0b4cf04a078c0..128a705b78d79 100644 --- a/go.mod +++ b/go.mod @@ -90,6 +90,7 @@ require ( github.com/pion/webrtc/v3 v3.1.34 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 github.com/pkg/sftp v1.13.4 + github.com/prometheus/client_golang v1.12.1 github.com/quasilyte/go-ruleguard/dsl v0.3.19 github.com/robfig/cron/v3 v3.0.1 github.com/spf13/cobra v1.4.0 @@ -135,6 +136,7 @@ require ( github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.1.2 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/charmbracelet/bubbles v0.10.3 // indirect @@ -184,6 +186,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect github.com/miekg/dns v1.1.45 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect @@ -213,6 +216,9 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pquerna/cachecontrol v0.1.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/sirupsen/logrus v1.8.1 // indirect diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index da6cdc6925354..ee4add928bb11 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -91,6 +91,7 @@ export interface CreateWorkspaceBuildRequest { // This is likely an enum in an external package ("github.com/coder/coder/coderd/database.WorkspaceTransition") readonly transition: string readonly dry_run: boolean + readonly state: string } // From codersdk/organizations.go:52:6 @@ -265,12 +266,12 @@ export interface UpdateUserProfileRequest { readonly username: string } -// From codersdk/workspaces.go:94:6 +// From codersdk/workspaces.go:95:6 export interface UpdateWorkspaceAutostartRequest { readonly schedule: string } -// From codersdk/workspaces.go:114:6 +// From codersdk/workspaces.go:115:6 export interface UpdateWorkspaceAutostopRequest { readonly schedule: string } @@ -366,7 +367,7 @@ export interface WorkspaceAgentResourceMetadata { readonly cpu_mhz: number } -// From codersdk/workspacebuilds.go:17:6 +// From codersdk/workspacebuilds.go:18:6 export interface WorkspaceBuild { readonly id: string readonly created_at: string