coder · coadler · May 3, 2022 · May 2, 2022 · May 2, 2022 · May 2, 2022
diff --git a/cli/server.go b/cli/server.go
@@ -12,6 +12,7 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"net/http/pprof"
 	"net/url"
 	"os"
 	"os/signal"
@@ -23,6 +24,7 @@ import (
 	"github.com/google/go-github/v43/github"
 	"github.com/pion/turn/v2"
 	"github.com/pion/webrtc/v3"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
 	"golang.org/x/oauth2"
 	xgithub "golang.org/x/oauth2/github"
@@ -55,6 +57,10 @@ func server() *cobra.Command {
 	var (
 		accessURL       string
 		address         string
+		promEnabled     bool
+		promAddress     string
+		pprofEnabled    bool
+		pprofAddress    string
 		cacheDir        string
 		dev             bool
 		devUserEmail    string
@@ -245,6 +251,17 @@ func server() *cobra.Command {
 				}
 			}
 
+			// This prevents the pprof import from being accidentally deleted.
+			var _ = pprof.Handler
+			if pprofEnabled {
+				//nolint:revive
+				defer serveHandler(cmd.Context(), logger, nil, pprofAddress, "pprof")()
+			}
+			if promEnabled {
+				//nolint:revive
+				defer serveHandler(cmd.Context(), logger, promhttp.Handler(), promAddress, "prometheus")()
+			}
+
 			errCh := make(chan error, 1)
 			provisionerDaemons := make([]*provisionerd.Server, 0)
 			for i := 0; uint8(i) < provisionerDaemonCount; i++ {
@@ -400,8 +417,12 @@ func server() *cobra.Command {
 		},
 	}
 
-	cliflag.StringVarP(root.Flags(), &accessURL, "access-url", "", "CODER_ACCESS_URL", "", "Specifies the external URL to access Coder")
-	cliflag.StringVarP(root.Flags(), &address, "address", "a", "CODER_ADDRESS", "127.0.0.1:3000", "The address to serve the API and dashboard")
+	cliflag.StringVarP(root.Flags(), &accessURL, "access-url", "", "CODER_ACCESS_URL", "", "Specifies the external URL to access Coder.")
+	cliflag.StringVarP(root.Flags(), &address, "address", "a", "CODER_ADDRESS", "127.0.0.1:3000", "The address to serve the API and dashboard.")
+	cliflag.BoolVarP(root.Flags(), &promEnabled, "prometheus-enable", "", "CODER_PROMETHEUS_ENABLE", false, "Enable serving prometheus metrics on the addressdefined by --prometheus-address.")
+	cliflag.StringVarP(root.Flags(), &promAddress, "prometheus-address", "", "CODER_PROMETHEUS_ADDRESS", "127.0.0.1:2112", "The address to serve prometheus metrics.")
+	cliflag.BoolVarP(root.Flags(), &promEnabled, "pprof-enable", "", "CODER_PPROF_ENABLE", false, "Enable serving pprof metrics on the address defined by --pprof-address.")
+	cliflag.StringVarP(root.Flags(), &pprofAddress, "pprof-address", "", "CODER_PPROF_ADDRESS", "127.0.0.1:6060", "The address to serve pprof.")
 	// systemd uses the CACHE_DIRECTORY environment variable!
 	cliflag.StringVarP(root.Flags(), &cacheDir, "cache-dir", "", "CACHE_DIRECTORY", filepath.Join(os.TempDir(), "coder-cache"), "Specifies a directory to cache binaries for provision operations.")
 	cliflag.BoolVarP(root.Flags(), &dev, "dev", "", "CODER_DEV_MODE", false, "Serve Coder in dev mode for tinkering")
@@ -661,6 +682,20 @@ func configureGithubOAuth2(accessURL *url.URL, clientID, clientSecret string, al
 	}, nil
 }
 
+func serveHandler(ctx context.Context, logger slog.Logger, handler http.Handler, addr, name string) (closeFunc func()) {
+	logger.Debug(ctx, "http server listening", slog.F("addr", addr), slog.F("name", name))
+
+	srv := &http.Server{Addr: addr, Handler: handler}
+	go func() {
+		err := srv.ListenAndServe()
+		if err != nil && !xerrors.Is(err, http.ErrServerClosed) {
+			logger.Error(ctx, "http server listen", slog.F("name", name), slog.Error(err))
+		}
+	}()
+
+	return func() { _ = srv.Close() }
+}
+
 type datadogLogger struct {
 	logger slog.Logger
 }

diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/go-chi/chi/v5"
+	"github.com/go-chi/chi/v5/middleware"
 	"github.com/pion/webrtc/v3"
 	"google.golang.org/api/idtoken"
 
@@ -68,9 +69,18 @@ func New(options *Options) (http.Handler, func()) {
 	})
 
 	r := chi.NewRouter()
+	r.Use(
+		func(next http.Handler) http.Handler {
+			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				next.ServeHTTP(middleware.NewWrapResponseWriter(w, r.ProtoMajor), r)
+			})
+		},
+		httpmw.Prometheus,
+		chitrace.Middleware(),
+	)
+
 	r.Route("/api/v2", func(r chi.Router) {
 		r.Use(
-			chitrace.Middleware(),
 			// Specific routes can specify smaller limits.
 			httpmw.RateLimitPerMinute(options.APIRateLimit),
 			debugLogRequest(api.Logger),

diff --git a/coderd/httpmw/prometheus.go b/coderd/httpmw/prometheus.go
@@ -0,0 +1,110 @@
+package httpmw
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	chimw "github.com/go-chi/chi/v5/middleware"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	requestsProcessed = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "coderd",
+		Subsystem: "api",
+		Name:      "requests_processed_total",
+		Help:      "The total number of processed API requests",
+	}, []string{"code", "method", "path"})
+	requestsConcurrent = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Subsystem: "api",
+		Name:      "concurrent_requests",
+		Help:      "The number of concurrent API requests",
+	})
+	websocketsConcurrent = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Subsystem: "api",
+		Name:      "concurrent_websockets",
+		Help:      "The total number of concurrent API websockets",
+	})
+	websocketsDist = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "coderd",
+		Subsystem: "api",
+		Name:      "websocket_durations_ms",
+		Help:      "Websocket duration distribution of requests in milliseconds",
+		Buckets: []float64{
+			durationToFloatMs(01 * time.Millisecond),
+			durationToFloatMs(01 * time.Second),
+			durationToFloatMs(01 * time.Minute),
+			durationToFloatMs(01 * time.Hour),
+			durationToFloatMs(15 * time.Hour),
+			durationToFloatMs(30 * time.Hour),
+		},
+	}, []string{"path"})
+	requestsDist = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "coderd",
+		Subsystem: "api",
+		Name:      "request_latencies_ms",
+		Help:      "Latency distribution of requests in milliseconds",
+		Buckets:   []float64{1, 5, 10, 25, 50, 100, 500, 1000, 5000, 10000, 30000},
+	}, []string{"method", "path"})
+)
+
+func durationToFloatMs(d time.Duration) float64 {
+	return float64(d.Milliseconds())
+}
+
+func Prometheus(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var (
+			start  = time.Now()
+			method = r.Method
+			rctx   = chi.RouteContext(r.Context())
+		)
+		sw, ok := w.(chimw.WrapResponseWriter)
+		if !ok {
+			panic("dev error: http.ResponseWriter is not chimw.WrapResponseWriter")
+		}
+
+		var (
+			dist     *prometheus.HistogramVec
+			distOpts []string
+		)
+		// We want to count websockets separately.
+		if isWebsocketUpgrade(r) {
+			websocketsConcurrent.Inc()
+			defer websocketsConcurrent.Dec()
+
+			dist = websocketsDist
+		} else {
+			requestsConcurrent.Inc()
+			defer requestsConcurrent.Dec()
+
+			dist = requestsDist
+			distOpts = []string{method}
+		}
+
+		next.ServeHTTP(w, r)
+
+		path := rctx.RoutePattern()
+		distOpts = append(distOpts, path)
+		statusStr := strconv.Itoa(sw.Status())
+
+		requestsProcessed.WithLabelValues(statusStr, method, path).Inc()
+		dist.WithLabelValues(distOpts...).Observe(float64(time.Since(start)) / 1e6)
+	})
+}
+
+func isWebsocketUpgrade(r *http.Request) bool {
+	vs := r.Header.Values("Upgrade")
+	for _, v := range vs {
+		if v == "websocket" {
+			return true
+		}
+	}
+	return false
+}
diff --git a/go.mod b/go.mod
@@ -90,6 +90,7 @@ require (
 	github.com/pion/webrtc/v3 v3.1.34
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
 	github.com/pkg/sftp v1.13.4
+	github.com/prometheus/client_golang v1.12.1
 	github.com/quasilyte/go-ruleguard/dsl v0.3.19
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/spf13/cobra v1.4.0
@@ -135,6 +136,7 @@ require (
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
 	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/charmbracelet/bubbles v0.10.3 // indirect
@@ -184,6 +186,7 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
 	github.com/miekg/dns v1.1.45 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
@@ -213,6 +216,9 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/pquerna/cachecontrol v0.1.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect

diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts
@@ -91,6 +91,7 @@ export interface CreateWorkspaceBuildRequest {
   // This is likely an enum in an external package ("github.com/coder/coder/coderd/database.WorkspaceTransition")
   readonly transition: string
   readonly dry_run: boolean
+  readonly state: string
 }
 
 // From codersdk/organizations.go:52:6
@@ -265,12 +266,12 @@ export interface UpdateUserProfileRequest {
   readonly username: string
 }
 
-// From codersdk/workspaces.go:94:6
+// From codersdk/workspaces.go:95:6
 export interface UpdateWorkspaceAutostartRequest {
   readonly schedule: string
 }
 
-// From codersdk/workspaces.go:114:6
+// From codersdk/workspaces.go:115:6
 export interface UpdateWorkspaceAutostopRequest {
   readonly schedule: string
 }
@@ -366,7 +367,7 @@ export interface WorkspaceAgentResourceMetadata {
   readonly cpu_mhz: number
 }
 
-// From codersdk/workspacebuilds.go:17:6
+// From codersdk/workspacebuilds.go:18:6
 export interface WorkspaceBuild {
   readonly id: string
   readonly created_at: string