Skip to content

Commit 55ad97b

Browse files
coadlercoadler
authored
feat: add pprof and prometheus metrics to coder server (#1266)
1 parent 5dcaf94 commit 55ad97b

File tree

5 files changed

+168
-6
lines changed

5 files changed

+168
-6
lines changed

‎cli/server.go

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"log"
1313
"net"
1414
"net/http"
15+
"net/http/pprof"
1516
"net/url"
1617
"os"
1718
"os/signal"
@@ -23,6 +24,7 @@ import (
2324
"github.com/google/go-github/v43/github"
2425
"github.com/pion/turn/v2"
2526
"github.com/pion/webrtc/v3"
27+
"github.com/prometheus/client_golang/prometheus/promhttp"
2628
"github.com/spf13/cobra"
2729
"golang.org/x/oauth2"
2830
xgithub "golang.org/x/oauth2/github"
@@ -55,6 +57,10 @@ func server() *cobra.Command {
5557
var (
5658
accessURL string
5759
address string
60+
promEnabled bool
61+
promAddress string
62+
pprofEnabled bool
63+
pprofAddress string
5864
cacheDir string
5965
dev bool
6066
devUserEmail string
@@ -245,6 +251,17 @@ func server() *cobra.Command {
245251
}
246252
}
247253

254+
// This prevents the pprof import from being accidentally deleted.
255+
var _ = pprof.Handler
256+
if pprofEnabled {
257+
//nolint:revive
258+
defer serveHandler(cmd.Context(), logger, nil, pprofAddress, "pprof")()
259+
}
260+
if promEnabled {
261+
//nolint:revive
262+
defer serveHandler(cmd.Context(), logger, promhttp.Handler(), promAddress, "prometheus")()
263+
}
264+
248265
errCh := make(chan error, 1)
249266
provisionerDaemons := make([]*provisionerd.Server, 0)
250267
for i := 0; uint8(i) < provisionerDaemonCount; i++ {
@@ -400,8 +417,12 @@ func server() *cobra.Command {
400417
},
401418
}
402419

403-
cliflag.StringVarP(root.Flags(), &accessURL, "access-url", "", "CODER_ACCESS_URL", "", "Specifies the external URL to access Coder")
404-
cliflag.StringVarP(root.Flags(), &address, "address", "a", "CODER_ADDRESS", "127.0.0.1:3000", "The address to serve the API and dashboard")
420+
cliflag.StringVarP(root.Flags(), &accessURL, "access-url", "", "CODER_ACCESS_URL", "", "Specifies the external URL to access Coder.")
421+
cliflag.StringVarP(root.Flags(), &address, "address", "a", "CODER_ADDRESS", "127.0.0.1:3000", "The address to serve the API and dashboard.")
422+
cliflag.BoolVarP(root.Flags(), &promEnabled, "prometheus-enable", "", "CODER_PROMETHEUS_ENABLE", false, "Enable serving prometheus metrics on the addressdefined by --prometheus-address.")
423+
cliflag.StringVarP(root.Flags(), &promAddress, "prometheus-address", "", "CODER_PROMETHEUS_ADDRESS", "127.0.0.1:2112", "The address to serve prometheus metrics.")
424+
cliflag.BoolVarP(root.Flags(), &promEnabled, "pprof-enable", "", "CODER_PPROF_ENABLE", false, "Enable serving pprof metrics on the address defined by --pprof-address.")
425+
cliflag.StringVarP(root.Flags(), &pprofAddress, "pprof-address", "", "CODER_PPROF_ADDRESS", "127.0.0.1:6060", "The address to serve pprof.")
405426
// systemd uses the CACHE_DIRECTORY environment variable!
406427
cliflag.StringVarP(root.Flags(), &cacheDir, "cache-dir", "", "CACHE_DIRECTORY", filepath.Join(os.TempDir(), "coder-cache"), "Specifies a directory to cache binaries for provision operations.")
407428
cliflag.BoolVarP(root.Flags(), &dev, "dev", "", "CODER_DEV_MODE", false, "Serve Coder in dev mode for tinkering")
@@ -661,6 +682,20 @@ func configureGithubOAuth2(accessURL *url.URL, clientID, clientSecret string, al
661682
}, nil
662683
}
663684

685+
func serveHandler(ctx context.Context, logger slog.Logger, handler http.Handler, addr, name string) (closeFunc func()) {
686+
logger.Debug(ctx, "http server listening", slog.F("addr", addr), slog.F("name", name))
687+
688+
srv := &http.Server{Addr: addr, Handler: handler}
689+
go func() {
690+
err := srv.ListenAndServe()
691+
if err != nil && !xerrors.Is(err, http.ErrServerClosed) {
692+
logger.Error(ctx, "http server listen", slog.F("name", name), slog.Error(err))
693+
}
694+
}()
695+
696+
return func() { _ = srv.Close() }
697+
}
698+
664699
type datadogLogger struct {
665700
logger slog.Logger
666701
}

‎coderd/coderd.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"time"
1111

1212
"github.com/go-chi/chi/v5"
13+
"github.com/go-chi/chi/v5/middleware"
1314
"github.com/pion/webrtc/v3"
1415
"google.golang.org/api/idtoken"
1516

@@ -68,9 +69,18 @@ func New(options *Options) (http.Handler, func()) {
6869
})
6970

7071
r := chi.NewRouter()
72+
r.Use(
73+
func(next http.Handler) http.Handler {
74+
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
75+
next.ServeHTTP(middleware.NewWrapResponseWriter(w, r.ProtoMajor), r)
76+
})
77+
},
78+
httpmw.Prometheus,
79+
chitrace.Middleware(),
80+
)
81+
7182
r.Route("/api/v2", func(r chi.Router) {
7283
r.Use(
73-
chitrace.Middleware(),
7484
// Specific routes can specify smaller limits.
7585
httpmw.RateLimitPerMinute(options.APIRateLimit),
7686
debugLogRequest(api.Logger),

‎coderd/httpmw/prometheus.go

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
package httpmw
2+
3+
import (
4+
"net/http"
5+
"strconv"
6+
"time"
7+
8+
"github.com/go-chi/chi/v5"
9+
chimw "github.com/go-chi/chi/v5/middleware"
10+
11+
"github.com/prometheus/client_golang/prometheus"
12+
"github.com/prometheus/client_golang/prometheus/promauto"
13+
)
14+
15+
var (
16+
requestsProcessed = promauto.NewCounterVec(prometheus.CounterOpts{
17+
Namespace: "coderd",
18+
Subsystem: "api",
19+
Name: "requests_processed_total",
20+
Help: "The total number of processed API requests",
21+
}, []string{"code", "method", "path"})
22+
requestsConcurrent = promauto.NewGauge(prometheus.GaugeOpts{
23+
Namespace: "coderd",
24+
Subsystem: "api",
25+
Name: "concurrent_requests",
26+
Help: "The number of concurrent API requests",
27+
})
28+
websocketsConcurrent = promauto.NewGauge(prometheus.GaugeOpts{
29+
Namespace: "coderd",
30+
Subsystem: "api",
31+
Name: "concurrent_websockets",
32+
Help: "The total number of concurrent API websockets",
33+
})
34+
websocketsDist = promauto.NewHistogramVec(prometheus.HistogramOpts{
35+
Namespace: "coderd",
36+
Subsystem: "api",
37+
Name: "websocket_durations_ms",
38+
Help: "Websocket duration distribution of requests in milliseconds",
39+
Buckets: []float64{
40+
durationToFloatMs(01 * time.Millisecond),
41+
durationToFloatMs(01 * time.Second),
42+
durationToFloatMs(01 * time.Minute),
43+
durationToFloatMs(01 * time.Hour),
44+
durationToFloatMs(15 * time.Hour),
45+
durationToFloatMs(30 * time.Hour),
46+
},
47+
}, []string{"path"})
48+
requestsDist = promauto.NewHistogramVec(prometheus.HistogramOpts{
49+
Namespace: "coderd",
50+
Subsystem: "api",
51+
Name: "request_latencies_ms",
52+
Help: "Latency distribution of requests in milliseconds",
53+
Buckets: []float64{1, 5, 10, 25, 50, 100, 500, 1000, 5000, 10000, 30000},
54+
}, []string{"method", "path"})
55+
)
56+
57+
func durationToFloatMs(d time.Duration) float64 {
58+
return float64(d.Milliseconds())
59+
}
60+
61+
func Prometheus(next http.Handler) http.Handler {
62+
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
63+
var (
64+
start = time.Now()
65+
method = r.Method
66+
rctx = chi.RouteContext(r.Context())
67+
)
68+
sw, ok := w.(chimw.WrapResponseWriter)
69+
if !ok {
70+
panic("dev error: http.ResponseWriter is not chimw.WrapResponseWriter")
71+
}
72+
73+
var (
74+
dist *prometheus.HistogramVec
75+
distOpts []string
76+
)
77+
// We want to count websockets separately.
78+
if isWebsocketUpgrade(r) {
79+
websocketsConcurrent.Inc()
80+
defer websocketsConcurrent.Dec()
81+
82+
dist = websocketsDist
83+
} else {
84+
requestsConcurrent.Inc()
85+
defer requestsConcurrent.Dec()
86+
87+
dist = requestsDist
88+
distOpts = []string{method}
89+
}
90+
91+
next.ServeHTTP(w, r)
92+
93+
path := rctx.RoutePattern()
94+
distOpts = append(distOpts, path)
95+
statusStr := strconv.Itoa(sw.Status())
96+
97+
requestsProcessed.WithLabelValues(statusStr, method, path).Inc()
98+
dist.WithLabelValues(distOpts...).Observe(float64(time.Since(start)) / 1e6)
99+
})
100+
}
101+
102+
func isWebsocketUpgrade(r *http.Request) bool {
103+
vs := r.Header.Values("Upgrade")
104+
for _, v := range vs {
105+
if v == "websocket" {
106+
return true
107+
}
108+
}
109+
return false
110+
}

‎go.mod

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ require (
9090
github.com/pion/webrtc/v3 v3.1.34
9191
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
9292
github.com/pkg/sftp v1.13.4
93+
github.com/prometheus/client_golang v1.12.1
9394
github.com/quasilyte/go-ruleguard/dsl v0.3.19
9495
github.com/robfig/cron/v3 v3.0.1
9596
github.com/spf13/cobra v1.4.0
@@ -135,6 +136,7 @@ require (
135136
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
136137
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
137138
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect
139+
github.com/beorn7/perks v1.0.1 // indirect
138140
github.com/cenkalti/backoff/v4 v4.1.2 // indirect
139141
github.com/cespare/xxhash/v2 v2.1.2 // indirect
140142
github.com/charmbracelet/bubbles v0.10.3 // indirect
@@ -184,6 +186,7 @@ require (
184186
github.com/mailru/easyjson v0.7.7 // indirect
185187
github.com/mattn/go-colorable v0.1.12 // indirect
186188
github.com/mattn/go-runewidth v0.0.13 // indirect
189+
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
187190
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
188191
github.com/miekg/dns v1.1.45 // indirect
189192
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
@@ -213,6 +216,9 @@ require (
213216
github.com/pkg/errors v0.9.1 // indirect
214217
github.com/pmezard/go-difflib v1.0.0 // indirect
215218
github.com/pquerna/cachecontrol v0.1.0 // indirect
219+
github.com/prometheus/client_model v0.2.0 // indirect
220+
github.com/prometheus/common v0.32.1 // indirect
221+
github.com/prometheus/procfs v0.7.3 // indirect
216222
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
217223
github.com/rivo/uniseg v0.2.0 // indirect
218224
github.com/sirupsen/logrus v1.8.1 // indirect

‎site/src/api/typesGenerated.ts

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ export interface CreateWorkspaceBuildRequest {
9191
// This is likely an enum in an external package ("github.com/coder/coder/coderd/database.WorkspaceTransition")
9292
readonly transition: string
9393
readonly dry_run: boolean
94+
readonly state: string
9495
}
9596

9697
// From codersdk/organizations.go:52:6
@@ -265,12 +266,12 @@ export interface UpdateUserProfileRequest {
265266
readonly username: string
266267
}
267268

268-
// From codersdk/workspaces.go:94:6
269+
// From codersdk/workspaces.go:95:6
269270
export interface UpdateWorkspaceAutostartRequest {
270271
readonly schedule: string
271272
}
272273

273-
// From codersdk/workspaces.go:114:6
274+
// From codersdk/workspaces.go:115:6
274275
export interface UpdateWorkspaceAutostopRequest {
275276
readonly schedule: string
276277
}
@@ -366,7 +367,7 @@ export interface WorkspaceAgentResourceMetadata {
366367
readonly cpu_mhz: number
367368
}
368369

369-
// From codersdk/workspacebuilds.go:17:6
370+
// From codersdk/workspacebuilds.go:18:6
370371
export interface WorkspaceBuild {
371372
readonly id: string
372373
readonly created_at: string

0 commit comments

Comments
 (0)