diff --git a/cli/server.go b/cli/server.go index 26d0c8f110403..f9e744761b22e 100644 --- a/cli/server.go +++ b/cli/server.go @@ -55,6 +55,7 @@ import ( "cdr.dev/slog" "cdr.dev/slog/sloggers/sloghuman" + "github.com/coder/coder/v2/coderd/pproflabel" "github.com/coder/pretty" "github.com/coder/quartz" "github.com/coder/retry" @@ -1459,14 +1460,14 @@ func newProvisionerDaemon( tracer := coderAPI.TracerProvider.Tracer(tracing.TracerName) terraformClient, terraformServer := drpcsdk.MemTransportPipe() wg.Add(1) - go func() { + pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceTerraformProvisioner), func(ctx context.Context) { defer wg.Done() <-ctx.Done() _ = terraformClient.Close() _ = terraformServer.Close() - }() + }) wg.Add(1) - go func() { + pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceTerraformProvisioner), func(ctx context.Context) { defer wg.Done() defer cancel() @@ -1485,7 +1486,7 @@ func newProvisionerDaemon( default: } } - }() + }) connector[string(database.ProvisionerTypeTerraform)] = sdkproto.NewDRPCProvisionerClient(terraformClient) default: diff --git a/coderd/autobuild/lifecycle_executor.go b/coderd/autobuild/lifecycle_executor.go index 234a72de04c50..16072e6517125 100644 --- a/coderd/autobuild/lifecycle_executor.go +++ b/coderd/autobuild/lifecycle_executor.go @@ -20,6 +20,7 @@ import ( "cdr.dev/slog" "github.com/coder/coder/v2/coderd/files" + "github.com/coder/coder/v2/coderd/pproflabel" "github.com/coder/coder/v2/coderd/audit" "github.com/coder/coder/v2/coderd/database" @@ -107,10 +108,10 @@ func (e *Executor) WithStatsChannel(ch chan<- Stats) *Executor { // tick from its channel. It will stop when its context is Done, or when // its channel is closed. func (e *Executor) Run() { - go func() { + pproflabel.Go(e.ctx, pproflabel.Service(pproflabel.ServiceLifecycles), func(ctx context.Context) { for { select { - case <-e.ctx.Done(): + case <-ctx.Done(): return case t, ok := <-e.tick: if !ok { @@ -120,15 +121,15 @@ func (e *Executor) Run() { e.metrics.autobuildExecutionDuration.Observe(stats.Elapsed.Seconds()) if e.statsCh != nil { select { - case <-e.ctx.Done(): + case <-ctx.Done(): return case e.statsCh <- stats: } } - e.log.Debug(e.ctx, "run stats", slog.F("elapsed", stats.Elapsed), slog.F("transitions", stats.Transitions)) + e.log.Debug(ctx, "run stats", slog.F("elapsed", stats.Elapsed), slog.F("transitions", stats.Transitions)) } } - }() + }) } func (e *Executor) runOnce(t time.Time) Stats { diff --git a/coderd/coderd.go b/coderd/coderd.go index 26bf4a7bf9b63..9b0e35b21032b 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -852,6 +852,7 @@ func New(options *Options) *API { r.Use( httpmw.Recover(api.Logger), + httpmw.WithProfilingLabels, tracing.StatusWriterMiddleware, tracing.Middleware(api.TracerProvider), httpmw.AttachRequestID, diff --git a/coderd/httpmw/pprof.go b/coderd/httpmw/pprof.go new file mode 100644 index 0000000000000..eee3e9c9fdbe1 --- /dev/null +++ b/coderd/httpmw/pprof.go @@ -0,0 +1,30 @@ +package httpmw + +import ( + "context" + "net/http" + "runtime/pprof" + + "github.com/coder/coder/v2/coderd/pproflabel" +) + +// WithProfilingLabels adds a pprof label to all http request handlers. This is +// primarily used to determine if load is coming from background jobs, or from +// http traffic. +func WithProfilingLabels(next http.Handler) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Label to differentiate between http and websocket requests. Websocket requests + // are assumed to be long-lived and more resource consuming. + requestType := "http" + if r.Header.Get("Upgrade") == "websocket" { + requestType = "websocket" + } + + pprof.Do(ctx, pproflabel.Service(pproflabel.ServiceHTTPServer, "request_type", requestType), func(ctx context.Context) { + r = r.WithContext(ctx) + next.ServeHTTP(rw, r) + }) + }) +} diff --git a/coderd/pproflabel/pproflabel.go b/coderd/pproflabel/pproflabel.go new file mode 100644 index 0000000000000..cd803b0f1baea --- /dev/null +++ b/coderd/pproflabel/pproflabel.go @@ -0,0 +1,25 @@ +package pproflabel + +import ( + "context" + "runtime/pprof" +) + +// Go is just a convince wrapper to set off a labeled goroutine. +func Go(ctx context.Context, labels pprof.LabelSet, f func(context.Context)) { + go pprof.Do(ctx, labels, f) +} + +const ( + ServiceTag = "service" + + ServiceHTTPServer = "http-api" + ServiceLifecycles = "lifecycle-executor" + ServiceMetricCollector = "metrics-collector" + ServicePrebuildReconciler = "prebuilds-reconciler" + ServiceTerraformProvisioner = "terraform-provisioner" +) + +func Service(name string, pairs ...string) pprof.LabelSet { + return pprof.Labels(append([]string{ServiceTag, name}, pairs...)...) +} diff --git a/coderd/prometheusmetrics/insights/metricscollector.go b/coderd/prometheusmetrics/insights/metricscollector.go index 41d3a0220f391..a095968526ca8 100644 --- a/coderd/prometheusmetrics/insights/metricscollector.go +++ b/coderd/prometheusmetrics/insights/metricscollector.go @@ -14,6 +14,7 @@ import ( "cdr.dev/slog" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/pproflabel" "github.com/coder/coder/v2/coderd/util/slice" "github.com/coder/coder/v2/codersdk" ) @@ -158,7 +159,7 @@ func (mc *MetricsCollector) Run(ctx context.Context) (func(), error) { }) } - go func() { + pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceMetricCollector), func(ctx context.Context) { defer close(done) defer ticker.Stop() for { @@ -170,7 +171,7 @@ func (mc *MetricsCollector) Run(ctx context.Context) (func(), error) { doTick() } } - }() + }) return func() { closeFunc() <-done diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 9583e14cd7fd3..40569ead70658 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -12,20 +12,20 @@ import ( "sync" "time" - "github.com/coder/quartz" - "github.com/coder/coder/v2/buildinfo" "github.com/coder/coder/v2/coderd/appearance" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/entitlements" "github.com/coder/coder/v2/coderd/idpsync" agplportsharing "github.com/coder/coder/v2/coderd/portsharing" + "github.com/coder/coder/v2/coderd/pproflabel" agplprebuilds "github.com/coder/coder/v2/coderd/prebuilds" "github.com/coder/coder/v2/coderd/rbac/policy" "github.com/coder/coder/v2/coderd/wsbuilder" "github.com/coder/coder/v2/enterprise/coderd/connectionlog" "github.com/coder/coder/v2/enterprise/coderd/enidpsync" "github.com/coder/coder/v2/enterprise/coderd/portsharing" + "github.com/coder/quartz" "golang.org/x/xerrors" "tailscale.com/tailcfg" @@ -903,7 +903,9 @@ func (api *API) updateEntitlements(ctx context.Context) error { } api.AGPL.PrebuildsReconciler.Store(&reconciler) - go reconciler.Run(context.Background()) + // TODO: Should this context be the api.ctx context? To cancel when + // the API (and entire app) is closed via shutdown? + pproflabel.Go(context.Background(), pproflabel.Service(pproflabel.ServicePrebuildReconciler), reconciler.Run) api.AGPL.PrebuildsClaimer.Store(&claimer) } diff --git a/enterprise/wsproxy/wsproxy.go b/enterprise/wsproxy/wsproxy.go index 69241d8aa1c17..c2ac1baf2db4e 100644 --- a/enterprise/wsproxy/wsproxy.go +++ b/enterprise/wsproxy/wsproxy.go @@ -333,6 +333,7 @@ func New(ctx context.Context, opts *Options) (*Server, error) { r.Use( // TODO: @emyrk Should we standardize these in some other package? httpmw.Recover(s.Logger), + httpmw.WithProfilingLabels, tracing.StatusWriterMiddleware, tracing.Middleware(s.TracerProvider), httpmw.AttachRequestID,