feat: Add metric for provisioner daemons

goodspark · goodspark · commit 5201e202880e · 2023-06-06T14:52:01.000Z
So we can alert on provisioner saturation/capacity.
diff --git a/cli/server.go b/cli/server.go
@@ -822,6 +822,7 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 				}
 				provisionerDaemons = append(provisionerDaemons, daemon)
 			}
+			provisionerdMetrics.Runner.NumDaemons.Set(float64(len(provisionerDaemons)))
 
 			shutdownConnsCtx, shutdownConns := context.WithCancel(ctx)
 			defer shutdownConns()
diff --git a/provisionerd/provisionerd.go b/provisionerd/provisionerd.go
@@ -140,6 +140,12 @@ func NewMetrics(reg prometheus.Registerer) Metrics {
 				Name:      "jobs_current",
 				Help:      "The number of currently running provisioner jobs.",
 			}, []string{"provisioner"}),
+			NumDaemons: auto.NewGauge(prometheus.GaugeOpts{
+				Namespace: "coderd",
+				Subsystem: "provisionerd",
+				Name:      "num_daemons",
+				Help:      "The number of provisioner daemons.",
+			}),
 			JobTimings: auto.NewHistogramVec(prometheus.HistogramOpts{
 				Namespace: "coderd",
 				Subsystem: "provisionerd",
diff --git a/provisionerd/runner/runner.go b/provisionerd/runner/runner.go
@@ -88,6 +88,7 @@ type Runner struct {
 
 type Metrics struct {
 	ConcurrentJobs *prometheus.GaugeVec
+	NumDaemons     prometheus.Gauge
 	// JobTimings also counts the total amount of jobs.
 	JobTimings *prometheus.HistogramVec
 	// WorkspaceBuilds counts workspace build successes and failures.

Original file line number	Diff line number	Diff line change
`@@ -822,6 +822,7 @@ func (r RootCmd) Server(newAPI func(context.Context, coderd.Options) (*coderd.`
`822`	`822`	`}`
`823`	`823`	`provisionerDaemons = append(provisionerDaemons, daemon)`
`824`	`824`	`}`
	`825`	`+ provisionerdMetrics.Runner.NumDaemons.Set(float64(len(provisionerDaemons)))`
`825`	`826`
`826`	`827`	`shutdownConnsCtx, shutdownConns := context.WithCancel(ctx)`
`827`	`828`	`defer shutdownConns()`