feat(coderd): export metric indicating each experiment's status (#12657)

dannykopping · web-flow · commit 9cfd5baa9115 · 2024-03-19T14:11:27.000+02:00
diff --git a/cli/server.go b/cli/server.go
@@ -258,6 +258,7 @@ func enablePrometheus(
 	), nil
 }
 
+//nolint:gocognit // TODO(dannyk): reduce complexity of this function
 func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.API, io.Closer, error)) *serpent.Command {
 	if newAPI == nil {
 		newAPI = func(_ context.Context, o *coderd.Options) (*coderd.API, io.Closer, error) {
@@ -893,6 +894,15 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 					return xerrors.Errorf("register agents prometheus metric: %w", err)
 				}
 				defer closeAgentsFunc()
+
+				var active codersdk.Experiments
+				for _, exp := range options.DeploymentValues.Experiments.Value() {
+					active = append(active, codersdk.Experiment(exp))
+				}
+
+				if err = prometheusmetrics.Experiments(options.PrometheusRegistry, active); err != nil {
+					return xerrors.Errorf("register experiments metric: %w", err)
+				}
 			}
 
 			client := codersdk.New(localURL)
diff --git a/coderd/prometheusmetrics/prometheusmetrics.go b/coderd/prometheusmetrics/prometheusmetrics.go
@@ -516,6 +516,32 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R
 	}, nil
 }
 
+// Experiments registers a metric which indicates whether each experiment is enabled or not.
+func Experiments(registerer prometheus.Registerer, active codersdk.Experiments) error {
+	experimentsGauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Name:      "experiments",
+		Help:      "Indicates whether each experiment is enabled (1) or not (0)",
+	}, []string{"experiment"})
+	if err := registerer.Register(experimentsGauge); err != nil {
+		return err
+	}
+
+	for _, exp := range codersdk.ExperimentsAll {
+		var val float64
+		for _, enabled := range active {
+			if exp == enabled {
+				val = 1
+				break
+			}
+		}
+
+		experimentsGauge.WithLabelValues(string(exp)).Set(val)
+	}
+
+	return nil
+}
+
 // filterAcceptableAgentLabels handles a slightly messy situation whereby `prometheus-aggregate-agent-stats-by` can control on
 // which labels agent stats are aggregated, but for these specific metrics in this file there is no `template` label value,
 // and therefore we have to exclude it from the list of acceptable labels.
diff --git a/coderd/prometheusmetrics/prometheusmetrics_test.go b/coderd/prometheusmetrics/prometheusmetrics_test.go
@@ -500,6 +500,88 @@ func TestAgentStats(t *testing.T) {
 	assert.EqualValues(t, golden, collected)
 }
 
+func TestExperimentsMetric(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name        string
+		experiments codersdk.Experiments
+		expected    map[codersdk.Experiment]float64
+	}{
+		{
+			name:        "Enabled experiment is exported in metrics",
+			experiments: codersdk.Experiments{codersdk.ExperimentSharedPorts},
+			expected: map[codersdk.Experiment]float64{
+				codersdk.ExperimentSharedPorts: 1,
+			},
+		},
+		{
+			name:        "Disabled experiment is exported in metrics",
+			experiments: codersdk.Experiments{},
+			expected: map[codersdk.Experiment]float64{
+				codersdk.ExperimentSharedPorts: 0,
+			},
+		},
+		{
+			name:        "Unknown experiment is not exported in metrics",
+			experiments: codersdk.Experiments{codersdk.Experiment("bob")},
+			expected:    map[codersdk.Experiment]float64{},
+		},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			reg := prometheus.NewRegistry()
+
+			require.NoError(t, prometheusmetrics.Experiments(reg, tc.experiments))
+
+			out, err := reg.Gather()
+			require.NoError(t, err)
+			require.Lenf(t, out, 1, "unexpected number of registered metrics")
+
+			seen := make(map[codersdk.Experiment]float64)
+
+			for _, metric := range out[0].GetMetric() {
+				require.Equal(t, "coderd_experiments", out[0].GetName())
+
+				labels := metric.GetLabel()
+				require.Lenf(t, labels, 1, "unexpected number of labels")
+
+				experiment := codersdk.Experiment(labels[0].GetValue())
+				value := metric.GetGauge().GetValue()
+
+				seen[experiment] = value
+
+				expectedValue := 0
+
+				// Find experiment we expect to be enabled.
+				for _, exp := range tc.experiments {
+					if experiment == exp {
+						expectedValue = 1
+						break
+					}
+				}
+
+				require.EqualValuesf(t, expectedValue, value, "expected %d value for experiment %q", expectedValue, experiment)
+			}
+
+			// We don't want to define the state of all experiments because codersdk.ExperimentAll will change at some
+			// point and break these tests; so we only validate the experiments we know about.
+			for exp, val := range seen {
+				expectedVal, found := tc.expected[exp]
+				if !found {
+					t.Logf("ignoring experiment %q; it is not listed in expectations", exp)
+					continue
+				}
+				require.Equalf(t, expectedVal, val, "experiment %q did not match expected value %v", exp, expectedVal)
+			}
+		})
+	}
+}
+
 func prepareWorkspaceAndAgent(t *testing.T, client *codersdk.Client, user codersdk.CreateFirstUserResponse, workspaceNum int) *agentsdk.Client {
 	authToken := uuid.NewString()
 
