fix: only collect prometheus database metrics when explicitly enabled

coder · johnstcn · Jun 15, 2023 · Jun 15, 2023 · Jun 15, 2023 · Jun 15, 2023
commit 65492d82b8dd1d1dcc684cf72f84b43199d6421d
diff --git a/cli/server.go b/cli/server.go
@@ -589,7 +589,10 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 
 			if cfg.InMemoryDatabase {
 				// This is only used for testing.
-				options.Database = dbmetrics.New(dbfake.New(), options.PrometheusRegistry)
+				options.Database = dbfake.New()
+				if options.DeploymentValues.Prometheus.Enable && options.DeploymentValues.Prometheus.CollectDBMetrics {
+					options.Database = dbmetrics.New(options.Database, options.PrometheusRegistry)
+				}
 				options.Pubsub = pubsub.NewInMemory()
 			} else {
 				sqlDB, err := connectToPostgres(ctx, logger, sqlDriver, cfg.PostgresURL.String())
@@ -600,7 +603,10 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 					_ = sqlDB.Close()
 				}()
 
-				options.Database = dbmetrics.New(database.New(sqlDB), options.PrometheusRegistry)
+				options.Database = database.New(sqlDB)
+				if options.DeploymentValues.Prometheus.Enable && options.DeploymentValues.Prometheus.CollectDBMetrics {
+					options.Database = dbmetrics.New(options.Database, options.PrometheusRegistry)
+				}
 				options.Pubsub, err = pubsub.New(ctx, sqlDB, cfg.PostgresURL.String())
 				if err != nil {
 					return xerrors.Errorf("create pubsub: %w", err)

diff --git a/cli/server_test.go b/cli/server_test.go
@@ -896,6 +896,7 @@ func TestServer(t *testing.T) {
 			"--provisioner-daemons", "1",
 			"--prometheus-enable",
 			"--prometheus-address", ":"+strconv.Itoa(randomPort),
+			// "--prometheus-collect-db-metrics", // Explicitly not enabled.
 			"--cache-dir", t.TempDir(),
 		)
 
@@ -929,6 +930,9 @@ func TestServer(t *testing.T) {
 				hasWorkspaces = true
 				continue
 			}
+			if strings.HasPrefix(scanner.Text(), "coderd_db_query_latencies_seconds") {
+				t.Fatal("db metrics should not be tracked when --prometheus-collect-db-metrics is not enabled")
+			}
 			t.Logf("scanned %s", scanner.Text())
 		}
 		require.NoError(t, scanner.Err())

diff --git a/cli/testdata/coder_server_--help.golden b/cli/testdata/coder_server_--help.golden
@@ -93,6 +93,9 @@ Use a YAML configuration file when your server launch become unwieldy.
       --prometheus-collect-agent-stats bool, $CODER_PROMETHEUS_COLLECT_AGENT_STATS
           Collect agent stats (may increase charges for metrics storage).
 
+      --prometheus-collect-db-metrics bool, $CODER_PROMETHEUS_COLLECT_DB_METRICS (default: false)
+          Collect database metrics (may increase charges for metrics storage).
+
       --prometheus-enable bool, $CODER_PROMETHEUS_ENABLE
           Serve prometheus metrics on the address defined by prometheus address.
 

diff --git a/cli/testdata/server-config.yaml.golden b/cli/testdata/server-config.yaml.golden
@@ -152,6 +152,9 @@ introspection:
     # Collect agent stats (may increase charges for metrics storage).
     # (default: <unset>, type: bool)
     collect_agent_stats: false
+    # Collect database metrics (may increase charges for metrics storage).
+    # (default: false, type: bool)
+    collect_db_metrics: false
   pprof:
     # Serve pprof metrics on the address defined by pprof address.
     # (default: <unset>, type: bool)

diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go
diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json
diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -192,7 +192,7 @@ func New(options *Options) *API {
 		options.Authorizer = rbac.NewCachingAuthorizer(options.PrometheusRegistry)
 	}
 	// The below are no-ops if already wrapped.
-	if options.PrometheusRegistry != nil {
+	if options.PrometheusRegistry != nil && options.DeploymentValues.Prometheus.CollectDBMetrics {
 		options.Database = dbmetrics.New(options.Database, options.PrometheusRegistry)
 	}
 	options.Database = dbauthz.New(

diff --git a/codersdk/deployment.go b/codersdk/deployment.go
@@ -229,6 +229,7 @@ type PrometheusConfig struct {
 	Enable            clibase.Bool     `json:"enable" typescript:",notnull"`
 	Address           clibase.HostPort `json:"address" typescript:",notnull"`
 	CollectAgentStats clibase.Bool     `json:"collect_agent_stats" typescript:",notnull"`
+	CollectDBMetrics  clibase.Bool     `json:"collect_db_metrics" typescript:",notnull"`
 }
 
 type PprofConfig struct {
@@ -760,6 +761,16 @@ when required by your organization's security policy.`,
 			Group:       &deploymentGroupIntrospectionPrometheus,
 			YAML:        "collect_agent_stats",
 		},
+		{
+			Name:        "Prometheus Collect Database Metrics",
+			Description: "Collect database metrics (may increase charges for metrics storage).",
+			Flag:        "prometheus-collect-db-metrics",
+			Env:         "CODER_PROMETHEUS_COLLECT_DB_METRICS",
+			Value:       &c.Prometheus.CollectDBMetrics,
+			Group:       &deploymentGroupIntrospectionPrometheus,
+			YAML:        "collect_db_metrics",
+			Default:     "false",
+		},
 		// Pprof settings
 		{
 			Name:        "pprof Enable",

diff --git a/docs/api/general.md b/docs/api/general.md
@@ -275,6 +275,7 @@ curl -X GET http://coder-server:8080/api/v2/deployment/config \
         "port": "string"
       },
       "collect_agent_stats": true,
+      "collect_db_metrics": true,
       "enable": true
     },
     "provisioner": {

diff --git a/docs/api/schemas.md b/docs/api/schemas.md
@@ -1943,6 +1943,7 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in
         "port": "string"
       },
       "collect_agent_stats": true,
+      "collect_db_metrics": true,
       "enable": true
     },
     "provisioner": {
@@ -2270,6 +2271,7 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in
       "port": "string"
     },
     "collect_agent_stats": true,
+    "collect_db_metrics": true,
     "enable": true
   },
   "provisioner": {
@@ -3092,6 +3094,7 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in
     "port": "string"
   },
   "collect_agent_stats": true,
+  "collect_db_metrics": true,
   "enable": true
 }
 ```
@@ -3102,6 +3105,7 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in
 | --------------------- | ------------------------------------ | -------- | ------------ | ----------- |
 | `address`             | [clibase.HostPort](#clibasehostport) | false    |              |             |
 | `collect_agent_stats` | boolean                              | false    |              |             |
+| `collect_db_metrics`  | boolean                              | false    |              |             |
 | `enable`              | boolean                              | false    |              |             |
 
 ## codersdk.ProvisionerConfig

diff --git a/docs/cli/server.md b/docs/cli/server.md
@@ -565,6 +565,17 @@ The bind address to serve prometheus metrics.
 
 Collect agent stats (may increase charges for metrics storage).
 
+### --prometheus-collect-db-metrics
+
+|             |                                                          |
+| ----------- | -------------------------------------------------------- |
+| Type        | <code>bool</code>                                        |
+| Environment | <code>$CODER_PROMETHEUS_COLLECT_DB_METRICS</code>        |
+| YAML        | <code>introspection.prometheus.collect_db_metrics</code> |
+| Default     | <code>false</code>                                       |
+
+Collect database metrics (may increase charges for metrics storage).
+
 ### --prometheus-enable
 
 |             |                                              |

diff --git a/enterprise/cli/testdata/coder_server_--help.golden b/enterprise/cli/testdata/coder_server_--help.golden
@@ -93,6 +93,9 @@ Use a YAML configuration file when your server launch become unwieldy.
       --prometheus-collect-agent-stats bool, $CODER_PROMETHEUS_COLLECT_AGENT_STATS
           Collect agent stats (may increase charges for metrics storage).
 
+      --prometheus-collect-db-metrics bool, $CODER_PROMETHEUS_COLLECT_DB_METRICS (default: false)
+          Collect database metrics (may increase charges for metrics storage).
+
       --prometheus-enable bool, $CODER_PROMETHEUS_ENABLE
           Serve prometheus metrics on the address defined by prometheus address.
 

diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
@@ -102,6 +102,8 @@ coder:
       value: "true"
     - name: "CODER_PROMETHEUS_COLLECT_AGENT_STATS"
       value: "true"
+    - name: "CODER_PROMETHEUS_COLLECT_DB_METRICS"
+      value: "true"
     - name: "CODER_VERBOSE"
       value: "true"
   image:

diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts
@@ -606,6 +606,7 @@ export interface PrometheusConfig {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any -- External type
   readonly address: any
   readonly collect_agent_stats: boolean
+  readonly collect_db_metrics: boolean
 }
 
 // From codersdk/deployment.go