Skip to content

Commit 6e59823

Browse files
johnstcnjohnstcn
authored
fix: only collect prometheus database metrics when explicitly enabled (coder#8045)
* fix: only collect prometheus database metrics when explicitly enabled * add missing test * de-duplicate wrapping
1 parent 3f81f38 commit 6e59823

File tree

14 files changed

+153
-53
lines changed

14 files changed

+153
-53
lines changed

‎cli/server.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -589,7 +589,7 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
589589

590590
if cfg.InMemoryDatabase {
591591
// This is only used for testing.
592-
options.Database = dbmetrics.New(dbfake.New(), options.PrometheusRegistry)
592+
options.Database = dbfake.New()
593593
options.Pubsub = pubsub.NewInMemory()
594594
} else {
595595
sqlDB, err := connectToPostgres(ctx, logger, sqlDriver, cfg.PostgresURL.String())
@@ -600,14 +600,18 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
600600
_ = sqlDB.Close()
601601
}()
602602

603-
options.Database = dbmetrics.New(database.New(sqlDB), options.PrometheusRegistry)
603+
options.Database = database.New(sqlDB)
604604
options.Pubsub, err = pubsub.New(ctx, sqlDB, cfg.PostgresURL.String())
605605
if err != nil {
606606
return xerrors.Errorf("create pubsub: %w", err)
607607
}
608608
defer options.Pubsub.Close()
609609
}
610610

611+
if options.DeploymentValues.Prometheus.Enable && options.DeploymentValues.Prometheus.CollectDBMetrics {
612+
options.Database = dbmetrics.New(options.Database, options.PrometheusRegistry)
613+
}
614+
611615
var deploymentID string
612616
err = options.Database.InTx(func(tx database.Store) error {
613617
// This will block until the lock is acquired, and will be

‎cli/server_test.go

Lines changed: 102 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -881,59 +881,115 @@ func TestServer(t *testing.T) {
881881
})
882882
t.Run("Prometheus", func(t *testing.T) {
883883
t.Parallel()
884-
random, err := net.Listen("tcp", "127.0.0.1:0")
885-
require.NoError(t, err)
886-
_ = random.Close()
887-
tcpAddr, valid := random.Addr().(*net.TCPAddr)
888-
require.True(t, valid)
889-
randomPort := tcpAddr.Port
890884

891-
inv, cfg := clitest.New(t,
892-
"server",
893-
"--in-memory",
894-
"--http-address", ":0",
895-
"--access-url", "http://example.com",
896-
"--provisioner-daemons", "1",
897-
"--prometheus-enable",
898-
"--prometheus-address", ":"+strconv.Itoa(randomPort),
899-
"--cache-dir", t.TempDir(),
900-
)
885+
randomPort := func(t *testing.T) int {
886+
random, err := net.Listen("tcp", "127.0.0.1:0")
887+
require.NoError(t, err)
888+
_ = random.Close()
889+
tcpAddr, valid := random.Addr().(*net.TCPAddr)
890+
require.True(t, valid)
891+
return tcpAddr.Port
892+
}
901893

902-
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
903-
defer cancel()
894+
t.Run("DBMetricsDisabled", func(t *testing.T) {
895+
t.Parallel()
904896

905-
clitest.Start(t, inv)
906-
_ = waitAccessURL(t, cfg)
897+
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
898+
defer cancel()
907899

908-
var res *http.Response
909-
require.Eventually(t, func() bool {
910-
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://127.0.0.1:%d", randomPort), nil)
911-
assert.NoError(t, err)
912-
// nolint:bodyclose
913-
res, err = http.DefaultClient.Do(req)
914-
return err == nil
915-
}, testutil.WaitShort, testutil.IntervalFast)
916-
defer res.Body.Close()
900+
randPort := randomPort(t)
901+
inv, cfg := clitest.New(t,
902+
"server",
903+
"--in-memory",
904+
"--http-address", ":0",
905+
"--access-url", "http://example.com",
906+
"--provisioner-daemons", "1",
907+
"--prometheus-enable",
908+
"--prometheus-address", ":"+strconv.Itoa(randPort),
909+
// "--prometheus-collect-db-metrics", // disabled by default
910+
"--cache-dir", t.TempDir(),
911+
)
917912

918-
scanner := bufio.NewScanner(res.Body)
919-
hasActiveUsers := false
920-
hasWorkspaces := false
921-
for scanner.Scan() {
922-
// This metric is manually registered to be tracked in the server. That's
923-
// why we test it's tracked here.
924-
if strings.HasPrefix(scanner.Text(), "coderd_api_active_users_duration_hour") {
925-
hasActiveUsers = true
926-
continue
913+
clitest.Start(t, inv)
914+
_ = waitAccessURL(t, cfg)
915+
916+
var res *http.Response
917+
require.Eventually(t, func() bool {
918+
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://127.0.0.1:%d", randPort), nil)
919+
assert.NoError(t, err)
920+
// nolint:bodyclose
921+
res, err = http.DefaultClient.Do(req)
922+
return err == nil
923+
}, testutil.WaitShort, testutil.IntervalFast)
924+
defer res.Body.Close()
925+
926+
scanner := bufio.NewScanner(res.Body)
927+
hasActiveUsers := false
928+
hasWorkspaces := false
929+
for scanner.Scan() {
930+
// This metric is manually registered to be tracked in the server. That's
931+
// why we test it's tracked here.
932+
if strings.HasPrefix(scanner.Text(), "coderd_api_active_users_duration_hour") {
933+
hasActiveUsers = true
934+
continue
935+
}
936+
if strings.HasPrefix(scanner.Text(), "coderd_api_workspace_latest_build_total") {
937+
hasWorkspaces = true
938+
continue
939+
}
940+
if strings.HasPrefix(scanner.Text(), "coderd_db_query_latencies_seconds") {
941+
t.Fatal("db metrics should not be tracked when --prometheus-collect-db-metrics is not enabled")
942+
}
943+
t.Logf("scanned %s", scanner.Text())
927944
}
928-
if strings.HasPrefix(scanner.Text(), "coderd_api_workspace_latest_build_total") {
929-
hasWorkspaces = true
930-
continue
945+
require.NoError(t, scanner.Err())
946+
require.True(t, hasActiveUsers)
947+
require.True(t, hasWorkspaces)
948+
})
949+
950+
t.Run("DBMetricsEnabled", func(t *testing.T) {
951+
t.Parallel()
952+
953+
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
954+
defer cancel()
955+
956+
randPort := randomPort(t)
957+
inv, cfg := clitest.New(t,
958+
"server",
959+
"--in-memory",
960+
"--http-address", ":0",
961+
"--access-url", "http://example.com",
962+
"--provisioner-daemons", "1",
963+
"--prometheus-enable",
964+
"--prometheus-address", ":"+strconv.Itoa(randPort),
965+
"--prometheus-collect-db-metrics",
966+
"--cache-dir", t.TempDir(),
967+
)
968+
969+
clitest.Start(t, inv)
970+
_ = waitAccessURL(t, cfg)
971+
972+
var res *http.Response
973+
require.Eventually(t, func() bool {
974+
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://127.0.0.1:%d", randPort), nil)
975+
assert.NoError(t, err)
976+
// nolint:bodyclose
977+
res, err = http.DefaultClient.Do(req)
978+
return err == nil
979+
}, testutil.WaitShort, testutil.IntervalFast)
980+
defer res.Body.Close()
981+
982+
scanner := bufio.NewScanner(res.Body)
983+
hasDBMetrics := false
984+
for scanner.Scan() {
985+
if strings.HasPrefix(scanner.Text(), "coderd_db_query_latencies_seconds") {
986+
hasDBMetrics = true
987+
}
988+
t.Logf("scanned %s", scanner.Text())
931989
}
932-
t.Logf("scanned %s", scanner.Text())
933-
}
934-
require.NoError(t, scanner.Err())
935-
require.True(t, hasActiveUsers)
936-
require.True(t, hasWorkspaces)
990+
require.NoError(t, scanner.Err())
991+
require.True(t, hasDBMetrics)
992+
})
937993
})
938994
t.Run("GitHubOAuth", func(t *testing.T) {
939995
t.Parallel()

‎cli/testdata/coder_server_--help.golden

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,9 @@ Use a YAML configuration file when your server launch become unwieldy.
9393
--prometheus-collect-agent-stats bool, $CODER_PROMETHEUS_COLLECT_AGENT_STATS
9494
Collect agent stats (may increase charges for metrics storage).
9595

96+
--prometheus-collect-db-metrics bool, $CODER_PROMETHEUS_COLLECT_DB_METRICS (default: false)
97+
Collect database metrics (may increase charges for metrics storage).
98+
9699
--prometheus-enable bool, $CODER_PROMETHEUS_ENABLE
97100
Serve prometheus metrics on the address defined by prometheus address.
98101

‎cli/testdata/server-config.yaml.golden

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,9 @@ introspection:
152152
# Collect agent stats (may increase charges for metrics storage).
153153
# (default: <unset>, type: bool)
154154
collect_agent_stats: false
155+
# Collect database metrics (may increase charges for metrics storage).
156+
# (default: false, type: bool)
157+
collect_db_metrics: false
155158
pprof:
156159
# Serve pprof metrics on the address defined by pprof address.
157160
# (default: <unset>, type: bool)

‎coderd/apidoc/docs.go

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/apidoc/swagger.json

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/coderd.go

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@ import (
4747
"github.com/coder/coder/coderd/awsidentity"
4848
"github.com/coder/coder/coderd/database"
4949
"github.com/coder/coder/coderd/database/dbauthz"
50-
"github.com/coder/coder/coderd/database/dbmetrics"
5150
"github.com/coder/coder/coderd/database/pubsub"
5251
"github.com/coder/coder/coderd/gitauth"
5352
"github.com/coder/coder/coderd/gitsshkey"
@@ -191,10 +190,6 @@ func New(options *Options) *API {
191190
if options.Authorizer == nil {
192191
options.Authorizer = rbac.NewCachingAuthorizer(options.PrometheusRegistry)
193192
}
194-
// The below are no-ops if already wrapped.
195-
if options.PrometheusRegistry != nil {
196-
options.Database = dbmetrics.New(options.Database, options.PrometheusRegistry)
197-
}
198193
options.Database = dbauthz.New(
199194
options.Database,
200195
options.Authorizer,

‎codersdk/deployment.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,7 @@ type PrometheusConfig struct {
229229
Enable clibase.Bool `json:"enable" typescript:",notnull"`
230230
Address clibase.HostPort `json:"address" typescript:",notnull"`
231231
CollectAgentStats clibase.Bool `json:"collect_agent_stats" typescript:",notnull"`
232+
CollectDBMetrics clibase.Bool `json:"collect_db_metrics" typescript:",notnull"`
232233
}
233234

234235
type PprofConfig struct {
@@ -760,6 +761,16 @@ when required by your organization's security policy.`,
760761
Group: &deploymentGroupIntrospectionPrometheus,
761762
YAML: "collect_agent_stats",
762763
},
764+
{
765+
Name: "Prometheus Collect Database Metrics",
766+
Description: "Collect database metrics (may increase charges for metrics storage).",
767+
Flag: "prometheus-collect-db-metrics",
768+
Env: "CODER_PROMETHEUS_COLLECT_DB_METRICS",
769+
Value: &c.Prometheus.CollectDBMetrics,
770+
Group: &deploymentGroupIntrospectionPrometheus,
771+
YAML: "collect_db_metrics",
772+
Default: "false",
773+
},
763774
// Pprof settings
764775
{
765776
Name: "pprof Enable",

‎docs/api/general.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,7 @@ curl -X GET http://coder-server:8080/api/v2/deployment/config \
275275
"port": "string"
276276
},
277277
"collect_agent_stats": true,
278+
"collect_db_metrics": true,
278279
"enable": true
279280
},
280281
"provisioner": {

‎docs/api/schemas.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1943,6 +1943,7 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in
19431943
"port": "string"
19441944
},
19451945
"collect_agent_stats": true,
1946+
"collect_db_metrics": true,
19461947
"enable": true
19471948
},
19481949
"provisioner": {
@@ -2270,6 +2271,7 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in
22702271
"port": "string"
22712272
},
22722273
"collect_agent_stats": true,
2274+
"collect_db_metrics": true,
22732275
"enable": true
22742276
},
22752277
"provisioner": {
@@ -3092,6 +3094,7 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in
30923094
"port": "string"
30933095
},
30943096
"collect_agent_stats": true,
3097+
"collect_db_metrics": true,
30953098
"enable": true
30963099
}
30973100
```
@@ -3102,6 +3105,7 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in
31023105
| --------------------- | ------------------------------------ | -------- | ------------ | ----------- |
31033106
| `address` | [clibase.HostPort](#clibasehostport) | false | | |
31043107
| `collect_agent_stats` | boolean | false | | |
3108+
| `collect_db_metrics` | boolean | false | | |
31053109
| `enable` | boolean | false | | |
31063110

31073111
## codersdk.ProvisionerConfig

‎docs/cli/server.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -565,6 +565,17 @@ The bind address to serve prometheus metrics.
565565

566566
Collect agent stats (may increase charges for metrics storage).
567567

568+
### --prometheus-collect-db-metrics
569+
570+
| | |
571+
| ----------- | -------------------------------------------------------- |
572+
| Type | <code>bool</code> |
573+
| Environment | <code>$CODER_PROMETHEUS_COLLECT_DB_METRICS</code> |
574+
| YAML | <code>introspection.prometheus.collect_db_metrics</code> |
575+
| Default | <code>false</code> |
576+
577+
Collect database metrics (may increase charges for metrics storage).
578+
568579
### --prometheus-enable
569580

570581
| | |

‎enterprise/cli/testdata/coder_server_--help.golden

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,9 @@ Use a YAML configuration file when your server launch become unwieldy.
9393
--prometheus-collect-agent-stats bool, $CODER_PROMETHEUS_COLLECT_AGENT_STATS
9494
Collect agent stats (may increase charges for metrics storage).
9595

96+
--prometheus-collect-db-metrics bool, $CODER_PROMETHEUS_COLLECT_DB_METRICS (default: false)
97+
Collect database metrics (may increase charges for metrics storage).
98+
9699
--prometheus-enable bool, $CODER_PROMETHEUS_ENABLE
97100
Serve prometheus metrics on the address defined by prometheus address.
98101

‎scaletest/terraform/coder.tf

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,8 @@ coder:
102102
value: "true"
103103
- name: "CODER_PROMETHEUS_COLLECT_AGENT_STATS"
104104
value: "true"
105+
- name: "CODER_PROMETHEUS_COLLECT_DB_METRICS"
106+
value: "true"
105107
- name: "CODER_VERBOSE"
106108
value: "true"
107109
image:

‎site/src/api/typesGenerated.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -606,6 +606,7 @@ export interface PrometheusConfig {
606606
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- External type
607607
readonly address: any
608608
readonly collect_agent_stats: boolean
609+
readonly collect_db_metrics: boolean
609610
}
610611

611612
// From codersdk/deployment.go

0 commit comments

Comments
 (0)