Skip to content

Commit 38bdae7

Browse files
mtojekdeansheatherghuntley
authored
docs: Prometheus metrics + generator (#5179)
* docs: Prometheus metrics * Fix * Typo * Typo * Typo * Fix: link * Update docs/admin/prometheus.md Co-authored-by: Dean Sheather <dean@deansheather.com> * Update docs/admin/prometheus.md Co-authored-by: Dean Sheather <dean@deansheather.com> * Update docs/admin/prometheus.md Co-authored-by: Dean Sheather <dean@deansheather.com> * Update docs/admin/prometheus.md Co-authored-by: Dean Sheather <dean@deansheather.com> * Update docs/admin/prometheus.md Co-authored-by: Dean Sheather <dean@deansheather.com> * Rephrase * notice * use ```shell * Generator * gosec * fix: lint * PR comments * not needed anymore Co-authored-by: Dean Sheather <dean@deansheather.com> Co-authored-by: Geoffrey Huntley <ghuntley@ghuntley.com>
1 parent be79ae7 commit 38bdae7

File tree

6 files changed

+825
-2
lines changed

6 files changed

+825
-2
lines changed

Makefile

+12-2
Original file line numberDiff line numberDiff line change
@@ -399,13 +399,14 @@ gen: \
399399
coderd/database/querier.go \
400400
provisionersdk/proto/provisioner.pb.go \
401401
provisionerd/proto/provisionerd.pb.go \
402-
site/src/api/typesGenerated.ts
402+
site/src/api/typesGenerated.ts \
403+
docs/admin/prometheus.md
403404
.PHONY: gen
404405

405406
# Mark all generated files as fresh so make thinks they're up-to-date. This is
406407
# used during releases so we don't run generation scripts.
407408
gen/mark-fresh:
408-
files="coderd/database/dump.sql coderd/database/querier.go provisionersdk/proto/provisioner.pb.go provisionerd/proto/provisionerd.pb.go site/src/api/typesGenerated.ts"
409+
files="coderd/database/dump.sql coderd/database/querier.go provisionersdk/proto/provisioner.pb.go provisionerd/proto/provisionerd.pb.go site/src/api/typesGenerated.ts docs/admin/prometheus.md"
409410
for file in $$files; do
410411
echo "$$file"
411412
if [ ! -f "$$file" ]; then
@@ -448,6 +449,15 @@ site/src/api/typesGenerated.ts: scripts/apitypings/main.go $(shell find codersdk
448449
cd site
449450
yarn run format:types
450451

452+
docs/admin/prometheus.md: scripts/metricsdocgen/main.go scripts/metricsdocgen/metrics
453+
go run scripts/metricsdocgen/main.go
454+
cd site
455+
ifdef CI
456+
yarn run format:check
457+
else
458+
yarn run format:write
459+
endif
460+
451461
update-golden-files: cli/testdata/.gen-golden
452462
.PHONY: update-golden-files
453463

docs/admin/prometheus.md

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# Prometheus
2+
3+
Coder exposes many metrics which can be consumed by a Prometheus server, and give insight into the current state of a live Coder deployment.
4+
5+
If you don't have an Prometheus server installed, you can follow the Prometheus [Getting started](https://prometheus.io/docs/prometheus/latest/getting_started/) guide.
6+
7+
## Enable Prometheus metrics
8+
9+
Coder server exports metrics via the HTTP endpoint, which can be enabled using either the environment variable `CODER_PROMETHEUS_ENABLE` or the flag `--prometheus-enable`.
10+
11+
The Prometheus endpoint address is `http://localhost:2112/` by default. You can use either the environment variable `CODER_PROMETHEUS_ADDRESS` or the flag ` --prometheus-address <network-interface>:<port>` to select a different listen address.
12+
13+
__Notice__: Prometheus endpoint is not supported by the official Coder Helm chart yet.
14+
15+
If `coder server --prometheus-enable` is started locally, you can preview the metrics endpoint in your browser or by using curl: <!-- markdown-link-check-disable -->http://localhost:2112/<!-- markdown-link-check-enable -->.
16+
17+
```shell
18+
$ curl http://localhost:2112/
19+
# HELP coderd_api_active_users_duration_hour The number of users that have been active within the last hour.
20+
# TYPE coderd_api_active_users_duration_hour gauge
21+
coderd_api_active_users_duration_hour 0
22+
...
23+
```
24+
25+
## Available metrics
26+
27+
<!-- Code generated by 'make docs/admin/prometheus.md'. DO NOT EDIT -->
28+
29+
| Name | Type | Description | Labels |
30+
| - | - | - | - |
31+
| `coderd_api_active_users_duration_hour` | gauge | The number of users that have been active within the last hour. | |
32+
| `coderd_api_concurrent_requests` | gauge | The number of concurrent API requests | |
33+
| `coderd_api_concurrent_websockets` | gauge | The total number of concurrent API websockets | |
34+
| `coderd_api_request_latencies_ms` | histogram | Latency distribution of requests in milliseconds | `method` `path` |
35+
| `coderd_api_requests_processed_total` | counter | The total number of processed API requests | `code` `method` `path` |
36+
| `coderd_api_websocket_durations_ms` | histogram | Websocket duration distribution of requests in milliseconds | `path` |
37+
| `coderd_api_workspace_latest_build_total` | gauge | The latest workspace builds with a status. | `status` |
38+
| `coderd_provisionerd_job_timings_ms` | histogram | | `provisioner` `status` |
39+
| `coderd_provisionerd_jobs_current` | gauge | | `provisioner` |
40+
| `go_gc_duration_seconds` | summary | A summary of the pause duration of garbage collection cycles. | |
41+
| `go_goroutines` | gauge | Number of goroutines that currently exist. | |
42+
| `go_info` | gauge | Information about the Go environment. | `version` |
43+
| `go_memstats_alloc_bytes` | gauge | Number of bytes allocated and still in use. | |
44+
| `go_memstats_alloc_bytes_total` | counter | Total number of bytes allocated, even if freed. | |
45+
| `go_memstats_buck_hash_sys_bytes` | gauge | Number of bytes used by the profiling bucket hash table. | |
46+
| `go_memstats_frees_total` | counter | Total number of frees. | |
47+
| `go_memstats_gc_sys_bytes` | gauge | Number of bytes used for garbage collection system metadata. | |
48+
| `go_memstats_heap_alloc_bytes` | gauge | Number of heap bytes allocated and still in use. | |
49+
| `go_memstats_heap_idle_bytes` | gauge | Number of heap bytes waiting to be used. | |
50+
| `go_memstats_heap_inuse_bytes` | gauge | Number of heap bytes that are in use. | |
51+
| `go_memstats_heap_objects` | gauge | Number of allocated objects. | |
52+
| `go_memstats_heap_released_bytes` | gauge | Number of heap bytes released to OS. | |
53+
| `go_memstats_heap_sys_bytes` | gauge | Number of heap bytes obtained from system. | |
54+
| `go_memstats_last_gc_time_seconds` | gauge | Number of seconds since 1970 of last garbage collection. | |
55+
| `go_memstats_lookups_total` | counter | Total number of pointer lookups. | |
56+
| `go_memstats_mallocs_total` | counter | Total number of mallocs. | |
57+
| `go_memstats_mcache_inuse_bytes` | gauge | Number of bytes in use by mcache structures. | |
58+
| `go_memstats_mcache_sys_bytes` | gauge | Number of bytes used for mcache structures obtained from system. | |
59+
| `go_memstats_mspan_inuse_bytes` | gauge | Number of bytes in use by mspan structures. | |
60+
| `go_memstats_mspan_sys_bytes` | gauge | Number of bytes used for mspan structures obtained from system. | |
61+
| `go_memstats_next_gc_bytes` | gauge | Number of heap bytes when next garbage collection will take place. | |
62+
| `go_memstats_other_sys_bytes` | gauge | Number of bytes used for other system allocations. | |
63+
| `go_memstats_stack_inuse_bytes` | gauge | Number of bytes in use by the stack allocator. | |
64+
| `go_memstats_stack_sys_bytes` | gauge | Number of bytes obtained from system for stack allocator. | |
65+
| `go_memstats_sys_bytes` | gauge | Number of bytes obtained from system. | |
66+
| `go_threads` | gauge | Number of OS threads created. | |
67+
| `promhttp_metric_handler_requests_in_flight` | gauge | Current number of scrapes being served. | |
68+
| `promhttp_metric_handler_requests_total` | counter | Total number of scrapes by HTTP status code. | `code` |
69+
70+
<!-- End generated by 'make docs/admin/prometheus.md'. -->

docs/images/icons/speed.svg

+1
Loading

docs/manifest.json

+6
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,12 @@
274274
"path": "./admin/high-availability.md",
275275
"state": "enterprise"
276276
},
277+
{
278+
"title": "Prometheus",
279+
"description": "Learn how to collect Prometheus metrics",
280+
"icon_path": "./images/icons/speed.svg",
281+
"path": "./admin/prometheus.md"
282+
},
277283
{
278284
"title": "Telemetry",
279285
"description": "Learn what usage telemetry Coder collects",

scripts/metricsdocgen/main.go

+160
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
package main
2+
3+
import (
4+
"bytes"
5+
"errors"
6+
"flag"
7+
"io"
8+
"log"
9+
"os"
10+
"sort"
11+
"strings"
12+
13+
dto "github.com/prometheus/client_model/go"
14+
"github.com/prometheus/common/expfmt"
15+
"golang.org/x/xerrors"
16+
)
17+
18+
var (
19+
metricsFile string
20+
prometheusDocFile string
21+
dryRun bool
22+
23+
generatorPrefix = []byte("<!-- Code generated by 'make docs/admin/prometheus.md'. DO NOT EDIT -->")
24+
generatorSuffix = []byte("<!-- End generated by 'make docs/admin/prometheus.md'. -->")
25+
)
26+
27+
func main() {
28+
flag.StringVar(&metricsFile, "metrics-file", "scripts/metricsdocgen/metrics", "Path to Prometheus metrics file")
29+
flag.StringVar(&prometheusDocFile, "prometheus-doc-file", "docs/admin/prometheus.md", "Path to prometheus doc file")
30+
flag.BoolVar(&dryRun, "dry-run", false, "Dry run")
31+
flag.Parse()
32+
33+
metrics, err := readMetrics()
34+
if err != nil {
35+
log.Fatal("can't read metrics: ", err)
36+
}
37+
38+
doc, err := readPrometheusDoc()
39+
if err != nil {
40+
log.Fatal("can't read Prometheus doc: ", err)
41+
}
42+
43+
doc, err = updatePrometheusDoc(doc, metrics)
44+
if err != nil {
45+
log.Fatal("can't update Prometheus doc: ", err)
46+
}
47+
48+
if dryRun {
49+
log.Println(string(doc))
50+
return
51+
}
52+
53+
err = writePrometheusDoc(doc)
54+
if err != nil {
55+
log.Fatal("can't write updated Prometheus doc: ", err)
56+
}
57+
}
58+
59+
func readMetrics() ([]dto.MetricFamily, error) {
60+
f, err := os.Open(metricsFile)
61+
if err != nil {
62+
return nil, xerrors.New("can't open metrics file")
63+
}
64+
65+
var metrics []dto.MetricFamily
66+
67+
decoder := expfmt.NewDecoder(f, expfmt.FmtProtoText)
68+
for {
69+
var m dto.MetricFamily
70+
err = decoder.Decode(&m)
71+
if errors.Is(err, io.EOF) {
72+
break
73+
} else if err != nil {
74+
return nil, err
75+
}
76+
metrics = append(metrics, m)
77+
}
78+
79+
sort.Slice(metrics, func(i, j int) bool {
80+
return sort.StringsAreSorted([]string{*metrics[i].Name, *metrics[j].Name})
81+
})
82+
return metrics, nil
83+
}
84+
85+
func readPrometheusDoc() ([]byte, error) {
86+
doc, err := os.ReadFile(prometheusDocFile)
87+
if err != nil {
88+
return nil, err
89+
}
90+
return doc, nil
91+
}
92+
93+
func updatePrometheusDoc(doc []byte, metricFamilies []dto.MetricFamily) ([]byte, error) {
94+
i := bytes.Index(doc, generatorPrefix)
95+
if i < 0 {
96+
return nil, xerrors.New("generator prefix tag not found")
97+
}
98+
tableStartIndex := i + len(generatorPrefix) + 1
99+
100+
j := bytes.Index(doc[tableStartIndex:], generatorSuffix)
101+
if j < 0 {
102+
return nil, xerrors.New("generator suffix tag not found")
103+
}
104+
tableEndIndex := tableStartIndex + j
105+
106+
var buffer bytes.Buffer
107+
buffer.Write(doc[:tableStartIndex])
108+
buffer.WriteByte('\n')
109+
110+
buffer.WriteString("| Name | Type | Description | Labels |\n")
111+
buffer.WriteString("| - | - | - | - |\n")
112+
for _, mf := range metricFamilies {
113+
buffer.WriteString("| ")
114+
buffer.Write([]byte("`" + *mf.Name + "`"))
115+
buffer.WriteString(" | ")
116+
buffer.Write([]byte(strings.ToLower(mf.Type.String())))
117+
buffer.WriteString(" | ")
118+
if mf.Help != nil {
119+
buffer.Write([]byte(*mf.Help))
120+
}
121+
buffer.WriteString(" | ")
122+
123+
labels := map[string]struct{}{}
124+
metrics := mf.GetMetric()
125+
for _, m := range metrics {
126+
for _, label := range m.Label {
127+
labels["`"+*label.Name+"`"] = struct{}{}
128+
}
129+
}
130+
131+
if len(labels) > 0 {
132+
buffer.WriteString(strings.Join(sortedKeys(labels), " "))
133+
}
134+
135+
buffer.WriteString(" |\n")
136+
}
137+
138+
buffer.WriteByte('\n')
139+
buffer.Write(doc[tableEndIndex:])
140+
return buffer.Bytes(), nil
141+
}
142+
143+
func writePrometheusDoc(doc []byte) error {
144+
// G306: Expect WriteFile permissions to be 0600 or less
145+
/* #nosec G306 */
146+
err := os.WriteFile(prometheusDocFile, doc, 0644)
147+
if err != nil {
148+
return err
149+
}
150+
return nil
151+
}
152+
153+
func sortedKeys(m map[string]struct{}) []string {
154+
var keys []string
155+
for k := range m {
156+
keys = append(keys, k)
157+
}
158+
sort.Strings(keys)
159+
return keys
160+
}

0 commit comments

Comments
 (0)