coder · mtojek · Nov 30, 2022 · Nov 28, 2022 · Nov 28, 2022 · Nov 28, 2022
diff --git a/Makefile b/Makefile
@@ -399,13 +399,14 @@ gen: \
 	coderd/database/querier.go \
 	provisionersdk/proto/provisioner.pb.go \
 	provisionerd/proto/provisionerd.pb.go \
-	site/src/api/typesGenerated.ts
+	site/src/api/typesGenerated.ts \
+	docs/admin/prometheus.md
 .PHONY: gen
 
 # Mark all generated files as fresh so make thinks they're up-to-date. This is
 # used during releases so we don't run generation scripts.
 gen/mark-fresh:
-	files="coderd/database/dump.sql coderd/database/querier.go provisionersdk/proto/provisioner.pb.go provisionerd/proto/provisionerd.pb.go site/src/api/typesGenerated.ts"
+	files="coderd/database/dump.sql coderd/database/querier.go provisionersdk/proto/provisioner.pb.go provisionerd/proto/provisionerd.pb.go site/src/api/typesGenerated.ts docs/admin/prometheus.md"
 	for file in $$files; do
 		echo "$$file"
 		if [ ! -f "$$file" ]; then
@@ -448,6 +449,10 @@ site/src/api/typesGenerated.ts: scripts/apitypings/main.go $(shell find codersdk
 	cd site
 	yarn run format:types
 
+docs/admin/prometheus.md: scripts/metricsdocgen/main.go
+	go run scripts/metricsdocgen/main.go
+.PHONY: docs/admin/prometheus.md # As the .md file can be edited manually and the generator works in-place, we need to use .PHONY.
+
 update-golden-files: cli/testdata/.gen-golden
 .PHONY: update-golden-files
 

diff --git a/docs/admin/prometheus.md b/docs/admin/prometheus.md
@@ -0,0 +1,70 @@
+# Prometheus
+
+Coder exposes many metrics which can be consumed by a Prometheus server, and give insight into the current state of a live Coder deployment.
+
+If you don't have an Prometheus server installed, you can follow the Prometheus [Getting started](https://prometheus.io/docs/prometheus/latest/getting_started/) guide.
+
+## Enable Prometheus metrics
+
+Coder server exports metrics via the HTTP endpoint, which can be enabled using either the environment variable `CODER_PROMETHEUS_ENABLE` or the flag `--prometheus-enable`.
+
+The Prometheus endpoint address is `http://localhost:2112/` by default. You can use either the environment variable `CODER_PROMETHEUS_ADDRESS` or the flag ` --prometheus-address <network-interface>:<port>` to select a different listen address.
+
+__Notice__: Prometheus endpoint is not supported by the official Coder Helm chart yet.
+
+If `coder server --prometheus-enable` is started locally, you can preview the metrics endpoint in your browser or by using curl: <!-- markdown-link-check-disable -->http://localhost:2112/<!-- markdown-link-check-enable -->.
+
+```shell
+$ curl http://localhost:2112/
+# HELP coderd_api_active_users_duration_hour The number of users that have been active within the last hour.
+# TYPE coderd_api_active_users_duration_hour gauge
+coderd_api_active_users_duration_hour 0
+...
+```
+
+## Available metrics
+
+<!-- Code generated by 'make docs/admin/prometheus.md'. DO NOT EDIT -->
+
+| Name | Type | Description | Labels |
+| - | - | - | - |
+| `coderd_api_active_users_duration_hour` | gauge | The number of users that have been active within the last hour. |  |
+| `coderd_api_concurrent_requests` | gauge | The number of concurrent API requests |  |
+| `coderd_api_concurrent_websockets` | gauge | The total number of concurrent API websockets |  |
+| `coderd_api_request_latencies_ms` | histogram | Latency distribution of requests in milliseconds | `method` `path` |
+| `coderd_api_requests_processed_total` | counter | The total number of processed API requests | `code` `method` `path` |
+| `coderd_api_websocket_durations_ms` | histogram | Websocket duration distribution of requests in milliseconds | `path` |
+| `coderd_api_workspace_latest_build_total` | gauge | The latest workspace builds with a status. | `status` |
+| `coderd_provisionerd_job_timings_ms` | histogram |  | `provisioner` `status` |
+| `coderd_provisionerd_jobs_current` | gauge |  | `provisioner` |
+| `go_gc_duration_seconds` | summary | A summary of the pause duration of garbage collection cycles. |  |
+| `go_goroutines` | gauge | Number of goroutines that currently exist. |  |
+| `go_info` | gauge | Information about the Go environment. | `version` |
+| `go_memstats_alloc_bytes` | gauge | Number of bytes allocated and still in use. |  |
+| `go_memstats_alloc_bytes_total` | counter | Total number of bytes allocated, even if freed. |  |
+| `go_memstats_buck_hash_sys_bytes` | gauge | Number of bytes used by the profiling bucket hash table. |  |
+| `go_memstats_frees_total` | counter | Total number of frees. |  |
+| `go_memstats_gc_sys_bytes` | gauge | Number of bytes used for garbage collection system metadata. |  |
+| `go_memstats_heap_alloc_bytes` | gauge | Number of heap bytes allocated and still in use. |  |
+| `go_memstats_heap_idle_bytes` | gauge | Number of heap bytes waiting to be used. |  |
+| `go_memstats_heap_inuse_bytes` | gauge | Number of heap bytes that are in use. |  |
+| `go_memstats_heap_objects` | gauge | Number of allocated objects. |  |
+| `go_memstats_heap_released_bytes` | gauge | Number of heap bytes released to OS. |  |
+| `go_memstats_heap_sys_bytes` | gauge | Number of heap bytes obtained from system. |  |
+| `go_memstats_last_gc_time_seconds` | gauge | Number of seconds since 1970 of last garbage collection. |  |
+| `go_memstats_lookups_total` | counter | Total number of pointer lookups. |  |
+| `go_memstats_mallocs_total` | counter | Total number of mallocs. |  |
+| `go_memstats_mcache_inuse_bytes` | gauge | Number of bytes in use by mcache structures. |  |
+| `go_memstats_mcache_sys_bytes` | gauge | Number of bytes used for mcache structures obtained from system. |  |
+| `go_memstats_mspan_inuse_bytes` | gauge | Number of bytes in use by mspan structures. |  |
+| `go_memstats_mspan_sys_bytes` | gauge | Number of bytes used for mspan structures obtained from system. |  |
+| `go_memstats_next_gc_bytes` | gauge | Number of heap bytes when next garbage collection will take place. |  |
+| `go_memstats_other_sys_bytes` | gauge | Number of bytes used for other system allocations. |  |
+| `go_memstats_stack_inuse_bytes` | gauge | Number of bytes in use by the stack allocator. |  |
+| `go_memstats_stack_sys_bytes` | gauge | Number of bytes obtained from system for stack allocator. |  |
+| `go_memstats_sys_bytes` | gauge | Number of bytes obtained from system. |  |
+| `go_threads` | gauge | Number of OS threads created. |  |
+| `promhttp_metric_handler_requests_in_flight` | gauge | Current number of scrapes being served. |  |
+| `promhttp_metric_handler_requests_total` | counter | Total number of scrapes by HTTP status code. | `code` |
+
+<!-- End generated by 'make docs/admin/prometheus.md'. -->
diff --git a/docs/images/icons/speed.svg b/docs/images/icons/speed.svg
diff --git a/docs/manifest.json b/docs/manifest.json
@@ -274,6 +274,12 @@
           "path": "./admin/high-availability.md",
           "state": "enterprise"
         },
+        {
+          "title": "Prometheus",
+          "description": "Learn how to collect Prometheus metrics",
+          "icon_path": "./images/icons/speed.svg",
+          "path": "./admin/prometheus.md"
+        },
         {
           "title": "Telemetry",
           "description": "Learn what usage telemetry Coder collects",

diff --git a/scripts/metricsdocgen/main.go b/scripts/metricsdocgen/main.go
@@ -0,0 +1,160 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"flag"
+	"io"
+	"log"
+	"os"
+	"sort"
+	"strings"
+
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
+	"golang.org/x/xerrors"
+)
+
+var (
+	metricsFile       string
+	prometheusDocFile string
+	dryRun            bool
+
+	generatorPrefix = []byte("<!-- Code generated by 'make docs/admin/prometheus.md'. DO NOT EDIT -->")
+	generatorSuffix = []byte("<!-- End generated by 'make docs/admin/prometheus.md'. -->")
+)
+
+func main() {
+	flag.StringVar(&metricsFile, "metrics-file", "scripts/metricsdocgen/metrics", "Path to Prometheus metrics file")
+	flag.StringVar(&prometheusDocFile, "prometheus-doc-file", "docs/admin/prometheus.md", "Path to prometheus doc file")
+	flag.BoolVar(&dryRun, "dry-run", false, "Dry run")
+	flag.Parse()
+
+	metrics, err := readMetrics()
+	if err != nil {
+		log.Fatal("can't read metrics: ", err)
+	}
+
+	doc, err := readPrometheusDoc()
+	if err != nil {
+		log.Fatal("can't read Prometheus doc: ", err)
+	}
+
+	doc, err = updatePrometheusDoc(doc, metrics)
+	if err != nil {
+		log.Fatal("can't update Prometheus doc: ", err)
+	}
+
+	if dryRun {
+		log.Println(string(doc))
+		return
+	}
+
+	err = writePrometheusDoc(doc)
+	if err != nil {
+		log.Fatal("can't write updated Prometheus doc: ", err)
+	}
+}
+
+func readMetrics() ([]dto.MetricFamily, error) {
+	f, err := os.Open(metricsFile)
+	if err != nil {
+		return nil, xerrors.New("can't open metrics file")
+	}
+
+	var metrics []dto.MetricFamily
+
+	decoder := expfmt.NewDecoder(f, expfmt.FmtProtoText)
+	for {
+		var m dto.MetricFamily
+		err = decoder.Decode(&m)
+		if errors.Is(err, io.EOF) {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+		metrics = append(metrics, m)
+	}
+
+	sort.Slice(metrics, func(i, j int) bool {
+		return sort.StringsAreSorted([]string{*metrics[i].Name, *metrics[j].Name})
+	})
+	return metrics, nil
+}
+
+func readPrometheusDoc() ([]byte, error) {
+	doc, err := os.ReadFile(prometheusDocFile)
+	if err != nil {
+		return nil, err
+	}
+	return doc, nil
+}
+
+func updatePrometheusDoc(doc []byte, metricFamilies []dto.MetricFamily) ([]byte, error) {
+	i := bytes.Index(doc, generatorPrefix)
+	if i < 0 {
+		return nil, xerrors.New("generator prefix tag not found")
+	}
+	tableStartIndex := i + len(generatorPrefix) + 1
+
+	j := bytes.Index(doc[tableStartIndex:], generatorSuffix)
+	if j < 0 {
+		return nil, xerrors.New("generator suffix tag not found")
+	}
+	tableEndIndex := tableStartIndex + j
+
+	var buffer bytes.Buffer
+	buffer.Write(doc[:tableStartIndex])
+	buffer.WriteByte('\n')
+
+	buffer.WriteString("| Name | Type | Description | Labels |\n")
+	buffer.WriteString("| - | - | - | - |\n")
+	for _, mf := range metricFamilies {
+		buffer.WriteString("| ")
+		buffer.Write([]byte("`" + *mf.Name + "`"))
+		buffer.WriteString(" | ")
+		buffer.Write([]byte(strings.ToLower(mf.Type.String())))
+		buffer.WriteString(" | ")
+		if mf.Help != nil {
+			buffer.Write([]byte(*mf.Help))
+		}
+		buffer.WriteString(" | ")
+
+		labels := map[string]struct{}{}
+		metrics := mf.GetMetric()
+		for _, m := range metrics {
+			for _, label := range m.Label {
+				labels["`"+*label.Name+"`"] = struct{}{}
+			}
+		}
+
+		if len(labels) > 0 {
+			buffer.WriteString(strings.Join(sortedKeys(labels), " "))
+		}
+
+		buffer.WriteString(" |\n")
+	}
+
+	buffer.WriteByte('\n')
+	buffer.Write(doc[tableEndIndex:])
+	return buffer.Bytes(), nil
+}
+
+func writePrometheusDoc(doc []byte) error {
+	// G306: Expect WriteFile permissions to be 0600 or less
+	/* #nosec G306 */
+	err := os.WriteFile(prometheusDocFile, doc, 0644)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func sortedKeys(m map[string]struct{}) []string {
+	var keys []string
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}