Skip to content

Commit 30d9d84

Browse files
mtojekmtojek
authored
fix: use flag to enable Prometheus (#12345)
1 parent bedd2c5 commit 30d9d84

8 files changed

+160
-11
lines changed

‎cli/clitest/clitest.go

+10
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,10 @@ func extractTar(t *testing.T, data []byte, directory string) {
141141
// Start runs the command in a goroutine and cleans it up when the test
142142
// completed.
143143
func Start(t *testing.T, inv *clibase.Invocation) {
144+
StartWithAssert(t, inv, nil)
145+
}
146+
147+
func StartWithAssert(t *testing.T, inv *clibase.Invocation, assertCallback func(t *testing.T, err error)) { //nolint:revive
144148
t.Helper()
145149

146150
closeCh := make(chan struct{})
@@ -155,6 +159,12 @@ func Start(t *testing.T, inv *clibase.Invocation) {
155159
go func() {
156160
defer close(closeCh)
157161
err := waiter.Wait()
162+
163+
if assertCallback != nil {
164+
assertCallback(t, err)
165+
return
166+
}
167+
158168
switch {
159169
case errors.Is(err, context.Canceled):
160170
return

‎docs/admin/provisioners.md

+14
Original file line numberDiff line numberDiff line change
@@ -229,3 +229,17 @@ This can be disabled with a server-wide
229229
```shell
230230
coder server --provisioner-daemons=0
231231
```
232+
233+
## Prometheus metrics
234+
235+
Coder provisioner daemon exports metrics via the HTTP endpoint, which can be
236+
enabled using either the environment variable `CODER_PROMETHEUS_ENABLE` or the
237+
flag `--prometheus-enable`.
238+
239+
The Prometheus endpoint address is `http://localhost:2112/` by default. You can
240+
use either the environment variable `CODER_PROMETHEUS_ADDRESS` or the flag
241+
`--prometheus-address <network-interface>:<port>` to select a different listen
242+
address.
243+
244+
If you have provisioners daemons deployed as pods, it is advised to monitor them
245+
separately.

‎docs/admin/workspace-proxies.md

+11
Original file line numberDiff line numberDiff line change
@@ -184,3 +184,14 @@ goes offline, the session will fall back to the primary proxy. This could take
184184
up to 60 seconds.
185185

186186
![Workspace proxy picker](../images/admin/workspace-proxy-picker.png)
187+
188+
## Step 3: Observability
189+
190+
Coder workspace proxy exports metrics via the HTTP endpoint, which can be
191+
enabled using either the environment variable `CODER_PROMETHEUS_ENABLE` or the
192+
flag `--prometheus-enable`.
193+
194+
The Prometheus endpoint address is `http://localhost:2112/` by default. You can
195+
use either the environment variable `CODER_PROMETHEUS_ADDRESS` or the flag
196+
`--prometheus-address <network-interface>:<port>` to select a different listen
197+
address.

‎docs/cli/provisionerd_start.md

+10
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎enterprise/cli/provisionerdaemons.go

+9-1
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
7171
preSharedKey string
7272
verbose bool
7373

74+
prometheusEnable bool
7475
prometheusAddress string
7576
)
7677
client := new(codersdk.Client)
@@ -171,7 +172,7 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
171172
}()
172173

173174
var metrics *provisionerd.Metrics
174-
if prometheusAddress != "" {
175+
if prometheusEnable {
175176
logger.Info(ctx, "starting Prometheus endpoint", slog.F("address", prometheusAddress))
176177

177178
prometheusRegistry := prometheus.NewRegistry()
@@ -315,6 +316,13 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
315316
Value: clibase.StringArrayOf(&logFilter),
316317
Default: "",
317318
},
319+
{
320+
Flag: "prometheus-enable",
321+
Env: "CODER_PROMETHEUS_ENABLE",
322+
Description: "Serve prometheus metrics on the address defined by prometheus address.",
323+
Value: clibase.BoolOf(&prometheusEnable),
324+
Default: "false",
325+
},
318326
{
319327
Flag: "prometheus-address",
320328
Env: "CODER_PROMETHEUS_ADDRESS",

‎enterprise/cli/provisionerdaemons_test.go

+11-10
Original file line numberDiff line numberDiff line change
@@ -170,15 +170,6 @@ func TestProvisionerDaemon_SessionToken(t *testing.T) {
170170
t.Run("PrometheusEnabled", func(t *testing.T) {
171171
t.Parallel()
172172

173-
// Helper function to find a free random port
174-
randomPort := func(t *testing.T) int {
175-
random, err := net.Listen("tcp", "127.0.0.1:0")
176-
require.NoError(t, err)
177-
_ = random.Close()
178-
tcpAddr, valid := random.Addr().(*net.TCPAddr)
179-
require.True(t, valid)
180-
return tcpAddr.Port
181-
}
182173
prometheusPort := randomPort(t)
183174

184175
// Configure CLI client
@@ -191,7 +182,7 @@ func TestProvisionerDaemon_SessionToken(t *testing.T) {
191182
},
192183
})
193184
anotherClient, _ := coderdtest.CreateAnotherUser(t, client, admin.OrganizationID, rbac.RoleTemplateAdmin())
194-
inv, conf := newCLI(t, "provisionerd", "start", "--name", "daemon-with-prometheus", "--prometheus-address", fmt.Sprintf("127.0.0.1:%d", prometheusPort))
185+
inv, conf := newCLI(t, "provisionerd", "start", "--name", "daemon-with-prometheus", "--prometheus-enable", "--prometheus-address", fmt.Sprintf("127.0.0.1:%d", prometheusPort))
195186
clitest.SetupConfig(t, anotherClient, conf)
196187
pty := ptytest.New(t).Attach(inv)
197188
ctx, cancel := context.WithTimeout(inv.Context(), testutil.WaitLong)
@@ -251,3 +242,13 @@ func TestProvisionerDaemon_SessionToken(t *testing.T) {
251242
require.True(t, hasPromHTTP, "Prometheus HTTP metrics are missing")
252243
})
253244
}
245+
246+
// randomPort is a helper function to find a free random port, for instance to spawn Prometheus endpoint.
247+
func randomPort(t *testing.T) int {
248+
random, err := net.Listen("tcp", "127.0.0.1:0")
249+
require.NoError(t, err)
250+
_ = random.Close()
251+
tcpAddr, valid := random.Addr().(*net.TCPAddr)
252+
require.True(t, valid)
253+
return tcpAddr.Port
254+
}

‎enterprise/cli/proxyserver_test.go

+92
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,23 @@
11
package cli_test
22

33
import (
4+
"bufio"
5+
"context"
6+
"errors"
47
"fmt"
58
"net/http"
69
"net/http/httptest"
10+
"strings"
11+
"sync"
712
"sync/atomic"
813
"testing"
914

1015
"github.com/stretchr/testify/assert"
1116
"github.com/stretchr/testify/require"
1217

18+
"github.com/coder/coder/v2/cli/clitest"
1319
"github.com/coder/coder/v2/pty/ptytest"
20+
"github.com/coder/coder/v2/testutil"
1421
)
1522

1623
func Test_Headers(t *testing.T) {
@@ -52,3 +59,88 @@ func Test_Headers(t *testing.T) {
5259

5360
assert.EqualValues(t, 1, atomic.LoadInt64(&called))
5461
}
62+
63+
func TestWorkspaceProxy_Server_PrometheusEnabled(t *testing.T) {
64+
t.Parallel()
65+
66+
prometheusPort := randomPort(t)
67+
68+
var wg sync.WaitGroup
69+
wg.Add(1)
70+
71+
// Start fake coderd
72+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
73+
if r.URL.Path == "/api/v2/workspaceproxies/me/register" {
74+
// Give fake app_security_key (96 bytes)
75+
w.WriteHeader(http.StatusCreated)
76+
_, _ = w.Write([]byte(`{"app_security_key": "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789123456012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789123456"}`))
77+
return
78+
}
79+
if r.URL.Path == "/api/v2/workspaceproxies/me/coordinate" {
80+
// Slow down proxy registration, so that test runner can check if Prometheus endpoint is exposed.
81+
wg.Wait()
82+
83+
// Does not matter, we are not going to implement a real workspace proxy.
84+
w.WriteHeader(http.StatusNotImplemented)
85+
return
86+
}
87+
88+
w.Header().Add("Content-Type", "application/json")
89+
_, _ = w.Write([]byte(`{}`)) // build info can be ignored
90+
}))
91+
defer srv.Close()
92+
defer wg.Done()
93+
94+
// Configure CLI client
95+
inv, _ := newCLI(t, "wsproxy", "server",
96+
"--primary-access-url", srv.URL,
97+
"--proxy-session-token", "test-token",
98+
"--access-url", "http://foobar:3001",
99+
"--http-address", fmt.Sprintf("127.0.0.1:%d", randomPort(t)),
100+
"--prometheus-enable",
101+
"--prometheus-address", fmt.Sprintf("127.0.0.1:%d", prometheusPort),
102+
)
103+
pty := ptytest.New(t).Attach(inv)
104+
105+
ctx, cancel := context.WithTimeout(inv.Context(), testutil.WaitLong)
106+
defer cancel()
107+
108+
// Start "wsproxy server" command
109+
clitest.StartWithAssert(t, inv, func(t *testing.T, err error) {
110+
assert.Error(t, err)
111+
assert.False(t, errors.Is(err, context.Canceled), "error was expected, but context was canceled")
112+
})
113+
pty.ExpectMatchContext(ctx, "Started HTTP listener at")
114+
115+
// Fetch metrics from Prometheus endpoint
116+
var res *http.Response
117+
require.Eventually(t, func() bool {
118+
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://127.0.0.1:%d", prometheusPort), nil)
119+
assert.NoError(t, err)
120+
// nolint:bodyclose
121+
res, err = http.DefaultClient.Do(req)
122+
return err == nil
123+
}, testutil.WaitShort, testutil.IntervalFast)
124+
defer res.Body.Close()
125+
126+
// Scan for metric patterns
127+
scanner := bufio.NewScanner(res.Body)
128+
hasGoStats := false
129+
hasPromHTTP := false
130+
for scanner.Scan() {
131+
if strings.HasPrefix(scanner.Text(), "go_goroutines") {
132+
hasGoStats = true
133+
continue
134+
}
135+
if strings.HasPrefix(scanner.Text(), "promhttp_metric_handler_requests_total") {
136+
hasPromHTTP = true
137+
continue
138+
}
139+
t.Logf("scanned %s", scanner.Text())
140+
}
141+
require.NoError(t, scanner.Err())
142+
143+
// Verify patterns
144+
require.True(t, hasGoStats, "Go stats are missing")
145+
require.True(t, hasPromHTTP, "Prometheus HTTP metrics are missing")
146+
}

‎enterprise/cli/testdata/coder_provisionerd_start_--help.golden

+3
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,9 @@ OPTIONS:
3535
--prometheus-address string, $CODER_PROMETHEUS_ADDRESS (default: 127.0.0.1:2112)
3636
The bind address to serve prometheus metrics.
3737

38+
--prometheus-enable bool, $CODER_PROMETHEUS_ENABLE (default: false)
39+
Serve prometheus metrics on the address defined by prometheus address.
40+
3841
--psk string, $CODER_PROVISIONER_DAEMON_PSK
3942
Pre-shared key to authenticate with Coder server.
4043

0 commit comments

Comments
 (0)