coder · johnstcn · May 26, 2023 · May 15, 2023 · May 16, 2023 · May 16, 2023
diff --git a/.gitignore b/.gitignore
@@ -59,3 +59,4 @@ site/stats/
 ./scaletest/terraform/.terraform
 ./scaletest/terraform/.terraform.lock.hcl
 terraform.tfstate.*
+**/*.tfvars
diff --git a/.prettierignore b/.prettierignore
@@ -62,6 +62,7 @@ site/stats/
 ./scaletest/terraform/.terraform
 ./scaletest/terraform/.terraform.lock.hcl
 terraform.tfstate.*
+**/*.tfvars
 # .prettierignore.include:
 # Helm templates contain variables that are invalid YAML and can't be formatted
 # by Prettier.

diff --git a/cli/scaletest.go b/cli/scaletest.go
@@ -14,9 +14,14 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/xerrors"
 
+	"cdr.dev/slog"
+	"cdr.dev/slog/sloggers/sloghuman"
+
 	"github.com/coder/coder/cli/clibase"
 	"github.com/coder/coder/cli/cliui"
 	"github.com/coder/coder/coderd/httpapi"
@@ -896,8 +901,11 @@ func (r *RootCmd) scaletestCreateWorkspaces() *clibase.Cmd {
 
 func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
 	var (
-		tickInterval    time.Duration
-		bytesPerTick    int64
+		tickInterval               time.Duration
+		bytesPerTick               int64
+		scaletestPrometheusAddress string
+		scaletestPrometheusWait    time.Duration
+
 		client          = &codersdk.Client{}
 		tracingFlags    = &scaletestTracingFlags{}
 		strategy        = &scaletestStrategyFlags{}
@@ -913,6 +921,12 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
 		),
 		Handler: func(inv *clibase.Invocation) error {
 			ctx := inv.Context()
+			reg := prometheus.NewRegistry()
+			metrics := workspacetraffic.NewMetrics(reg, "username", "workspace_name", "agent_name")
+
+			logger := slog.Make(sloghuman.Sink(io.Discard))
+			prometheusSrvClose := ServeHandler(ctx, logger, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), scaletestPrometheusAddress, "prometheus")
+			defer prometheusSrvClose()
 
 			// Bypass rate limiting
 			client.HTTPClient = &http.Client{
@@ -943,6 +957,9 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
 				_, _ = fmt.Fprintln(inv.Stderr, "\nUploading traces...")
 				if err := closeTracing(ctx); err != nil {
 					_, _ = fmt.Fprintf(inv.Stderr, "\nError uploading traces: %+v\n", err)
+					// Wait for prometheus metrics to be scraped
+					_, _ = fmt.Fprintf(inv.Stderr, "Waiting %s for prometheus metrics to be scraped\n", scaletestPrometheusWait)
+					<-time.After(scaletestPrometheusWait)
 				}
 			}()
 			tracer := tracerProvider.Tracer(scaletestTracerName)
@@ -955,16 +972,18 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
 			th := harness.NewTestHarness(strategy.toStrategy(), cleanupStrategy.toStrategy())
 			for idx, ws := range workspaces {
 				var (
-					agentID uuid.UUID
-					name    = "workspace-traffic"
-					id      = strconv.Itoa(idx)
+					agentID   uuid.UUID
+					agentName string
+					name      = "workspace-traffic"
+					id        = strconv.Itoa(idx)
 				)
 
 				for _, res := range ws.LatestBuild.Resources {
 					if len(res.Agents) == 0 {
 						continue
 					}
 					agentID = res.Agents[0].ID
+					agentName = res.Agents[0].Name
 				}
 
 				if agentID == uuid.Nil {
@@ -974,16 +993,20 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
 
 				// Setup our workspace agent connection.
 				config := workspacetraffic.Config{
-					AgentID:      agentID,
-					BytesPerTick: bytesPerTick,
-					Duration:     strategy.timeout,
-					TickInterval: tickInterval,
+					AgentID:        agentID,
+					AgentName:      agentName,
+					BytesPerTick:   bytesPerTick,
+					Duration:       strategy.timeout,
+					TickInterval:   tickInterval,
+					WorkspaceName:  ws.Name,
+					WorkspaceOwner: ws.OwnerName,
+					Registry:       reg,
 				}
 
 				if err := config.Validate(); err != nil {
 					return xerrors.Errorf("validate config: %w", err)
 				}
-				var runner harness.Runnable = workspacetraffic.NewRunner(client, config)
+				var runner harness.Runnable = workspacetraffic.NewRunner(client, config, metrics)
 				if tracingEnabled {
 					runner = &runnableTraceWrapper{
 						tracer:   tracer,
@@ -1034,6 +1057,20 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
 			Description: "How often to send traffic.",
 			Value:       clibase.DurationOf(&tickInterval),
 		},
+		{
+			Flag:        "scaletest-prometheus-address",
+			Env:         "CODER_SCALETEST_PROMETHEUS_ADDRESS",
+			Default:     "0.0.0.0:21112",
+			Description: "Address on which to expose scaletest Prometheus metrics.",
+			Value:       clibase.StringOf(&scaletestPrometheusAddress),
+		},
+		{
+			Flag:        "scaletest-prometheus-wait",
+			Env:         "CODER_SCALETEST_PROMETHEUS_WAIT",
+			Default:     "5s",
+			Description: "How long to wait before exiting in order to allow Prometheus metrics to be scraped.",
+			Value:       clibase.DurationOf(&scaletestPrometheusWait),
+		},
 	}
 
 	tracingFlags.attach(&cmd.Options)

diff --git a/cli/scaletest_test.go b/cli/scaletest_test.go
@@ -8,17 +8,12 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/coder/coder/agent"
 	"github.com/coder/coder/cli/clitest"
 	"github.com/coder/coder/coderd/coderdtest"
 	"github.com/coder/coder/codersdk"
-	"github.com/coder/coder/codersdk/agentsdk"
-	"github.com/coder/coder/provisioner/echo"
-	"github.com/coder/coder/provisionersdk/proto"
 	"github.com/coder/coder/pty/ptytest"
 	"github.com/coder/coder/scaletest/harness"
 	"github.com/coder/coder/testutil"
@@ -205,70 +200,28 @@ param3: 1
 	})
 }
 
-// This test pretends to stand up a workspace and run a no-op traffic generation test.
-// It's not a real test, but it's useful for debugging.
-// We do not perform any cleanup.
+// This test just validates that the CLI command accepts its known arguments.
+// A more comprehensive test is performed in workspacetraffic/run_test.go
 func TestScaleTestWorkspaceTraffic(t *testing.T) {
 	t.Parallel()
 
 	ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitMedium)
 	defer cancelFunc()
 
-	client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
-	user := coderdtest.CreateFirstUser(t, client)
-
-	authToken := uuid.NewString()
-	version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
-		Parse:         echo.ParseComplete,
-		ProvisionPlan: echo.ProvisionComplete,
-		ProvisionApply: []*proto.Provision_Response{{
-			Type: &proto.Provision_Response_Complete{
-				Complete: &proto.Provision_Complete{
-					Resources: []*proto.Resource{{
-						Name: "example",
-						Type: "aws_instance",
-						Agents: []*proto.Agent{{
-							Id:   uuid.NewString(),
-							Name: "agent",
-							Auth: &proto.Agent_Token{
-								Token: authToken,
-							},
-							Apps: []*proto.App{},
-						}},
-					}},
-				},
-			},
-		}},
-	})
-	template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
-	coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
-
-	ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID, func(cwr *codersdk.CreateWorkspaceRequest) {
-		cwr.Name = "scaletest-test"
-	})
-	coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
-
-	agentClient := agentsdk.New(client.URL)
-	agentClient.SetSessionToken(authToken)
-	agentCloser := agent.New(agent.Options{
-		Client: agentClient,
-	})
-	t.Cleanup(func() {
-		_ = agentCloser.Close()
-	})
-
-	coderdtest.AwaitWorkspaceAgents(t, client, ws.ID)
+	client := coderdtest.New(t, nil)
+	_ = coderdtest.CreateFirstUser(t, client)
 
 	inv, root := clitest.New(t, "scaletest", "workspace-traffic",
 		"--timeout", "1s",
 		"--bytes-per-tick", "1024",
 		"--tick-interval", "100ms",
+		"--scaletest-prometheus-address", "127.0.0.1:0",
+		"--scaletest-prometheus-wait", "0s",
 	)
 	clitest.SetupConfig(t, client, root)
 	var stdout, stderr bytes.Buffer
 	inv.Stdout = &stdout
 	inv.Stderr = &stderr
 	err := inv.WithContext(ctx).Run()
-	require.NoError(t, err)
-	require.Contains(t, stdout.String(), "Pass:  1")
+	require.ErrorContains(t, err, "no scaletest workspaces exist")
 }
diff --git a/cli/testdata/coder_scaletest_workspace-traffic_--help.golden b/cli/testdata/coder_scaletest_workspace-traffic_--help.golden
@@ -27,6 +27,13 @@ Generate traffic to scaletest workspaces through coderd
           Output format specs in the format "<format>[:<path>]". Not specifying
           a path will default to stdout. Available formats: text, json.
 
+      --scaletest-prometheus-address string, $CODER_SCALETEST_PROMETHEUS_ADDRESS (default: 0.0.0.0:21112)
+          Address on which to expose scaletest Prometheus metrics.
+
+      --scaletest-prometheus-wait duration, $CODER_SCALETEST_PROMETHEUS_WAIT (default: 5s)
+          How long to wait before exiting in order to allow Prometheus metrics
+          to be scraped.
+
       --tick-interval duration, $CODER_SCALETEST_WORKSPACE_TRAFFIC_TICK_INTERVAL (default: 100ms)
           How often to send traffic.
 

diff --git a/docs/cli/scaletest_workspace-traffic.md b/docs/cli/scaletest_workspace-traffic.md
@@ -82,6 +82,26 @@ Timeout per job. Jobs may take longer to complete under higher concurrency limit
 
 Output format specs in the format "<format>[:<path>]". Not specifying a path will default to stdout. Available formats: text, json.
 
+### --scaletest-prometheus-address
+
+|             |                                                  |
+| ----------- | ------------------------------------------------ |
+| Type        | <code>string</code>                              |
+| Environment | <code>$CODER_SCALETEST_PROMETHEUS_ADDRESS</code> |
+| Default     | <code>0.0.0.0:21112</code>                       |
+
+Address on which to expose scaletest Prometheus metrics.
+
+### --scaletest-prometheus-wait
+
+|             |                                               |
+| ----------- | --------------------------------------------- |
+| Type        | <code>duration</code>                         |
+| Environment | <code>$CODER_SCALETEST_PROMETHEUS_WAIT</code> |
+| Default     | <code>5s</code>                               |
+
+How long to wait before exiting in order to allow Prometheus metrics to be scraped.
+
 ### --tick-interval
 
 |             |                                                               |

diff --git a/scaletest/terraform/README.md b/scaletest/terraform/README.md
@@ -32,9 +32,12 @@ project_id = "some_google_project_id"
 1. Run `coder_init.sh <coder_url>` to setup an initial user and a pre-configured Kubernetes
    template. It will also download the Coder CLI from the Coder instance locally.
 
-1. Do whatever you need to do with the Coder instance.
+1. Do whatever you need to do with the Coder instance:
 
-   > To run Coder commands against the instance, you can use `coder_shim.sh <command>`.
+   > Note: To run Coder commands against the instance, you can use `coder_shim.sh <command>`.
    > You don't need to run `coder login` yourself.
 
+   - To create workspaces, run `./coder_shim.sh scaletest create-workspaces --template="kubernetes" --count=N`
+   - To generate workspace traffic, run `./coder_trafficgen.sh <name of loadtest from your Terraform vars>`. This will keep running until you delete the pod `coder-scaletest-workspace-traffic`.
+
 1. When you are finished, you can run `terraform destroy -var-file=override.tfvars`.
diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
@@ -128,34 +128,6 @@ EOF
   ]
 }
 
-resource "local_file" "coder-monitoring-manifest" {
-  filename = "${path.module}/.coderv2/coder-monitoring.yaml"
-  content  = <<EOF
-apiVersion: monitoring.googleapis.com/v1
-kind: PodMonitoring
-metadata:
-  namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
-  name: coder-monitoring
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: coder
-  endpoints:
-  - port: prometheus-http
-    interval: 30s
-  EOF
-}
-
-resource "null_resource" "coder-monitoring-manifest_apply" {
-  provisioner "local-exec" {
-    working_dir = "${abspath(path.module)}/.coderv2"
-    command     = <<EOF
-KUBECONFIG=${var.name}-cluster.kubeconfig gcloud container clusters get-credentials ${var.name}-cluster --project=${var.project_id} --zone=${var.zone} && \
-KUBECONFIG=${var.name}-cluster.kubeconfig kubectl apply -f ${abspath(local_file.coder-monitoring-manifest.filename)}
-    EOF
-  }
-}
-
 resource "local_file" "kubernetes_template" {
   filename = "${path.module}/.coderv2/templates/kubernetes/main.tf"
   content  = <<EOF

diff --git a/scaletest/terraform/coder_workspacetraffic.sh b/scaletest/terraform/coder_workspacetraffic.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [[ $# -lt 1 ]]; then
+	echo "Usage: $0 <loadtest name>"
+	exit 1
+fi
+
+# Allow toggling verbose output
+[[ -n ${VERBOSE:-} ]] && set -x
+
+LOADTEST_NAME="$1"
+CODER_TOKEN=$(./coder_shim.sh tokens create)
+CODER_URL="http://coder.coder-${LOADTEST_NAME}.svc.cluster.local"
+export KUBECONFIG="${PWD}/.coderv2/${LOADTEST_NAME}-cluster.kubeconfig"
+
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: coder-scaletest-workspace-traffic
+  namespace: coder-${LOADTEST_NAME}
+  labels:
+    app.kubernetes.io/name: coder-scaletest-workspace-traffic
+spec:
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: cloud.google.com/gke-nodepool
+            operator: In
+            values:
+            - ${LOADTEST_NAME}-misc
+  containers:
+  - command:
+    - sh
+    - -c
+    - "curl -fsSL $CODER_URL/bin/coder-linux-amd64 -o /tmp/coder && chmod +x /tmp/coder && /tmp/coder --url=$CODER_URL --token=$CODER_TOKEN scaletest workspace-traffic"
+    env:
+    - name: CODER_URL
+      value: $CODER_URL
+    - name: CODER_TOKEN
+      value: $CODER_TOKEN
+    - name: CODER_SCALETEST_PROMETHEUS_ADDRESS
+      value: "0.0.0.0:21112"
+    - name: CODER_SCALETEST_JOB_TIMEOUT
+      value: "30m"
+    - name: CODER_SCALETEST_CONCURRENCY
+      value: "0"
+    - name: CODER_SCALETEST_WORKSPACE_TRAFFIC_BYTES_PER_TICK
+      value: "2048"
+    ports:
+    - containerPort: 21112
+      name: prometheus-http
+      protocol: TCP
+    name: cli
+    image: docker.io/codercom/enterprise-minimal:ubuntu
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  namespace: coder-${LOADTEST_NAME}
+  name: coder-workspacetraffic-monitoring
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: coder-scaletest-workspace-traffic
+  podMetricsEndpoints:
+  - port: prometheus-http
+    interval: 15s
+EOF