chore: add scaletest convenience script

coder · johnstcn · Jun 8, 2023 · Jun 2, 2023 · Jun 2, 2023 · Jun 2, 2023
commit ee6c3f08f581de16ff258c843d86d786b0f6af16
diff --git a/.gitignore b/.gitignore
@@ -58,5 +58,5 @@ site/stats/
 # Loadtesting
 ./scaletest/terraform/.terraform
 ./scaletest/terraform/.terraform.lock.hcl
-terraform.tfstate.*
-**/*.tfvars
+scaletest/terraform/secrets.tfvars
+.terraform.tfstate.*
diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
@@ -96,6 +96,8 @@ coder:
         secretKeyRef:
           name: "${kubernetes_secret.coder-db.metadata.0.name}"
           key: url
+    - name: "CODER_PPROF_ENABLE"
+      value: "true"
     - name: "CODER_PROMETHEUS_ENABLE"
       value: "true"
     - name: "CODER_VERBOSE"
@@ -216,6 +218,11 @@ resource "local_file" "kubernetes_template" {
   EOF
 }
 
+resource "local_file" "output_vars" {
+  filename = "${path.module}/.coderv2/url"
+  content = local.coder_url
+}
+
 output "coder_url" {
   description = "URL of the Coder deployment"
   value       = local.coder_url

diff --git a/scaletest/terraform/coder_init.sh b/scaletest/terraform/coder_init.sh
@@ -18,6 +18,12 @@ if [[ "$ARCH" == "x86_64" ]]; then
 fi
 PLATFORM="$(uname | tr '[:upper:]' '[:lower:]')"
 
+if [[ -f "${CONFIG_DIR}/coder.env" ]]; then
+	echo "Found existing coder.env in ${CONFIG_DIR}!"
+	echo "Nothing to do, exiting."
+	exit 0
+fi
+
 mkdir -p "${CONFIG_DIR}"
 echo "Fetching Coder CLI for first-time setup!"
 curl -fsSLk "${CODER_URL}/bin/coder-${PLATFORM}-${ARCH}" -o "${CONFIG_DIR}/coder"

diff --git a/scaletest/terraform/coder_workspacetraffic.sh b/scaletest/terraform/coder_workspacetraffic.sh
@@ -15,6 +15,9 @@ CODER_TOKEN=$(./coder_shim.sh tokens create)
 CODER_URL="http://coder.coder-${LOADTEST_NAME}.svc.cluster.local"
 export KUBECONFIG="${PWD}/.coderv2/${LOADTEST_NAME}-cluster.kubeconfig"
 
+# Clean up any pre-existing pods
+kubectl -n "coder-${LOADTEST_NAME}" delete pod coder-scaletest-workspace-traffic --force || true
+
 cat <<EOF | kubectl apply -f -
 apiVersion: v1
 kind: Pod
@@ -37,7 +40,7 @@ spec:
   - command:
     - sh
     - -c
-    - "curl -fsSL $CODER_URL/bin/coder-linux-amd64 -o /tmp/coder && chmod +x /tmp/coder && /tmp/coder --url=$CODER_URL --token=$CODER_TOKEN scaletest workspace-traffic"
+    - "curl -fsSL $CODER_URL/bin/coder-linux-amd64 -o /tmp/coder && chmod +x /tmp/coder && /tmp/coder --verbose --url=$CODER_URL --token=$CODER_TOKEN scaletest workspace-traffic --concurrency=0 --bytes-per-tick=4096 --tick-interval=100ms"
     env:
     - name: CODER_URL
       value: $CODER_URL

diff --git a/scaletest/terraform/scenario-large.tfvars b/scaletest/terraform/scenario-large.tfvars
@@ -0,0 +1,4 @@
+nodepool_machine_type_coder = "t2d-standard-8"
+nodepool_machine_type_workspaces = "t2d-standard-8"
+coder_cpu = "7"
+coder_mem = "31Gi"
diff --git a/scaletest/terraform/scenario-large2x.tfvars b/scaletest/terraform/scenario-large2x.tfvars
@@ -0,0 +1,5 @@
+nodepool_machine_type_coder = "t2d-standard-8"
+nodepool_machine_type_workspaces = "t2d-standard-8"
+coder_replicas = 2
+coder_cpu = "7"
+coder_mem = "31Gi"
diff --git a/scaletest/terraform/scenario-medium.tfvars b/scaletest/terraform/scenario-medium.tfvars
@@ -0,0 +1,4 @@
+nodepool_machine_type_coder = "t2d-standard-4"
+nodepool_machine_type_workspaces = "t2d-standard-4"
+coder_cpu = "3500m"
+coder_mem = "15Gi"
diff --git a/scaletest/terraform/scenario-medium2x.tfvars b/scaletest/terraform/scenario-medium2x.tfvars
@@ -0,0 +1,5 @@
+nodepool_machine_type_coder = "t2d-standard-8"
+nodepool_machine_type_workspaces = "t2d-standard-8"
+nodepool_size_workspaces = 2
+coder_cpu = "3500m"
+coder_mem = "15Gi"
diff --git a/scaletest/terraform/scenario-small.tfvars b/scaletest/terraform/scenario-small.tfvars
@@ -0,0 +1,4 @@
+nodepool_machine_type_coder = "t2d-standard-2"
+nodepool_machine_type_workspaces = "t2d-standard-2"
+coder_cpu = "1500m"
+coder_mem = "7Gi"
diff --git a/scaletest/terraform/scenario-small2x.tfvars b/scaletest/terraform/scenario-small2x.tfvars
@@ -0,0 +1,5 @@
+nodepool_machine_type_coder = "t2d-standard-2"
+nodepool_machine_type_workspaces = "t2d-standard-2"
+nodepool_size_workspaces = 2
+coder_cpu = "1500m"
+coder_mem = "7Gi"
diff --git a/scaletest/terraform/secrets.tfvars.tpl b/scaletest/terraform/secrets.tfvars.tpl
@@ -0,0 +1,6 @@
+name = "${SCALETEST_NAME}"
+coder_image_repo = "gcr.io/coder-dev-1/coder-cian/coderv2"
+coder_image_tag = "dev"
+project_id = "${SCALETEST_PROJECT}"
+prometheus_remote_write_user = "${SCALETEST_PROMETHEUS_REMOTE_WRITE_USER}"
+prometheus_remote_write_password = "${SCALETEST_PROMETHEUS_REMOTE_WRITE_PASSWORD}"
diff --git a/scaletest/workspacetraffic/run.go b/scaletest/workspacetraffic/run.go
@@ -19,8 +19,6 @@ import (
 	"github.com/coder/coder/cryptorand"
 	"github.com/coder/coder/scaletest/harness"
 	"github.com/coder/coder/scaletest/loadtestutil"
-
-	promtest "github.com/prometheus/client_golang/prometheus/testutil"
 )
 
 type Runner struct {
@@ -144,15 +142,6 @@ func (r *Runner) Run(ctx context.Context, _ string, logs io.Writer) error {
 		return xerrors.Errorf("read from pty: %w", rErr)
 	}
 
-	duration := time.Since(start)
-	logger.Info(ctx, "Test Results",
-		slog.F("duration", duration),
-		slog.F("bytes_read_total", promtest.ToFloat64(r.metrics.BytesReadTotal)),
-		slog.F("bytes_written_total", promtest.ToFloat64(r.metrics.BytesWrittenTotal)),
-		slog.F("read_errors_total", promtest.ToFloat64(r.metrics.ReadErrorsTotal)),
-		slog.F("write_errors_total", promtest.ToFloat64(r.metrics.WriteErrorsTotal)),
-	)
-
 	return nil
 }
 

diff --git a/scripts/scaletest.sh b/scripts/scaletest.sh
@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+if [[ $# -lt 2 ]]; then
+	echo "Usage: $0 <name> <scenario> <num_workspaces>"
+fi
+
+[[ -n ${VERBOSE:-} ]] && set -x
+set -euo pipefail
+
+SCALETEST_NAME="$1"
+SCALETEST_SCENARIO="$2"
+SCALETEST_NUM_WORKSPACES="$3"
+SCALETEST_PROJECT="${SCALETEST_PROJECT:-}"
+PROJECT_ROOT="$(git rev-parse --show-toplevel)"
+SCALETEST_SCENARIO_VARS="${PROJECT_ROOT}/scaletest/terraform/scenario-${SCALETEST_SCENARIO}.tfvars"
+SCALETEST_SECRETS="${PROJECT_ROOT}/scaletest/terraform/secrets.tfvars"
+SCALETEST_SECRETS_TEMPLATE="${PROJECT_ROOT}/scaletest/terraform/secrets.tfvars.tpl"
+SCALETEST_SKIP_CLEANUP="${SCALETEST_SKIP_CLEANUP:-}"
+
+if [[ "${SCALETEST_SKIP_CLEANUP}" == "true" ]]; then
+	echo "WARNING: you told me to not clean up after myself, so this is now your job!"
+fi
+
+if [[ -z "${SCALETEST_PROJECT}" ]]; then
+	echo "Environment variable SCALETEST_PROJECT not set. Please set it and try again."
+	exit 1
+fi
+
+if [[ ! -f "${SCALETEST_SCENARIO_VARS}" ]] ; then
+	echo "No definition for scenario ${SCALETEST_SCENARIO} exists. Please create it and try again"
+	exit 1
+fi
+
+echo "Writing scaletest secrets to file."
+SCALETEST_NAME="${SCALETEST_NAME}" envsubst < "${SCALETEST_SECRETS_TEMPLATE}" > "${SCALETEST_SECRETS}"
+
+pushd "${PROJECT_ROOT}/scaletest/terraform"
+
+echo "Initializing terraform."
+terraform init
+
+echo "Setting up infrastructure."
+terraform plan --var-file="${SCALETEST_SCENARIO_VARS}" --var-file="${SCALETEST_SECRETS}" -out=scaletest.tfplan
+terraform apply -auto-approve scaletest.tfplan
+
+SCALETEST_CODER_URL=$(<./.coderv2/url)
+attempt_counter=0
+max_attempts=6 # 60 seconds
+echo -n "Waiting for Coder deployment at ${SCALETEST_CODER_URL} to become ready"
+until curl --output /dev/null --silent --fail "${SCALETEST_CODER_URL}/healthz"; do
+	if [[ $attempt_counter -eq $max_attempts ]]; then
+		echo
+		echo "Max attempts reached."
+		exit 1
+	fi
+
+	echo -n '.'
+	attempt_counter=$((attempt_counter+1))
+	sleep 10
+done
+
+echo "Initializing Coder deployment."
+./coder_init.sh "${SCALETEST_CODER_URL}"
+
+echo "Creating ${SCALETEST_NUM_WORKSPACES} workspaces."
+./coder_shim.sh scaletest create-workspaces \
+	--count "${SCALETEST_NUM_WORKSPACES}" \
+	--template=kubernetes \
+	--concurrency 10 \
+	--no-cleanup
+
+echo "Sleeping 10 minutes to establish a baseline measurement."
+sleep 600
+
+echo "Sending traffic to workspaces"
+./coder_workspacetraffic.sh "${SCALETEST_NAME}"
+export KUBECONFIG="${PWD}/.coderv2/${SCALETEST_NAME}-cluster.kubeconfig"
+kubectl -n "coder-${SCALETEST_NAME}" wait pods coder-scaletest-workspace-traffic --for condition=Ready
+kubectl -n "coder-${SCALETEST_NAME}" logs -f pod/coder-scaletest-workspace-traffic
+
+echo "Starting pprof"
+kubectl -n "coder-${SCALETEST_NAME}" port-forward deployment/coder 6061:6060 &
+pfpid=$!
+trap 'kill $pfpid' EXIT
+
+echo -n "Waiting for pprof endpoint to become available"
+pprof_attempt_counter=0
+while ! timeout 1 bash -c "echo > /dev/tcp/localhost/6061"; do
+	if [[ $pprof_attempt_counter -eq 10 ]]; then
+		echo
+		echo "pprof failed to become ready in time!"
+		exit 1
+	fi
+	sleep 3
+	echo -n "."
+done
+echo "Taking pprof snapshots"
+curl --silent --fail --output "${SCALETEST_NAME}-heap.pprof.gz" http://localhost:6061/debug/pprof/heap
+curl --silent --fail --output "${SCALETEST_NAME}-goroutine.pprof.gz" http://localhost:6061/debug/pprof/goroutine
+kill $pfpid
+
+if [[ "${SCALETEST_SKIP_CLEANUP}" == "true" ]]; then
+	echo "Leaving resources up for you to inspect."
+	echo "Please don't forget to clean up afterwards!"
+	exit 0
+fi
+
+echo "Cleaning up"
+terraform apply --destroy --var-file="${SCALETEST_SCENARIO_VARS}" --var-file="${SCALETEST_SECRETS}" --auto-approve