Skip to content

Commit ee6c3f0

Browse files
johnstcnjohnstcn
committed
chore: add scaletest convenience script
1 parent 2631a84 commit ee6c3f0

13 files changed

+161
-14
lines changed

‎.gitignore

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,5 +58,5 @@ site/stats/
5858
# Loadtesting
5959
./scaletest/terraform/.terraform
6060
./scaletest/terraform/.terraform.lock.hcl
61-
terraform.tfstate.*
62-
**/*.tfvars
61+
scaletest/terraform/secrets.tfvars
62+
.terraform.tfstate.*

‎scaletest/terraform/coder.tf

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@ coder:
9696
secretKeyRef:
9797
name: "${kubernetes_secret.coder-db.metadata.0.name}"
9898
key: url
99+
- name: "CODER_PPROF_ENABLE"
100+
value: "true"
99101
- name: "CODER_PROMETHEUS_ENABLE"
100102
value: "true"
101103
- name: "CODER_VERBOSE"
@@ -216,6 +218,11 @@ resource "local_file" "kubernetes_template" {
216218
EOF
217219
}
218220

221+
resource "local_file" "output_vars" {
222+
filename = "${path.module}/.coderv2/url"
223+
content = local.coder_url
224+
}
225+
219226
output "coder_url" {
220227
description = "URL of the Coder deployment"
221228
value = local.coder_url

‎scaletest/terraform/coder_init.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,12 @@ if [[ "$ARCH" == "x86_64" ]]; then
1818
fi
1919
PLATFORM="$(uname | tr '[:upper:]' '[:lower:]')"
2020

21+
if [[ -f "${CONFIG_DIR}/coder.env" ]]; then
22+
echo "Found existing coder.env in ${CONFIG_DIR}!"
23+
echo "Nothing to do, exiting."
24+
exit 0
25+
fi
26+
2127
mkdir -p "${CONFIG_DIR}"
2228
echo "Fetching Coder CLI for first-time setup!"
2329
curl -fsSLk "${CODER_URL}/bin/coder-${PLATFORM}-${ARCH}" -o "${CONFIG_DIR}/coder"

‎scaletest/terraform/coder_workspacetraffic.sh

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@ CODER_TOKEN=$(./coder_shim.sh tokens create)
1515
CODER_URL="http://coder.coder-${LOADTEST_NAME}.svc.cluster.local"
1616
export KUBECONFIG="${PWD}/.coderv2/${LOADTEST_NAME}-cluster.kubeconfig"
1717

18+
# Clean up any pre-existing pods
19+
kubectl -n "coder-${LOADTEST_NAME}" delete pod coder-scaletest-workspace-traffic --force || true
20+
1821
cat <<EOF | kubectl apply -f -
1922
apiVersion: v1
2023
kind: Pod
@@ -37,7 +40,7 @@ spec:
3740
- command:
3841
- sh
3942
- -c
40-
- "curl -fsSL $CODER_URL/bin/coder-linux-amd64 -o /tmp/coder && chmod +x /tmp/coder && /tmp/coder --url=$CODER_URL --token=$CODER_TOKEN scaletest workspace-traffic"
43+
- "curl -fsSL $CODER_URL/bin/coder-linux-amd64 -o /tmp/coder && chmod +x /tmp/coder && /tmp/coder --verbose --url=$CODER_URL --token=$CODER_TOKEN scaletest workspace-traffic --concurrency=0 --bytes-per-tick=4096 --tick-interval=100ms"
4144
env:
4245
- name: CODER_URL
4346
value: $CODER_URL

‎scaletest/terraform/scenario-large.tfvars

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
nodepool_machine_type_coder = "t2d-standard-8"
2+
nodepool_machine_type_workspaces = "t2d-standard-8"
3+
coder_cpu = "7"
4+
coder_mem = "31Gi"

‎scaletest/terraform/scenario-large2x.tfvars

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
nodepool_machine_type_coder = "t2d-standard-8"
2+
nodepool_machine_type_workspaces = "t2d-standard-8"
3+
coder_replicas = 2
4+
coder_cpu = "7"
5+
coder_mem = "31Gi"

‎scaletest/terraform/scenario-medium.tfvars

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
nodepool_machine_type_coder = "t2d-standard-4"
2+
nodepool_machine_type_workspaces = "t2d-standard-4"
3+
coder_cpu = "3500m"
4+
coder_mem = "15Gi"

‎scaletest/terraform/scenario-medium2x.tfvars

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
nodepool_machine_type_coder = "t2d-standard-8"
2+
nodepool_machine_type_workspaces = "t2d-standard-8"
3+
nodepool_size_workspaces = 2
4+
coder_cpu = "3500m"
5+
coder_mem = "15Gi"

‎scaletest/terraform/scenario-small.tfvars

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
nodepool_machine_type_coder = "t2d-standard-2"
2+
nodepool_machine_type_workspaces = "t2d-standard-2"
3+
coder_cpu = "1500m"
4+
coder_mem = "7Gi"

‎scaletest/terraform/scenario-small2x.tfvars

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
nodepool_machine_type_coder = "t2d-standard-2"
2+
nodepool_machine_type_workspaces = "t2d-standard-2"
3+
nodepool_size_workspaces = 2
4+
coder_cpu = "1500m"
5+
coder_mem = "7Gi"

‎scaletest/terraform/secrets.tfvars.tpl

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
name = "${SCALETEST_NAME}"
2+
coder_image_repo = "gcr.io/coder-dev-1/coder-cian/coderv2"
3+
coder_image_tag = "dev"
4+
project_id = "${SCALETEST_PROJECT}"
5+
prometheus_remote_write_user = "${SCALETEST_PROMETHEUS_REMOTE_WRITE_USER}"
6+
prometheus_remote_write_password = "${SCALETEST_PROMETHEUS_REMOTE_WRITE_PASSWORD}"

‎scaletest/workspacetraffic/run.go

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@ import (
1919
"github.com/coder/coder/cryptorand"
2020
"github.com/coder/coder/scaletest/harness"
2121
"github.com/coder/coder/scaletest/loadtestutil"
22-
23-
promtest "github.com/prometheus/client_golang/prometheus/testutil"
2422
)
2523

2624
type Runner struct {
@@ -144,15 +142,6 @@ func (r *Runner) Run(ctx context.Context, _ string, logs io.Writer) error {
144142
return xerrors.Errorf("read from pty: %w", rErr)
145143
}
146144

147-
duration := time.Since(start)
148-
logger.Info(ctx, "Test Results",
149-
slog.F("duration", duration),
150-
slog.F("bytes_read_total", promtest.ToFloat64(r.metrics.BytesReadTotal)),
151-
slog.F("bytes_written_total", promtest.ToFloat64(r.metrics.BytesWrittenTotal)),
152-
slog.F("read_errors_total", promtest.ToFloat64(r.metrics.ReadErrorsTotal)),
153-
slog.F("write_errors_total", promtest.ToFloat64(r.metrics.WriteErrorsTotal)),
154-
)
155-
156145
return nil
157146
}
158147

‎scripts/scaletest.sh

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
#!/usr/bin/env bash
2+
3+
if [[ $# -lt 2 ]]; then
4+
echo "Usage: $0 <name> <scenario> <num_workspaces>"
5+
fi
6+
7+
[[ -n ${VERBOSE:-} ]] && set -x
8+
set -euo pipefail
9+
10+
SCALETEST_NAME="$1"
11+
SCALETEST_SCENARIO="$2"
12+
SCALETEST_NUM_WORKSPACES="$3"
13+
SCALETEST_PROJECT="${SCALETEST_PROJECT:-}"
14+
PROJECT_ROOT="$(git rev-parse --show-toplevel)"
15+
SCALETEST_SCENARIO_VARS="${PROJECT_ROOT}/scaletest/terraform/scenario-${SCALETEST_SCENARIO}.tfvars"
16+
SCALETEST_SECRETS="${PROJECT_ROOT}/scaletest/terraform/secrets.tfvars"
17+
SCALETEST_SECRETS_TEMPLATE="${PROJECT_ROOT}/scaletest/terraform/secrets.tfvars.tpl"
18+
SCALETEST_SKIP_CLEANUP="${SCALETEST_SKIP_CLEANUP:-}"
19+
20+
if [[ "${SCALETEST_SKIP_CLEANUP}" == "true" ]]; then
21+
echo "WARNING: you told me to not clean up after myself, so this is now your job!"
22+
fi
23+
24+
if [[ -z "${SCALETEST_PROJECT}" ]]; then
25+
echo "Environment variable SCALETEST_PROJECT not set. Please set it and try again."
26+
exit 1
27+
fi
28+
29+
if [[ ! -f "${SCALETEST_SCENARIO_VARS}" ]] ; then
30+
echo "No definition for scenario ${SCALETEST_SCENARIO} exists. Please create it and try again"
31+
exit 1
32+
fi
33+
34+
echo "Writing scaletest secrets to file."
35+
SCALETEST_NAME="${SCALETEST_NAME}" envsubst < "${SCALETEST_SECRETS_TEMPLATE}" > "${SCALETEST_SECRETS}"
36+
37+
pushd "${PROJECT_ROOT}/scaletest/terraform"
38+
39+
echo "Initializing terraform."
40+
terraform init
41+
42+
echo "Setting up infrastructure."
43+
terraform plan --var-file="${SCALETEST_SCENARIO_VARS}" --var-file="${SCALETEST_SECRETS}" -out=scaletest.tfplan
44+
terraform apply -auto-approve scaletest.tfplan
45+
46+
SCALETEST_CODER_URL=$(<./.coderv2/url)
47+
attempt_counter=0
48+
max_attempts=6 # 60 seconds
49+
echo -n "Waiting for Coder deployment at ${SCALETEST_CODER_URL} to become ready"
50+
until curl --output /dev/null --silent --fail "${SCALETEST_CODER_URL}/healthz"; do
51+
if [[ $attempt_counter -eq $max_attempts ]]; then
52+
echo
53+
echo "Max attempts reached."
54+
exit 1
55+
fi
56+
57+
echo -n '.'
58+
attempt_counter=$((attempt_counter+1))
59+
sleep 10
60+
done
61+
62+
echo "Initializing Coder deployment."
63+
./coder_init.sh "${SCALETEST_CODER_URL}"
64+
65+
echo "Creating ${SCALETEST_NUM_WORKSPACES} workspaces."
66+
./coder_shim.sh scaletest create-workspaces \
67+
--count "${SCALETEST_NUM_WORKSPACES}" \
68+
--template=kubernetes \
69+
--concurrency 10 \
70+
--no-cleanup
71+
72+
echo "Sleeping 10 minutes to establish a baseline measurement."
73+
sleep 600
74+
75+
echo "Sending traffic to workspaces"
76+
./coder_workspacetraffic.sh "${SCALETEST_NAME}"
77+
export KUBECONFIG="${PWD}/.coderv2/${SCALETEST_NAME}-cluster.kubeconfig"
78+
kubectl -n "coder-${SCALETEST_NAME}" wait pods coder-scaletest-workspace-traffic --for condition=Ready
79+
kubectl -n "coder-${SCALETEST_NAME}" logs -f pod/coder-scaletest-workspace-traffic
80+
81+
echo "Starting pprof"
82+
kubectl -n "coder-${SCALETEST_NAME}" port-forward deployment/coder 6061:6060 &
83+
pfpid=$!
84+
trap 'kill $pfpid' EXIT
85+
86+
echo -n "Waiting for pprof endpoint to become available"
87+
pprof_attempt_counter=0
88+
while ! timeout 1 bash -c "echo > /dev/tcp/localhost/6061"; do
89+
if [[ $pprof_attempt_counter -eq 10 ]]; then
90+
echo
91+
echo "pprof failed to become ready in time!"
92+
exit 1
93+
fi
94+
sleep 3
95+
echo -n "."
96+
done
97+
echo "Taking pprof snapshots"
98+
curl --silent --fail --output "${SCALETEST_NAME}-heap.pprof.gz" http://localhost:6061/debug/pprof/heap
99+
curl --silent --fail --output "${SCALETEST_NAME}-goroutine.pprof.gz" http://localhost:6061/debug/pprof/goroutine
100+
kill $pfpid
101+
102+
if [[ "${SCALETEST_SKIP_CLEANUP}" == "true" ]]; then
103+
echo "Leaving resources up for you to inspect."
104+
echo "Please don't forget to clean up afterwards!"
105+
exit 0
106+
fi
107+
108+
echo "Cleaning up"
109+
terraform apply --destroy --var-file="${SCALETEST_SCENARIO_VARS}" --var-file="${SCALETEST_SECRETS}" --auto-approve

0 commit comments

Comments
 (0)