Skip to content

Commit df842b3

Browse files
johnstcnjohnstcn
authored
chore: fix miscellaneous issues in scaletest scripts (coder#8006)
* chore: scaletest: plumb through more options * bump terraform version * scaletest.sh: pprof during traffic gen * cli/scaletest: actually wait for prometheus metrics to be scraped * increase prometheus wait
1 parent 1da2570 commit df842b3

File tree

4 files changed

+83
-26
lines changed

4 files changed

+83
-26
lines changed

‎cli/scaletest.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -902,10 +902,10 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
902902
_, _ = fmt.Fprintln(inv.Stderr, "\nUploading traces...")
903903
if err := closeTracing(ctx); err != nil {
904904
_, _ = fmt.Fprintf(inv.Stderr, "\nError uploading traces: %+v\n", err)
905-
// Wait for prometheus metrics to be scraped
906-
_, _ = fmt.Fprintf(inv.Stderr, "Waiting %s for prometheus metrics to be scraped\n", scaletestPrometheusWait)
907-
<-time.After(scaletestPrometheusWait)
908905
}
906+
// Wait for prometheus metrics to be scraped
907+
_, _ = fmt.Fprintf(inv.Stderr, "Waiting %s for prometheus metrics to be scraped\n", scaletestPrometheusWait)
908+
<-time.After(scaletestPrometheusWait)
909909
}()
910910
tracer := tracerProvider.Tracer(scaletestTracerName)
911911

‎scaletest/lib/coder_workspacetraffic.sh

+53-17
Original file line numberDiff line numberDiff line change
@@ -2,29 +2,68 @@
22

33
set -euo pipefail
44

5-
if [[ $# -lt 1 ]]; then
6-
echo "Usage: $0 <loadtest name>"
7-
exit 1
8-
fi
5+
PROJECT_ROOT="$(git rev-parse --show-toplevel)"
6+
# shellcheck source=scripts/lib.sh
7+
source "${PROJECT_ROOT}/scripts/lib.sh"
98

109
# Allow toggling verbose output
1110
[[ -n ${VERBOSE:-} ]] && set -x
1211

13-
LOADTEST_NAME="$1"
14-
PROJECT_ROOT="$(git rev-parse --show-toplevel)"
12+
SCALETEST_NAME="${SCALETEST_NAME:-}"
13+
SCALETEST_TRAFFIC_BYTES_PER_TICK="${SCALETEST_TRAFFIC_BYTES_PER_TICK:-1024}"
14+
SCALETEST_TRAFFIC_TICK_INTERVAL="${SCALETEST_TRAFFIC_TICK_INTERVAL:-100ms}"
15+
16+
script_name=$(basename "$0")
17+
args="$(getopt -o "" -l help,name:,traffic-bytes-per-tick:,traffic-tick-interval:, -- "$@")"
18+
eval set -- "$args"
19+
while true; do
20+
case "$1" in
21+
--help)
22+
echo "Usage: $script_name --name <name> [--traffic-bytes-per-tick <bytes_per-tick>] [--traffic-tick-interval <ticks_per_second]"
23+
exit 1
24+
;;
25+
--name)
26+
SCALETEST_NAME="$2"
27+
shift 2
28+
;;
29+
--traffic-bytes-per-tick)
30+
SCALETEST_TRAFFIC_BYTES_PER_TICK="$2"
31+
shift 2
32+
;;
33+
--traffic-tick-interval)
34+
SCALETEST_TRAFFIC_TICK_INTERVAL="$2"
35+
shift 2
36+
;;
37+
--)
38+
shift
39+
break
40+
;;
41+
*)
42+
error "Unrecognized option: $1"
43+
;;
44+
esac
45+
done
46+
47+
dependencies kubectl
48+
49+
if [[ -z "${SCALETEST_NAME}" ]]; then
50+
echo "Must specify --name"
51+
exit 1
52+
fi
53+
1554
CODER_TOKEN=$("${PROJECT_ROOT}/scaletest/lib/coder_shim.sh" tokens create)
16-
CODER_URL="http://coder.coder-${LOADTEST_NAME}.svc.cluster.local"
17-
export KUBECONFIG="${PROJECT_ROOT}/scaletest/.coderv2/${LOADTEST_NAME}-cluster.kubeconfig"
55+
CODER_URL="http://coder.coder-${SCALETEST_NAME}.svc.cluster.local"
56+
export KUBECONFIG="${PROJECT_ROOT}/scaletest/.coderv2/${SCALETEST_NAME}-cluster.kubeconfig"
1857

1958
# Clean up any pre-existing pods
20-
kubectl -n "coder-${LOADTEST_NAME}" delete pod coder-scaletest-workspace-traffic --force || true
59+
kubectl -n "coder-${SCALETEST_NAME}" delete pod coder-scaletest-workspace-traffic --force || true
2160

2261
cat <<EOF | kubectl apply -f -
2362
apiVersion: v1
2463
kind: Pod
2564
metadata:
2665
name: coder-scaletest-workspace-traffic
27-
namespace: coder-${LOADTEST_NAME}
66+
namespace: coder-${SCALETEST_NAME}
2867
labels:
2968
app.kubernetes.io/name: coder-scaletest-workspace-traffic
3069
spec:
@@ -36,12 +75,12 @@ spec:
3675
- key: cloud.google.com/gke-nodepool
3776
operator: In
3877
values:
39-
- ${LOADTEST_NAME}-misc
78+
- ${SCALETEST_NAME}-misc
4079
containers:
4180
- command:
4281
- sh
4382
- -c
44-
- "curl -fsSL $CODER_URL/bin/coder-linux-amd64 -o /tmp/coder && chmod +x /tmp/coder && /tmp/coder --verbose --url=$CODER_URL --token=$CODER_TOKEN scaletest workspace-traffic --concurrency=0 --bytes-per-tick=4096 --tick-interval=100ms"
83+
- "curl -fsSL $CODER_URL/bin/coder-linux-amd64 -o /tmp/coder && chmod +x /tmp/coder && /tmp/coder --verbose --url=$CODER_URL --token=$CODER_TOKEN scaletest workspace-traffic --concurrency=0 --bytes-per-tick=${SCALETEST_TRAFFIC_BYTES_PER_TICK} --tick-interval=${SCALETEST_TRAFFIC_TICK_INTERVAL} --scaletest-prometheus-wait=60s"
4584
env:
4685
- name: CODER_URL
4786
value: $CODER_URL
@@ -51,21 +90,18 @@ spec:
5190
value: "0.0.0.0:21112"
5291
- name: CODER_SCALETEST_JOB_TIMEOUT
5392
value: "30m"
54-
- name: CODER_SCALETEST_CONCURRENCY
55-
value: "0"
56-
- name: CODER_SCALETEST_WORKSPACE_TRAFFIC_BYTES_PER_TICK
57-
value: "2048"
5893
ports:
5994
- containerPort: 21112
6095
name: prometheus-http
6196
protocol: TCP
6297
name: cli
6398
image: docker.io/codercom/enterprise-minimal:ubuntu
99+
restartPolicy: Never
64100
---
65101
apiVersion: monitoring.coreos.com/v1
66102
kind: PodMonitor
67103
metadata:
68-
namespace: coder-${LOADTEST_NAME}
104+
namespace: coder-${SCALETEST_NAME}
69105
name: coder-workspacetraffic-monitoring
70106
spec:
71107
selector:

‎scaletest/scaletest.sh

+26-5
Original file line numberDiff line numberDiff line change
@@ -15,18 +15,25 @@ SCALETEST_PROJECT="${SCALETEST_PROJECT:-}"
1515
SCALETEST_PROMETHEUS_REMOTE_WRITE_USER="${SCALETEST_PROMETHEUS_REMOTE_WRITE_USER:-}"
1616
SCALETEST_PROMETHEUS_REMOTE_WRITE_PASSWORD="${SCALETEST_PROMETHEUS_REMOTE_WRITE_PASSWORD:-}"
1717
SCALETEST_SKIP_CLEANUP="${SCALETEST_SKIP_CLEANUP:-0}"
18+
SCALETEST_CREATE_CONCURRENCY="${SCALETEST_CREATE_CONCURRENCY:-10}"
19+
SCALETEST_TRAFFIC_BYTES_PER_TICK="${SCALETEST_TRAFFIC_BYTES_PER_TICK:-1024}"
20+
SCALETEST_TRAFFIC_TICK_INTERVAL="${SCALETEST_TRAFFIC_TICK_INTERVAL:-10}"
1821

1922
script_name=$(basename "$0")
20-
args="$(getopt -o "" -l dry-run,help,name:,num-workspaces:,project:,scenario:,skip-cleanup -- "$@")"
23+
args="$(getopt -o "" -l create-concurrency:,dry-run,help,name:,num-workspaces:,project:,scenario:,skip-cleanup,traffic-bytes-per-tick:,traffic-tick-interval:, -- "$@")"
2124
eval set -- "$args"
2225
while true; do
2326
case "$1" in
27+
--create-concurrency)
28+
SCALETEST_CREATE_CONCURRENCY="$2"
29+
shift 2
30+
;;
2431
--dry-run)
2532
DRY_RUN=1
2633
shift
2734
;;
2835
--help)
29-
echo "Usage: $script_name --name <name> --project <project> --num-workspaces <num-workspaces> --scenario <scenario> [--dry-run] [--skip-cleanup]"
36+
echo "Usage: $script_name --name <name> --project <project> --num-workspaces <num-workspaces> --scenario <scenario> [--dry-run] [--skip-cleanup] [--create-concurrency=<create-concurrency>]"
3037
exit 1
3138
;;
3239
--name)
@@ -49,6 +56,14 @@ while true; do
4956
SCALETEST_SKIP_CLEANUP=1
5057
shift
5158
;;
59+
--traffic-bytes-per-tick)
60+
SCALETEST_TRAFFIC_BYTES_PER_TICK="$2"
61+
shift 2
62+
;;
63+
--traffic-tick-interval)
64+
SCALETEST_TRAFFIC_TICK_INTERVAL="$2"
65+
shift 2
66+
;;
5267
--)
5368
shift
5469
break
@@ -144,16 +159,21 @@ echo "Creating ${SCALETEST_NUM_WORKSPACES} workspaces."
144159
DRY_RUN="$DRY_RUN" "${PROJECT_ROOT}/scaletest/lib/coder_shim.sh" scaletest create-workspaces \
145160
--count "${SCALETEST_NUM_WORKSPACES}" \
146161
--template=kubernetes \
147-
--concurrency 10 \
162+
--concurrency "${SCALETEST_CREATE_CONCURRENCY}" \
148163
--no-cleanup
149164

150165
echo "Sleeping 10 minutes to establish a baseline measurement."
151166
maybedryrun "$DRY_RUN" sleep 600
152167

153168
echo "Sending traffic to workspaces"
154-
maybedryrun "$DRY_RUN" "${PROJECT_ROOT}/scaletest/lib/coder_workspacetraffic.sh" "${SCALETEST_NAME}"
169+
maybedryrun "$DRY_RUN" "${PROJECT_ROOT}/scaletest/lib/coder_workspacetraffic.sh" \
170+
--name "${SCALETEST_NAME}" \
171+
--traffic-bytes-per-tick "${SCALETEST_TRAFFIC_BYTES_PER_TICK}" \
172+
--traffic-tick-interval "${SCALETEST_TRAFFIC_TICK_INTERVAL}"
155173
maybedryrun "$DRY_RUN" kubectl --kubeconfig="${KUBECONFIG}" -n "coder-${SCALETEST_NAME}" wait pods coder-scaletest-workspace-traffic --for condition=Ready
156-
maybedryrun "$DRY_RUN" kubectl --kubeconfig="${KUBECONFIG}" -n "coder-${SCALETEST_NAME}" logs -f pod/coder-scaletest-workspace-traffic
174+
175+
echo "Sleeping 15 minutes for traffic generation"
176+
maybedryrun "$DRY_RUN" sleep 900
157177

158178
echo "Starting pprof"
159179
maybedryrun "$DRY_RUN" kubectl -n "coder-${SCALETEST_NAME}" port-forward deployment/coder 6061:6060 &
@@ -168,6 +188,7 @@ while ! maybedryrun "$DRY_RUN" timeout 1 bash -c "echo > /dev/tcp/localhost/6061
168188
echo "pprof failed to become ready in time!"
169189
exit 1
170190
fi
191+
pprof_attempt_counter+=1
171192
maybedryrun "$DRY_RUN" sleep 3
172193
done
173194

‎scaletest/terraform/main.tf

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,5 +31,5 @@ terraform {
3131
}
3232
}
3333

34-
required_version = "~> 1.4.0"
34+
required_version = "~> 1.5.0"
3535
}

0 commit comments

Comments
 (0)