Skip to content

Commit e300b03

Browse files
mafredrimafredri
authored
feat(scaletest): add greedy agent test to runner (#10559)
1 parent dca8125 commit e300b03

File tree

3 files changed

+153
-4
lines changed

3 files changed

+153
-4
lines changed

‎scaletest/templates/scaletest-runner/main.tf

Lines changed: 60 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,12 @@ data "coder_parameter" "workspace_template" {
198198
icon = "/emojis/1f436.png" # Dog.
199199
description = "Provisions a medium-sized workspace with no persistent storage."
200200
}
201+
option {
202+
name = "Medium (Greedy)"
203+
value = "kubernetes-medium-greedy"
204+
icon = "/emojis/1f436.png" # Dog.
205+
description = "Provisions a medium-sized workspace with no persistent storage. Greedy agent variant."
206+
}
201207
option {
202208
name = "Large"
203209
value = "kubernetes-large"
@@ -216,7 +222,7 @@ data "coder_parameter" "num_workspaces" {
216222

217223
validation {
218224
min = 0
219-
max = 1000
225+
max = 2000
220226
}
221227
}
222228

@@ -335,6 +341,56 @@ data "coder_parameter" "load_scenario_baseline_duration" {
335341
}
336342
}
337343

344+
data "coder_parameter" "greedy_agent" {
345+
order = 30
346+
type = "bool"
347+
name = "Greedy Agent"
348+
default = false
349+
description = "If true, the agent will attempt to consume all available resources."
350+
mutable = true
351+
ephemeral = true
352+
}
353+
354+
data "coder_parameter" "greedy_agent_template" {
355+
order = 31
356+
name = "Greedy Agent Template"
357+
display_name = "Greedy Agent Template"
358+
description = "The template used for the greedy agent workspace (must not be same as workspace template)."
359+
default = "kubernetes-medium-greedy"
360+
icon = "/emojis/1f4dc.png" # Scroll.
361+
mutable = true
362+
option {
363+
name = "Minimal"
364+
value = "kubernetes-minimal" # Feather.
365+
icon = "/emojis/1fab6.png"
366+
description = "Sized to fit approx. 32 per t2d-standard-8 instance."
367+
}
368+
option {
369+
name = "Small"
370+
value = "kubernetes-small"
371+
icon = "/emojis/1f42d.png" # Mouse.
372+
description = "Provisions a small-sized workspace with no persistent storage."
373+
}
374+
option {
375+
name = "Medium"
376+
value = "kubernetes-medium"
377+
icon = "/emojis/1f436.png" # Dog.
378+
description = "Provisions a medium-sized workspace with no persistent storage."
379+
}
380+
option {
381+
name = "Medium (Greedy)"
382+
value = "kubernetes-medium-greedy"
383+
icon = "/emojis/1f436.png" # Dog.
384+
description = "Provisions a medium-sized workspace with no persistent storage. Greedy agent variant."
385+
}
386+
option {
387+
name = "Large"
388+
value = "kubernetes-large"
389+
icon = "/emojis/1f434.png" # Horse.
390+
description = "Provisions a large-sized workspace with no persistent storage."
391+
}
392+
}
393+
338394
data "coder_parameter" "namespace" {
339395
order = 999
340396
type = "string"
@@ -395,6 +451,8 @@ resource "coder_agent" "main" {
395451
SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_TICK_INTERVAL : "${data.coder_parameter.load_scenario_web_terminal_tick_interval.value}",
396452
SCALETEST_PARAM_LOAD_SCENARIO_DASHBOARD_TRAFFIC_DURATION : "${data.coder_parameter.load_scenario_dashboard_traffic_duration.value}",
397453
SCALETEST_PARAM_LOAD_SCENARIO_BASELINE_DURATION : "${data.coder_parameter.load_scenario_baseline_duration.value}",
454+
SCALETEST_PARAM_GREEDY_AGENT : data.coder_parameter.greedy_agent.value ? "1" : "0",
455+
SCALETEST_PARAM_GREEDY_AGENT_TEMPLATE : data.coder_parameter.greedy_agent_template.value,
398456

399457
GRAFANA_URL : local.grafana_url,
400458

@@ -584,7 +642,7 @@ resource "kubernetes_pod" "main" {
584642
}
585643
# Set the pod delete timeout to termination_grace_period_seconds + 1m.
586644
timeouts {
587-
delete = "${(local.workspace_pod_termination_grace_period_seconds + 120) / 60}s"
645+
delete = "${(local.workspace_pod_termination_grace_period_seconds + 120)}s"
588646
}
589647
spec {
590648
security_context {

‎scaletest/templates/scaletest-runner/scripts/run.sh

Lines changed: 86 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,70 @@ end_phase
2626

2727
wait_baseline "${SCALETEST_PARAM_LOAD_SCENARIO_BASELINE_DURATION}"
2828

29+
non_greedy_agent_traffic_args=()
30+
if [[ ${SCALETEST_PARAM_GREEDY_AGENT} != 1 ]]; then
31+
greedy_agent_traffic() { :; }
32+
else
33+
echo "WARNING: Greedy agent enabled, this may cause the load tests to fail." >&2
34+
non_greedy_agent_traffic_args=(
35+
# Let the greedy agent traffic command be scraped.
36+
# --scaletest-prometheus-address 0.0.0.0:21113
37+
# --trace=false
38+
)
39+
40+
annotate_grafana greedy_agent "Create greedy agent"
41+
42+
coder exp scaletest create-workspaces \
43+
--count 1 \
44+
--template "${SCALETEST_PARAM_GREEDY_AGENT_TEMPLATE}" \
45+
--concurrency 1 \
46+
--timeout 5h \
47+
--job-timeout 5h \
48+
--no-cleanup \
49+
--output json:"${SCALETEST_RESULTS_DIR}/create-workspaces-greedy-agent.json"
50+
51+
wait_baseline "${SCALETEST_PARAM_LOAD_SCENARIO_BASELINE_DURATION}"
52+
53+
greedy_agent_traffic() {
54+
local timeout=${1} scenario=${2}
55+
# Run the greedy test for ~1/3 of the timeout.
56+
delay=$((timeout * 60 / 3))
57+
58+
local type=web-terminal
59+
args=()
60+
if [[ ${scenario} == "SSH Traffic" ]]; then
61+
type=ssh
62+
args+=(--ssh)
63+
fi
64+
65+
sleep "${delay}"
66+
annotate_grafana greedy_agent "${scenario}: Greedy agent traffic"
67+
68+
# Produce load at about 1000MB/s (25MB/40ms).
69+
set +e
70+
coder exp scaletest workspace-traffic \
71+
--template "${SCALETEST_PARAM_GREEDY_AGENT_TEMPLATE}" \
72+
--bytes-per-tick $((1024 * 1024 * 25)) \
73+
--tick-interval 40ms \
74+
--timeout "$((delay))s" \
75+
--job-timeout "$((delay))s" \
76+
--output json:"${SCALETEST_RESULTS_DIR}/traffic-${type}-greedy-agent.json" \
77+
--scaletest-prometheus-address 0.0.0.0:21113 \
78+
--trace=false \
79+
"${args[@]}"
80+
status=${?}
81+
show_json "${SCALETEST_RESULTS_DIR}/traffic-${type}-greedy-agent.json"
82+
83+
export GRAFANA_ADD_TAGS=
84+
if [[ ${status} != 0 ]]; then
85+
GRAFANA_ADD_TAGS=error
86+
fi
87+
annotate_grafana_end greedy_agent "${scenario}: Greedy agent traffic"
88+
89+
return ${status}
90+
}
91+
fi
92+
2993
declare -A failed=()
3094
for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
3195
start_phase "Load scenario: ${scenario}"
@@ -34,24 +98,40 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
3498
status=0
3599
case "${scenario}" in
36100
"SSH Traffic")
101+
greedy_agent_traffic "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_DURATION}" "${scenario}" &
37102
coder exp scaletest workspace-traffic \
103+
--template "${SCALETEST_PARAM_TEMPLATE}" \
38104
--ssh \
39105
--bytes-per-tick "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_BYTES_PER_TICK}" \
40106
--tick-interval "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_TICK_INTERVAL}ms" \
41107
--timeout "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_DURATION}m" \
42108
--job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_DURATION}m30s" \
43-
--output json:"${SCALETEST_RESULTS_DIR}/traffic-ssh.json"
109+
--output json:"${SCALETEST_RESULTS_DIR}/traffic-ssh.json" \
110+
"${non_greedy_agent_traffic_args[@]}"
44111
status=$?
112+
wait
113+
status2=$?
114+
if [[ ${status} == 0 ]]; then
115+
status=${status2}
116+
fi
45117
show_json "${SCALETEST_RESULTS_DIR}/traffic-ssh.json"
46118
;;
47119
"Web Terminal Traffic")
120+
greedy_agent_traffic "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_DURATION}" "${scenario}" &
48121
coder exp scaletest workspace-traffic \
122+
--template "${SCALETEST_PARAM_TEMPLATE}" \
49123
--bytes-per-tick "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_BYTES_PER_TICK}" \
50124
--tick-interval "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_TICK_INTERVAL}ms" \
51125
--timeout "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_DURATION}m" \
52126
--job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_DURATION}m30s" \
53-
--output json:"${SCALETEST_RESULTS_DIR}/traffic-web-terminal.json"
127+
--output json:"${SCALETEST_RESULTS_DIR}/traffic-web-terminal.json" \
128+
"${non_greedy_agent_traffic_args[@]}"
54129
status=$?
130+
wait
131+
status2=$?
132+
if [[ ${status} == 0 ]]; then
133+
status=${status2}
134+
fi
55135
show_json "${SCALETEST_RESULTS_DIR}/traffic-web-terminal.json"
56136
;;
57137
"Dashboard Traffic")
@@ -65,6 +145,10 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
65145
;;
66146

67147
# Debug scenarios, for testing the runner.
148+
"debug:greedy_agent_traffic")
149+
greedy_agent_traffic 10 "${scenario}"
150+
status=$?
151+
;;
68152
"debug:success")
69153
maybedryrun "$DRY_RUN" sleep 10
70154
status=0

‎scaletest/templates/scaletest-runner/startup.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,20 @@ set -euo pipefail
33

44
[[ $VERBOSE == 1 ]] && set -x
55

6+
if [[ ${SCALETEST_PARAM_GREEDY_AGENT_TEMPLATE} == "${SCALETEST_PARAM_TEMPLATE}" ]]; then
7+
echo "ERROR: Greedy agent template must be different from the scaletest template." >&2
8+
exit 1
9+
fi
10+
611
# Unzip scripts and add to path.
712
# shellcheck disable=SC2153
813
echo "Extracting scaletest scripts into ${SCRIPTS_DIR}..."
914
base64 -d <<<"${SCRIPTS_ZIP}" >/tmp/scripts.zip
1015
rm -rf "${SCRIPTS_DIR}" || true
1116
mkdir -p "${SCRIPTS_DIR}"
1217
unzip -o /tmp/scripts.zip -d "${SCRIPTS_DIR}"
18+
# Chmod to work around https://github.com/coder/coder/issues/10034
19+
chmod +x "${SCRIPTS_DIR}"/*.sh
1320
rm /tmp/scripts.zip
1421

1522
echo "Cloning coder/coder repo..."

0 commit comments

Comments
 (0)