Skip to content

Commit 569cfc1

Browse files
mafredrimafredri
committed
allow staggered start
1 parent 96100c0 commit 569cfc1

File tree

2 files changed

+40
-4
lines changed

2 files changed

+40
-4
lines changed

‎scaletest/templates/scaletest-runner/main.tf

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,15 @@ data "coder_parameter" "load_scenario_run_concurrently" {
259259
mutable = true
260260
}
261261

262+
data "coder_parameter" "load_scenario_concurrency_staggering" {
263+
order = 23
264+
name = "Load Scenario Concurrency Staggering"
265+
type = "number"
266+
default = 3
267+
description = "The number of minutes to wait between starting each load scenario when run concurrently."
268+
mutable = true
269+
}
270+
262271
data "coder_parameter" "load_scenario_ssh_traffic_duration" {
263272
order = 30
264273
name = "SSH Traffic Duration"
@@ -576,6 +585,7 @@ resource "coder_agent" "main" {
576585
SCALETEST_PARAM_CLEANUP_PREPARE : data.coder_parameter.cleanup_prepare.value ? "1" : "0",
577586
SCALETEST_PARAM_LOAD_SCENARIOS : data.coder_parameter.load_scenarios.value,
578587
SCALETEST_PARAM_LOAD_SCENARIO_RUN_CONCURRENTLY : data.coder_parameter.load_scenario_run_concurrently.value ? "1" : "0",
588+
SCALETEST_PARAM_LOAD_SCENARIO_CONCURRENCY_STAGGERING : "${data.coder_parameter.load_scenario_concurrency_staggering.value}",
579589
SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_DURATION : "${data.coder_parameter.load_scenario_ssh_traffic_duration.value}",
580590
SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_BYTES_PER_TICK : "${data.coder_parameter.load_scenario_ssh_bytes_per_tick.value}",
581591
SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_TICK_INTERVAL : "${data.coder_parameter.load_scenario_ssh_tick_interval.value}",

‎scaletest/templates/scaletest-runner/scripts/run.sh

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -94,10 +94,32 @@ if [[ ${SCALETEST_PARAM_LOAD_SCENARIO_RUN_CONCURRENTLY} == 1 ]]; then
9494
start_phase "Load scenarios: ${SCALETEST_PARAM_LOAD_SCENARIOS[*]}"
9595
fi
9696

97+
run_scenario_cmd() {
98+
local scenario=${1}
99+
shift
100+
local command=("$@")
101+
102+
set +e
103+
if [[ ${SCALETEST_PARAM_LOAD_SCENARIO_RUN_CONCURRENTLY} == 1 ]]; then
104+
annotate_grafana scenario "Load scenario: ${scenario}"
105+
fi
106+
"${command[@]}"
107+
status=${?}
108+
if [[ ${SCALETEST_PARAM_LOAD_SCENARIO_RUN_CONCURRENTLY} == 1 ]]; then
109+
export GRAFANA_ADD_TAGS=
110+
if [[ ${status} != 0 ]]; then
111+
GRAFANA_ADD_TAGS=error
112+
fi
113+
annotate_grafana_end scenario "Load scenario: ${scenario}"
114+
fi
115+
exit "${status}"
116+
}
117+
97118
declare -a pids=()
98119
declare -A failed=()
99120
target_start=0
100121
target_end=-1
122+
101123
for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
102124
if [[ ${SCALETEST_PARAM_LOAD_SCENARIO_RUN_CONCURRENTLY} == 0 ]]; then
103125
start_phase "Load scenario: ${scenario}"
@@ -117,7 +139,7 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
117139
target_start=0
118140
target_end=${target_count}
119141
fi
120-
coder exp scaletest workspace-traffic \
142+
run_scenario_cmd "${scenario}" coder exp scaletest workspace-traffic \
121143
--template "${SCALETEST_PARAM_TEMPLATE}" \
122144
--ssh \
123145
--bytes-per-tick "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_BYTES_PER_TICK}" \
@@ -153,7 +175,7 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
153175
target_start=0
154176
target_end=${target_count}
155177
fi
156-
coder exp scaletest workspace-traffic \
178+
run_scenario_cmd "${scenario}" coder exp scaletest workspace-traffic \
157179
--template "${SCALETEST_PARAM_TEMPLATE}" \
158180
--bytes-per-tick "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_BYTES_PER_TICK}" \
159181
--tick-interval "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_TICK_INTERVAL}ms" \
@@ -188,7 +210,7 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
188210
target_start=0
189211
target_end=${target_count}
190212
fi
191-
coder exp scaletest workspace-traffic \
213+
run_scenario_cmd "${scenario}" coder exp scaletest workspace-traffic \
192214
--template "${SCALETEST_PARAM_TEMPLATE}" \
193215
--bytes-per-tick "${SCALETEST_PARAM_LOAD_SCENARIO_APP_TRAFFIC_BYTES_PER_TICK}" \
194216
--tick-interval "${SCALETEST_PARAM_LOAD_SCENARIO_APP_TRAFFIC_TICK_INTERVAL}ms" \
@@ -221,7 +243,7 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
221243
target_start=0
222244
target_end=${target_count}
223245
fi
224-
coder exp scaletest dashboard \
246+
run_scenario_cmd "${scenario}" coder exp scaletest dashboard \
225247
--timeout "${SCALETEST_PARAM_LOAD_SCENARIO_DASHBOARD_TRAFFIC_DURATION}m" \
226248
--job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_DASHBOARD_TRAFFIC_DURATION}m30s" \
227249
--output json:"${SCALETEST_RESULTS_DIR}/traffic-dashboard.json" \
@@ -290,6 +312,10 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
290312
fi
291313

292314
wait_baseline "${SCALETEST_PARAM_LOAD_SCENARIO_BASELINE_DURATION}"
315+
else
316+
# Stagger the start of each scenario to avoid a burst of load and deted
317+
# problematic scenarios.
318+
sleep $((SCALETEST_PARAM_LOAD_SCENARIO_CONCURRENCY_STAGGERING * 60))
293319
fi
294320
done
295321

0 commit comments

Comments
 (0)