Skip to content

feat(scaletest/templates): run all scenarios even on failure #10290

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Oct 16, 2023
Merged

feat(scaletest/templates): run all scenarios even on failure #10290

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f9bc6a3
feat(scaletest/templates): run all scenarios even on failure
mafredri Oct 16, 2023
838b07a
add debug scenarios
mafredri Oct 16, 2023
5288898
fix assoc array count bug
mafredri Oct 16, 2023
567db6f
jq -n <3
mafredri Oct 16, 2023
01a2003
make shfmt complain less?
mafredri Oct 16, 2023
b73c2e1
update shfmt for peace
mafredri Oct 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ jobs:
go-version: 1.20.10

- name: Install shfmt
run: go install mvdan.cc/sh/v3/cmd/shfmt@v3.5.0
run: go install mvdan.cc/sh/v3/cmd/shfmt@v3.7.0

- name: make fmt
run: |
Expand Down
21 changes: 15 additions & 6 deletions scaletest/templates/scaletest-runner/scripts/lib.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ end_phase() {
log "End phase ${phase_num}: ${phase}"
echo "$(date -Ins) END:${phase_num}: ${phase}" >>"${SCALETEST_PHASE_FILE}"

GRAFANA_EXTRA_TAGS="${PHASE_TYPE:-phase-default}" annotate_grafana_end "phase" "Phase ${phase_num}: ${phase}"
GRAFANA_EXTRA_TAGS="${PHASE_TYPE:-phase-default}" GRAFANA_ADD_TAGS="${PHASE_ADD_TAGS:-}" annotate_grafana_end "phase" "Phase ${phase_num}: ${phase}"
}
get_phase() {
if [[ -f "${SCALETEST_PHASE_FILE}" ]]; then
Expand Down Expand Up @@ -183,11 +183,20 @@ annotate_grafana_end() {

log "Annotating Grafana (end=${end}): ${text} [${tags}]"

json="$(
jq \
--argjson timeEnd "${end}" \
'{timeEnd: $timeEnd}' <<<'{}'
)"
if [[ -n ${GRAFANA_ADD_TAGS:-} ]]; then
json="$(
jq -n \
--argjson timeEnd "${end}" \
--argjson tags "${tags},${GRAFANA_ADD_TAGS}" \
'{timeEnd: $timeEnd, tags: $tags | split(",")}'
)"
else
json="$(
jq -n \
--argjson timeEnd "${end}" \
'{timeEnd: $timeEnd}'
)"
fi
if [[ ${DRY_RUN} == 1 ]]; then
log "Would have patched Grafana annotation: id=${id}, data=${json}"
return 0
Expand Down
34 changes: 33 additions & 1 deletion scaletest/templates/scaletest-runner/scripts/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,12 @@ end_phase

wait_baseline "${SCALETEST_PARAM_LOAD_SCENARIO_BASELINE_DURATION}"

declare -A failed=()
for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
start_phase "Load scenario: ${scenario}"

set +e
status=0
case "${scenario}" in
"SSH Traffic")
coder exp scaletest workspace-traffic \
Expand All @@ -37,6 +41,7 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
--timeout "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_DURATION}m" \
--job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_DURATION}m30s" \
--output json:"${SCALETEST_RESULTS_DIR}/traffic-ssh.json"
status=$?
show_json "${SCALETEST_RESULTS_DIR}/traffic-ssh.json"
;;
"Web Terminal Traffic")
Expand All @@ -46,6 +51,7 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
--timeout "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_DURATION}m" \
--job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_DURATION}m30s" \
--output json:"${SCALETEST_RESULTS_DIR}/traffic-web-terminal.json"
status=$?
show_json "${SCALETEST_RESULTS_DIR}/traffic-web-terminal.json"
;;
"Dashboard Traffic")
Expand All @@ -54,13 +60,39 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
--job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_DASHBOARD_TRAFFIC_DURATION}m30s" \
--output json:"${SCALETEST_RESULTS_DIR}/traffic-dashboard.json" \
>"${SCALETEST_RESULTS_DIR}/traffic-dashboard-output.log"
status=$?
show_json "${SCALETEST_RESULTS_DIR}/traffic-dashboard.json"
;;

# Debug scenarios, for testing the runner.
"debug:success")
maybedryrun "$DRY_RUN" sleep 10
status=0
;;
"debug:error")
maybedryrun "$DRY_RUN" sleep 10
status=1
;;
Comment on lines +67 to +75
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

       _          
 _ __ (_) ___ ___ 
| '_ \| |/ __/ _ \
| | | | | (_|  __/
|_| |_|_|\___\___|

esac
end_phase
set -e
if ((status > 0)); then
log "Load scenario failed: ${scenario} (exit=${status})"
failed+=(["${scenario}"]="$status")
PHASE_ADD_TAGS=error end_phase
else
end_phase
fi

wait_baseline "${SCALETEST_PARAM_LOAD_SCENARIO_BASELINE_DURATION}"
done

if ((${#failed[@]} > 0)); then
log "Load scenarios failed: ${!failed[*]}"
for scenario in "${!failed[@]}"; do
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I didn't know you could do this :o

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup, it's the syntax for accessing associative array keys. 💪🏻 (I much prefer the Zsh way though for scenario in ${(k)failed} or even for scenario code in ${(kv)failed}. 😜)

log " ${scenario}: exit=${failed[$scenario]}"
done
exit 1
fi

log "Scaletest complete!"
set_status Complete