From f9bc6a3918df96d569612c8eabc2cd4556253293 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Mon, 16 Oct 2023 14:04:27 +0000 Subject: [PATCH 1/6] feat(scaletest/templates): run all scenarios even on failure We also end all failed phases and add the `error` tag in Grafana. --- .../templates/scaletest-runner/scripts/lib.sh | 21 +++++++++++----- .../templates/scaletest-runner/scripts/run.sh | 24 ++++++++++++++++++- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh index 07398bc58e577..0eed9bd036072 100644 --- a/scaletest/templates/scaletest-runner/scripts/lib.sh +++ b/scaletest/templates/scaletest-runner/scripts/lib.sh @@ -87,7 +87,7 @@ end_phase() { log "End phase ${phase_num}: ${phase}" echo "$(date -Ins) END:${phase_num}: ${phase}" >>"${SCALETEST_PHASE_FILE}" - GRAFANA_EXTRA_TAGS="${PHASE_TYPE:-phase-default}" annotate_grafana_end "phase" "Phase ${phase_num}: ${phase}" + GRAFANA_EXTRA_TAGS="${PHASE_TYPE:-phase-default}" GRAFANA_ADD_TAGS="${PHASE_ADD_TAGS:-}" annotate_grafana_end "phase" "Phase ${phase_num}: ${phase}" } get_phase() { if [[ -f "${SCALETEST_PHASE_FILE}" ]]; then @@ -183,11 +183,20 @@ annotate_grafana_end() { log "Annotating Grafana (end=${end}): ${text} [${tags}]" - json="$( - jq \ - --argjson timeEnd "${end}" \ - '{timeEnd: $timeEnd}' <<<'{}' - )" + if [[ -n ${GRAFANA_ADD_TAGS:-} ]]; then + json="$( + jq \ + --argjson timeEnd "${end}" \ + --argjson tags "${tags},${GRAFANA_ADD_TAGS}" \ + '{timeEnd: $timeEnd, tags: $tags | split(",")}' <<<'{}' + )" + else + json="$( + jq \ + --argjson timeEnd "${end}" \ + '{timeEnd: $timeEnd}' <<<'{}' + )" + fi if [[ ${DRY_RUN} == 1 ]]; then log "Would have patched Grafana annotation: id=${id}, data=${json}" return 0 diff --git a/scaletest/templates/scaletest-runner/scripts/run.sh b/scaletest/templates/scaletest-runner/scripts/run.sh index bba1cf2f79f23..41eaa68d52a48 100755 --- a/scaletest/templates/scaletest-runner/scripts/run.sh +++ b/scaletest/templates/scaletest-runner/scripts/run.sh @@ -26,8 +26,12 @@ end_phase wait_baseline "${SCALETEST_PARAM_LOAD_SCENARIO_BASELINE_DURATION}" +declare -A failed=() for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do start_phase "Load scenario: ${scenario}" + + set +e + status=0 case "${scenario}" in "SSH Traffic") coder exp scaletest workspace-traffic \ @@ -37,6 +41,7 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do --timeout "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_DURATION}m" \ --job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_DURATION}m30s" \ --output json:"${SCALETEST_RESULTS_DIR}/traffic-ssh.json" + status=$? show_json "${SCALETEST_RESULTS_DIR}/traffic-ssh.json" ;; "Web Terminal Traffic") @@ -46,6 +51,7 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do --timeout "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_DURATION}m" \ --job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_DURATION}m30s" \ --output json:"${SCALETEST_RESULTS_DIR}/traffic-web-terminal.json" + status=$? show_json "${SCALETEST_RESULTS_DIR}/traffic-web-terminal.json" ;; "Dashboard Traffic") @@ -54,13 +60,29 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do --job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_DASHBOARD_TRAFFIC_DURATION}m30s" \ --output json:"${SCALETEST_RESULTS_DIR}/traffic-dashboard.json" \ >"${SCALETEST_RESULTS_DIR}/traffic-dashboard-output.log" + status=$? show_json "${SCALETEST_RESULTS_DIR}/traffic-dashboard.json" ;; esac - end_phase + set -e + if (( status > 0 )); then + log "Load scenario failed: ${scenario} (exit=${status})" + failed+=(["${scenario}"]="$status") + PHASE_ADD_TAGS=error end_phase + else + end_phase + fi wait_baseline "${SCALETEST_PARAM_LOAD_SCENARIO_BASELINE_DURATION}" done +if ((${#failed} > 0)); then + log "Load scenarios failed: ${!failed[*]}" + for scenario in "${!failed[@]}"; do + log " ${scenario}: exit=${failed[$scenario]}" + done + exit 1 +fi + log "Scaletest complete!" set_status Complete From 838b07a2237596edc1c681e4602da678ac164065 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Mon, 16 Oct 2023 14:06:55 +0000 Subject: [PATCH 2/6] add debug scenarios --- scaletest/templates/scaletest-runner/scripts/run.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scaletest/templates/scaletest-runner/scripts/run.sh b/scaletest/templates/scaletest-runner/scripts/run.sh index 41eaa68d52a48..ccec6342a7733 100755 --- a/scaletest/templates/scaletest-runner/scripts/run.sh +++ b/scaletest/templates/scaletest-runner/scripts/run.sh @@ -63,9 +63,18 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do status=$? show_json "${SCALETEST_RESULTS_DIR}/traffic-dashboard.json" ;; + # Debug scenarios, for testing the runner. + "debug:success") + maybedryrun "$DRY_RUN" sleep 10 + status=0 + ;; + "debug:error") + maybedryrun "$DRY_RUN" sleep 10 + status=1 + ;; esac set -e - if (( status > 0 )); then + if ((status > 0)); then log "Load scenario failed: ${scenario} (exit=${status})" failed+=(["${scenario}"]="$status") PHASE_ADD_TAGS=error end_phase From 5288898053f94afe180bc9d3097a1a31631a82a0 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Mon, 16 Oct 2023 14:33:08 +0000 Subject: [PATCH 3/6] fix assoc array count bug --- scaletest/templates/scaletest-runner/scripts/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scaletest/templates/scaletest-runner/scripts/run.sh b/scaletest/templates/scaletest-runner/scripts/run.sh index ccec6342a7733..37161c8dc52bf 100755 --- a/scaletest/templates/scaletest-runner/scripts/run.sh +++ b/scaletest/templates/scaletest-runner/scripts/run.sh @@ -85,7 +85,7 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do wait_baseline "${SCALETEST_PARAM_LOAD_SCENARIO_BASELINE_DURATION}" done -if ((${#failed} > 0)); then +if ((${#failed[@]} > 0)); then log "Load scenarios failed: ${!failed[*]}" for scenario in "${!failed[@]}"; do log " ${scenario}: exit=${failed[$scenario]}" From 567db6fd0b9c2e4b75f3f70b0e83b3c75dc544e6 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Mon, 16 Oct 2023 14:33:49 +0000 Subject: [PATCH 4/6] jq -n <3 --- scaletest/templates/scaletest-runner/scripts/lib.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh index 0eed9bd036072..753dc1a5fefea 100644 --- a/scaletest/templates/scaletest-runner/scripts/lib.sh +++ b/scaletest/templates/scaletest-runner/scripts/lib.sh @@ -185,16 +185,16 @@ annotate_grafana_end() { if [[ -n ${GRAFANA_ADD_TAGS:-} ]]; then json="$( - jq \ + jq -n \ --argjson timeEnd "${end}" \ --argjson tags "${tags},${GRAFANA_ADD_TAGS}" \ - '{timeEnd: $timeEnd, tags: $tags | split(",")}' <<<'{}' + '{timeEnd: $timeEnd, tags: $tags | split(",")}' )" else json="$( - jq \ + jq -n \ --argjson timeEnd "${end}" \ - '{timeEnd: $timeEnd}' <<<'{}' + '{timeEnd: $timeEnd}' )" fi if [[ ${DRY_RUN} == 1 ]]; then From 01a2003753736d1aeeb05ee412ab29b83bdd8b5b Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Mon, 16 Oct 2023 14:45:18 +0000 Subject: [PATCH 5/6] make shfmt complain less? --- scaletest/templates/scaletest-runner/scripts/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scaletest/templates/scaletest-runner/scripts/run.sh b/scaletest/templates/scaletest-runner/scripts/run.sh index 37161c8dc52bf..c96995febc3d5 100755 --- a/scaletest/templates/scaletest-runner/scripts/run.sh +++ b/scaletest/templates/scaletest-runner/scripts/run.sh @@ -63,6 +63,7 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do status=$? show_json "${SCALETEST_RESULTS_DIR}/traffic-dashboard.json" ;; + # Debug scenarios, for testing the runner. "debug:success") maybedryrun "$DRY_RUN" sleep 10 From b73c2e1506615718b52883f69f70cd166adf6e0b Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Mon, 16 Oct 2023 14:56:20 +0000 Subject: [PATCH 6/6] update shfmt for peace --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 603d2328828d8..20a7da094d14e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -223,7 +223,7 @@ jobs: go-version: 1.20.10 - name: Install shfmt - run: go install mvdan.cc/sh/v3/cmd/shfmt@v3.5.0 + run: go install mvdan.cc/sh/v3/cmd/shfmt@v3.7.0 - name: make fmt run: |