diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 603d2328828d8..20a7da094d14e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -223,7 +223,7 @@ jobs: go-version: 1.20.10 - name: Install shfmt - run: go install mvdan.cc/sh/v3/cmd/shfmt@v3.5.0 + run: go install mvdan.cc/sh/v3/cmd/shfmt@v3.7.0 - name: make fmt run: | diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh index 07398bc58e577..753dc1a5fefea 100644 --- a/scaletest/templates/scaletest-runner/scripts/lib.sh +++ b/scaletest/templates/scaletest-runner/scripts/lib.sh @@ -87,7 +87,7 @@ end_phase() { log "End phase ${phase_num}: ${phase}" echo "$(date -Ins) END:${phase_num}: ${phase}" >>"${SCALETEST_PHASE_FILE}" - GRAFANA_EXTRA_TAGS="${PHASE_TYPE:-phase-default}" annotate_grafana_end "phase" "Phase ${phase_num}: ${phase}" + GRAFANA_EXTRA_TAGS="${PHASE_TYPE:-phase-default}" GRAFANA_ADD_TAGS="${PHASE_ADD_TAGS:-}" annotate_grafana_end "phase" "Phase ${phase_num}: ${phase}" } get_phase() { if [[ -f "${SCALETEST_PHASE_FILE}" ]]; then @@ -183,11 +183,20 @@ annotate_grafana_end() { log "Annotating Grafana (end=${end}): ${text} [${tags}]" - json="$( - jq \ - --argjson timeEnd "${end}" \ - '{timeEnd: $timeEnd}' <<<'{}' - )" + if [[ -n ${GRAFANA_ADD_TAGS:-} ]]; then + json="$( + jq -n \ + --argjson timeEnd "${end}" \ + --argjson tags "${tags},${GRAFANA_ADD_TAGS}" \ + '{timeEnd: $timeEnd, tags: $tags | split(",")}' + )" + else + json="$( + jq -n \ + --argjson timeEnd "${end}" \ + '{timeEnd: $timeEnd}' + )" + fi if [[ ${DRY_RUN} == 1 ]]; then log "Would have patched Grafana annotation: id=${id}, data=${json}" return 0 diff --git a/scaletest/templates/scaletest-runner/scripts/run.sh b/scaletest/templates/scaletest-runner/scripts/run.sh index bba1cf2f79f23..c96995febc3d5 100755 --- a/scaletest/templates/scaletest-runner/scripts/run.sh +++ b/scaletest/templates/scaletest-runner/scripts/run.sh @@ -26,8 +26,12 @@ end_phase wait_baseline "${SCALETEST_PARAM_LOAD_SCENARIO_BASELINE_DURATION}" +declare -A failed=() for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do start_phase "Load scenario: ${scenario}" + + set +e + status=0 case "${scenario}" in "SSH Traffic") coder exp scaletest workspace-traffic \ @@ -37,6 +41,7 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do --timeout "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_DURATION}m" \ --job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_DURATION}m30s" \ --output json:"${SCALETEST_RESULTS_DIR}/traffic-ssh.json" + status=$? show_json "${SCALETEST_RESULTS_DIR}/traffic-ssh.json" ;; "Web Terminal Traffic") @@ -46,6 +51,7 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do --timeout "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_DURATION}m" \ --job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_DURATION}m30s" \ --output json:"${SCALETEST_RESULTS_DIR}/traffic-web-terminal.json" + status=$? show_json "${SCALETEST_RESULTS_DIR}/traffic-web-terminal.json" ;; "Dashboard Traffic") @@ -54,13 +60,39 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do --job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_DASHBOARD_TRAFFIC_DURATION}m30s" \ --output json:"${SCALETEST_RESULTS_DIR}/traffic-dashboard.json" \ >"${SCALETEST_RESULTS_DIR}/traffic-dashboard-output.log" + status=$? show_json "${SCALETEST_RESULTS_DIR}/traffic-dashboard.json" ;; + + # Debug scenarios, for testing the runner. + "debug:success") + maybedryrun "$DRY_RUN" sleep 10 + status=0 + ;; + "debug:error") + maybedryrun "$DRY_RUN" sleep 10 + status=1 + ;; esac - end_phase + set -e + if ((status > 0)); then + log "Load scenario failed: ${scenario} (exit=${status})" + failed+=(["${scenario}"]="$status") + PHASE_ADD_TAGS=error end_phase + else + end_phase + fi wait_baseline "${SCALETEST_PARAM_LOAD_SCENARIO_BASELINE_DURATION}" done +if ((${#failed[@]} > 0)); then + log "Load scenarios failed: ${!failed[*]}" + for scenario in "${!failed[@]}"; do + log " ${scenario}: exit=${failed[$scenario]}" + done + exit 1 +fi + log "Scaletest complete!" set_status Complete