refactor phase, create lib

coder · mafredri · Sep 15, 2023 · Sep 13, 2023 · Sep 13, 2023 · Sep 13, 2023
commit dac2ff8d4f103b0152e52a2dcfbaedeaf2c1e963
diff --git a/scaletest/templates/scaletest-runner/metadata_phase.sh b/scaletest/templates/scaletest-runner/metadata_phase.sh
@@ -1,15 +1,15 @@
 #!/bin/bash
 
-if [ -f /tmp/.scaletest_phase_creating_workspaces ]; then
-	echo "Creating workspaces"
-elif [ -f /tmp/.scaletest_phase_ssh ]; then
-	echo "SSH traffic"
-elif [ -f /tmp/.scaletest_phase_rpty ]; then
-	echo "RPTY traffic"
-elif [ -f /tmp/.scaletest_phase_dashboard ]; then
-	echo "Dashboard traffic"
-elif [ -f /tmp/.scaletest_phase_wait_baseline ]; then
-	echo "Waiting $(</tmp/.scaletest_phase_wait_baseline)m (establishing baseline)"
+# shellcheck disable=SC2153 source=scaletest/templates/scaletest-runner/scripts/lib.sh
+. "${SCRIPTS_DIR}/lib.sh"
+
+if [[ -f "${SCALETEST_PHASE_FILE}" ]]; then
+	phase_raw="$(tail -n1 "${SCALETEST_PHASE_FILE}")"
+	phase="$(echo "${phase_raw}" | cut -d' ' -f3-)"
+	if [[ ${phase_raw} == *"END:"* ]]; then
+		phase+=" (done)"
+	fi
+	echo "${phase}"
 else
 	echo "None"
 fi
diff --git a/scaletest/templates/scaletest-runner/metadata_status.sh b/scaletest/templates/scaletest-runner/metadata_status.sh
@@ -1,12 +1,13 @@
 #!/bin/bash
 
-if [ -f /tmp/.scaletest_failed ]; then
+# Order of importance (reverse of creation).
+if [[ -f /tmp/.scaletest_failed ]]; then
 	echo "Failed"
-elif [ -f /tmp/.scaletest_complete ]; then
+elif [[ -f /tmp/.scaletest_complete ]]; then
 	echo "Complete"
-elif [ -f /tmp/.scaletest_running ]; then
+elif [[ -f /tmp/.scaletest_running ]]; then
 	echo "Running"
-elif [ -f /tmp/.scaletest_preparing ]; then
+elif [[ -f /tmp/.scaletest_preparing ]]; then
 	echo "Preparing"
 else
 	echo "Not started"

diff --git a/scaletest/templates/scaletest-runner/scripts/cleanup.sh b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
@@ -3,8 +3,8 @@ set -euo pipefail
 
 [[ $VERBOSE == 1 ]] && set -x
 
-# shellcheck source=scripts/lib.sh
-. ~/coder/scripts/lib.sh
+# shellcheck disable=SC2153 source=scaletest/templates/scaletest-runner/scripts/lib.sh
+. "${SCRIPTS_DIR}/lib.sh"
 
 event=${1:-}
 
@@ -21,15 +21,13 @@ if [[ $event = manual ]]; then
 	fi
 fi
 
-echo "Cleaning up scaletest resources (${event})..."
-
-maybedryrun "${DRY_RUN}" coder exp scaletest cleanup \
+phase_start "Cleanup (${event})"
+coder exp scaletest cleanup \
 	--cleanup-concurrency "${SCALETEST_CLEANUP_CONCURRENCY}" \
 	--cleanup-job-timeout 15m \
-	--cleanup-timeout 30m |
-	tee "result-cleanup-${event}.txt"
-
-echo "Cleanup complete!"
+	--cleanup-timeout 30m \
+	| tee "result-cleanup-${event}.txt"
+end_phase
 
 if [[ $event = manual ]]; then
 	echo 'Press any key to continue...'

diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+set -euo pipefail
+
+# Only source this script once, this env comes from sourcing
+# scripts/lib.sh from coder/coder below.
+if [[ ${SCRIPTS_LIB_IS_SOURCED} == 1 ]]; then
+	return 0
+fi
+
+# Source scripts/lib.sh from coder/coder for common functions.
+# shellcheck source=scripts/lib.sh
+. ~/coder/scripts/lib.sh
+
+# Environment variables shared between scripts.
+SCALETEST_PHASE_FILE=/tmp/.scaletest_phase
+
+coder() {
+	maybedryrun "$DRY_RUN" command coder "${@}"
+}
+
+show_json() {
+	maybedryrun "$DRY_RUN" jq 'del(.. | .logs?)' "${1}"
+}
+
+phase_num=0
+start_phase() {
+	((phase_num++))
+	log "Start phase ${phase_num}: ${*}"
+	echo "$(date -Iseconds) START:${phase_num}: ${*}" >>"${SCALETEST_PHASE_FILE}"
+}
+end_phase() {
+	phase="$(tail -n 1 "${SCALETEST_PHASE_FILE}" | grep "START:${phase_num}:" | cut -d' ' -f3-)"
+	if [[ -z ${phase} ]]; then
+		log "BUG: Could not find start phase ${phase_num} in ${SCALETEST_PHASE_FILE}"
+		exit 1
+	fi
+	log "End phase ${phase_num}: ${phase}"
+	echo "$(date -Iseconds) END:${phase_num}: ${phase}" >>"${SCALETEST_PHASE_FILE}"
+}
+
+wait_baseline() {
+	s=${1:-2}
+	start_phase "Waiting ${s}m to establish baseline"
+	sleep $((s * 60))
+	end_phase
+}
diff --git a/scaletest/templates/scaletest-runner/scripts/run.sh b/scaletest/templates/scaletest-runner/scripts/run.sh
@@ -3,30 +3,13 @@ set -euo pipefail
 
 [[ $VERBOSE == 1 ]] && set -x
 
-# shellcheck source=scripts/lib.sh
-. ~/coder/scripts/lib.sh
+# shellcheck disable=SC2153 source=scaletest/templates/scaletest-runner/scripts/lib.sh
+. "${SCRIPTS_DIR}/lib.sh"
 
-coder() {
-	maybedryrun "$DRY_RUN" command coder "${@}"
-}
-
-show_json() {
-	maybedryrun "$DRY_RUN" jq 'del(.. | .logs?)' "${1}"
-}
-
-wait_baseline() {
-	s=${1:-2}
-	echo "Waiting ${s}m (establishing baseline)..."
-	echo "${s}" >/tmp/.scaletest_phase_wait_baseline
-	sleep $((s * 60))
-	rm /tmp/.scaletest_phase_wait_baseline
-}
-
-echo "Running scaletest..."
+log "Running scaletest..."
 touch /tmp/.scaletest_running
 
-touch /tmp/.scaletest_phase_creating_workspaces
-
+start_phase "Creating workspaces"
 coder exp scaletest create-workspaces \
 	--count "${SCALETEST_NUM_WORKSPACES}" \
 	--template "${SCALETEST_TEMPLATE}" \
@@ -35,42 +18,42 @@ coder exp scaletest create-workspaces \
 	--no-cleanup \
 	--output json:"${SCALETEST_RUN_DIR}/result-create-workspaces.json"
 show_json "${SCALETEST_RUN_DIR}/result-create-workspaces.json"
-rm /tmp/.scaletest_phase_creating_workspaces
+end_phase
 
 wait_baseline 5
 
-touch /tmp/.scaletest_phase_ssh
+start_phase "SSH traffic"
 coder exp scaletest workspace-traffic \
 	--ssh \
 	--bytes-per-tick 10240 \
 	--tick-interval 1s \
 	--timeout 5m \
 	--output json:"${SCALETEST_RUN_DIR}/result-ssh.json"
 show_json "${SCALETEST_RUN_DIR}/result-ssh.json"
-rm /tmp/.scaletest_phase_ssh
+end_phase
 
 wait_baseline 5
 
-touch /tmp/.scaletest_phase_rpty
+start_phase "ReconnectingPTY traffic"
 coder exp scaletest workspace-traffic \
 	--bytes-per-tick 10240 \
 	--tick-interval 1s \
 	--timeout 5m \
-	--output json:"${SCALETEST_RUN_DIR}/result-rpty.json"
-show_json "${SCALETEST_RUN_DIR}/result-rpty.json"
-rm /tmp/.scaletest_phase_rpty
+	--output json:"${SCALETEST_RUN_DIR}/result-reconnectingpty.json"
+show_json "${SCALETEST_RUN_DIR}/result-reconnectingpty.json"
+end_phase
 
 wait_baseline 5
 
-touch /tmp/.scaletest_phase_dashboard
+start_phase "Dashboard traffic"
 coder exp scaletest dashboard \
 	--count "${SCALETEST_NUM_WORKSPACES}" \
 	--job-timeout 5m \
 	--output json:"${SCALETEST_RUN_DIR}/result-dashboard.json"
 show_json "${SCALETEST_RUN_DIR}/result-dashboard.json"
-rm /tmp/.scaletest_phase_dashboard
+end_phase
 
 wait_baseline 5
 
-echo "Scaletest complete!"
+log "Scaletest complete!"
 touch /tmp/.scaletest_complete
diff --git a/scaletest/templates/scaletest-runner/shutdown.sh b/scaletest/templates/scaletest-runner/shutdown.sh
@@ -1,6 +1,11 @@
 #!/bin/bash
 set -e
 
+[[ $VERBOSE == 1 ]] && set -x
+
+# shellcheck disable=SC2153 source=scaletest/templates/scaletest-runner/scripts/lib.sh
+. "${SCRIPTS_DIR}/lib.sh"
+
 cleanup() {
 	coder tokens remove scaletest_runner >/dev/null 2>&1 || true
 }

diff --git a/scaletest/templates/scaletest-runner/startup.sh b/scaletest/templates/scaletest-runner/startup.sh
@@ -4,13 +4,17 @@ set -euo pipefail
 [[ $VERBOSE == 1 ]] && set -x
 
 # Unzip scripts and add to path.
+# shellcheck disable=SC2153
 echo "Extracting scaletest scripts into ${SCRIPTS_DIR}..."
 base64 -d <<<"${SCRIPTS_ZIP}" >/tmp/scripts.zip
 rm -rf "${SCRIPTS_DIR}" || true
 mkdir -p "${SCRIPTS_DIR}"
 unzip -o /tmp/scripts.zip -d "${SCRIPTS_DIR}"
 rm /tmp/scripts.zip
 
+# shellcheck disable=SC2153 source=scaletest/templates/scaletest-runner/scripts/lib.sh
+. "${SCRIPTS_DIR}/lib.sh"
+
 # Clean up any previous scaletest runs (in case /tmp persists).
 rm -rf /tmp/.scaletest_* || true