Skip to content

Commit 8f39ec5

Browse files
mafredrimafredri
authored
fix(scaletest/templates): fix bugs and improve debugging (#10316)
1 parent 2f4ca0f commit 8f39ec5

File tree

3 files changed

+24
-17
lines changed

3 files changed

+24
-17
lines changed

‎scaletest/templates/scaletest-runner/scripts/lib.sh

+9-13
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ end_phase() {
8282
phase=$(tail -n 1 "${SCALETEST_PHASE_FILE}" | grep "START:${phase_num}:" | cut -d' ' -f3-)
8383
if [[ -z ${phase} ]]; then
8484
log "BUG: Could not find start phase ${phase_num} in ${SCALETEST_PHASE_FILE}"
85-
exit 1
85+
return 1
8686
fi
8787
log "End phase ${phase_num}: ${phase}"
8888
echo "$(date -Ins) END:${phase_num}: ${phase}" >>"${SCALETEST_PHASE_FILE}"
@@ -132,6 +132,7 @@ annotate_grafana() {
132132
'{time: $time, tags: $tags | split(","), text: $text}' <<<'{}'
133133
)"
134134
if [[ ${DRY_RUN} == 1 ]]; then
135+
echo "FAKEID:${tags}:${text}:${start}" >>"${SCALETEST_STATE_DIR}/grafana-annotations"
135136
log "Would have annotated Grafana, data=${json}"
136137
return 0
137138
fi
@@ -171,23 +172,18 @@ annotate_grafana_end() {
171172
tags="${tags},${GRAFANA_EXTRA_TAGS}"
172173
fi
173174

174-
if [[ ${DRY_RUN} == 1 ]]; then
175-
log "Would have updated Grafana annotation (end=${end}): ${text} [${tags}]"
176-
return 0
177-
fi
178-
179175
if ! id=$(grep ":${tags}:${text}:${start}" "${SCALETEST_STATE_DIR}/grafana-annotations" | sort -n | tail -n1 | cut -d: -f1); then
180176
log "NOTICE: Could not find Grafana annotation to end: '${tags}:${text}:${start}', skipping..."
181177
return 0
182178
fi
183179

184-
log "Annotating Grafana (end=${end}): ${text} [${tags}]"
180+
log "Updating Grafana annotation (end=${end}): ${text} [${tags}, add=${GRAFANA_ADD_TAGS:-}]"
185181

186182
if [[ -n ${GRAFANA_ADD_TAGS:-} ]]; then
187183
json="$(
188184
jq -n \
189185
--argjson timeEnd "${end}" \
190-
--argjson tags "${tags},${GRAFANA_ADD_TAGS}" \
186+
--arg tags "${tags},${GRAFANA_ADD_TAGS}" \
191187
'{timeEnd: $timeEnd, tags: $tags | split(",")}'
192188
)"
193189
else
@@ -275,7 +271,7 @@ coder_pods() {
275271
fetch_coder_full() {
276272
if [[ -x "${SCALETEST_CODER_BINARY}" ]]; then
277273
log "Full Coder binary already exists at ${SCALETEST_CODER_BINARY}"
278-
return
274+
return 0
279275
fi
280276
ns=$(namespace)
281277
if [[ -z "${ns}" ]]; then
@@ -286,12 +282,12 @@ fetch_coder_full() {
286282
pods=$(coder_pods)
287283
if [[ -z ${pods} ]]; then
288284
log "Could not find coder pods!"
289-
return
285+
return 1
290286
fi
291287
pod=$(cut -d ' ' -f 1 <<<"${pods}")
292288
if [[ -z ${pod} ]]; then
293289
log "Could not find coder pod!"
294-
return
290+
return 1
295291
fi
296292
log "Fetching full Coder binary from ${pod}"
297293
# We need --retries due to https://github.com/kubernetes/kubernetes/issues/60140 :(
@@ -309,8 +305,8 @@ fetch_coder_full() {
309305
# com.coder.scaletest.status. It will overwrite the previous status.
310306
set_pod_status_annotation() {
311307
if [[ $# -ne 1 ]]; then
312-
log "must specify an annotation value"
313-
return
308+
log "BUG: Must specify an annotation value"
309+
return 1
314310
else
315311
maybedryrun "${DRY_RUN}" kubectl --namespace "$(namespace)" annotate pod "$(hostname)" "com.coder.scaletest.status=$1" --overwrite
316312
fi

‎scaletest/templates/scaletest-runner/scripts/run.sh

+4
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,10 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
7373
maybedryrun "$DRY_RUN" sleep 10
7474
status=1
7575
;;
76+
77+
*)
78+
log "WARNING: Unknown load scenario: ${scenario}, skipping..."
79+
;;
7680
esac
7781
set -e
7882
if ((status > 0)); then

‎scaletest/templates/scaletest-runner/startup.sh

+11-4
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,11 @@ annotate_grafana "workspace" "Agent running" # Ended in shutdown.sh.
4747
trap 'trap - EXIT; kill -INT "${pids[@]}"; exit 1' INT EXIT
4848

4949
while :; do
50-
sleep 285 # ~300 when accounting for profile and trace.
50+
# Sleep for short periods of time so that we can exit quickly.
51+
# This adds up to ~300 when accounting for profile and trace.
52+
for ((i = 0; i < 285; i++)); do
53+
sleep 1
54+
done
5155
log "Grabbing pprof dumps"
5256
start="$(date +%s)"
5357
annotate_grafana "pprof" "Grab pprof dumps (start=${start})"
@@ -74,13 +78,13 @@ gather_logs() {
7478
annotate_grafana "logs" "Gather logs"
7579
podsraw="$(
7680
kubectl -n coder-big get pods -l app.kubernetes.io/name=coder -o name
77-
kubectl -n coder-big get pods -l app.kubernetes.io/name=coder-provisioner -o name
78-
kubectl -n coder-big get pods -l app.kubernetes.io/name=coder-workspace -o name | grep "^pod/scaletest-"
81+
kubectl -n coder-big get pods -l app.kubernetes.io/name=coder-provisioner -o name || true
82+
kubectl -n coder-big get pods -l app.kubernetes.io/name=coder-workspace -o name | grep "^pod/scaletest-" || true
7983
)"
8084
mapfile -t pods <<<"${podsraw}"
8185
for pod in "${pods[@]}"; do
8286
pod_name="${pod#pod/}"
83-
kubectl -n coder-big logs "${pod}" --since="${SCALETEST_RUN_START_TIME}" >"${SCALETEST_LOGS_DIR}/${pod_name}.txt"
87+
kubectl -n coder-big logs "${pod}" --since-time="${SCALETEST_RUN_START_TIME}" >"${SCALETEST_LOGS_DIR}/${pod_name}.txt"
8488
done
8589
annotate_grafana_end "logs" "Gather logs"
8690
}
@@ -131,6 +135,9 @@ on_exit() {
131135
set_appearance "${appearance_json}" "${message_color}" "${service_banner_message} | Scaletest ${message_status}: [${CODER_USER}/${CODER_WORKSPACE}](${CODER_URL}/@${CODER_USER}/${CODER_WORKSPACE})!"
132136

133137
annotate_grafana_end "" "Start scaletest: ${SCALETEST_COMMENT}"
138+
139+
wait "${pprof_pid}"
140+
exit "${code}"
134141
}
135142
trap on_exit EXIT
136143

0 commit comments

Comments
 (0)