Skip to content

Commit e068945

Browse files
mafredrimafredri
authored
fix(scripts): Use gh api to fetch ci logs more reliably (#7133)
1 parent dafb766 commit e068945

File tree

1 file changed

+105
-26
lines changed

1 file changed

+105
-26
lines changed

‎scripts/ci-report/fetch_stats_from_ci.sh

+105-26
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ set -euo pipefail
1212
dir="$(dirname "$0")"/ci-stats
1313
mkdir -p "${dir}"
1414

15+
# Disable gh run view logs, it's unreliable.
16+
USE_GH_RUN_VIEW_LOGS=0
17+
1518
pushd "${dir}" >/dev/null
1619

1720
# Stats step name, used for filtering log.
@@ -75,35 +78,111 @@ while read -r run; do
7578
job_log=run-"${database_id}"-job-"${job_database_id}"-"${job_name}".log
7679
if [[ ! -f "${job_log}" ]]; then
7780
echo "Fetching log for: ${job_name} (${job_database_id}, ${job_url})"
78-
# Example log (partial).
79-
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4063489Z ##[group]Run # Artifacts are not available after rerunning a job,
80-
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4063872Z # Artifacts are not available after rerunning a job,
81-
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4064188Z # so we need to print the test stats to the log.
82-
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4064642Z go run ./scripts/ci-report/main.go gotests.json | tee gotests_stats.json
83-
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4110112Z shell: /usr/bin/bash -e {0}
84-
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4110364Z ##[endgroup]
85-
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3440469Z {
86-
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3441078Z "packages": [
87-
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3441448Z {
88-
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3442927Z "name": "agent",
89-
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3443311Z "time": 17.538
90-
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3444048Z },
91-
# ...
92-
gh run view --job "${job_database_id}" --log >"${job_log}" || {
93-
# Sometimes gh fails to extract ZIP, etc. :'(
94-
rm -f "${job_log}"
95-
echo "Failed to fetch log for: ${job_name} (${job_database_id}, ${job_url}), skipping..."
96-
continue
97-
}
98-
log_lines="$(wc -l "${job_log}" | awk '{print $1}')"
99-
if [[ ${log_lines} -lt 2 ]]; then
100-
# Sometimes gh returns nothing and gives no error :'(
101-
rm -f "${job_log}"
102-
echo "Log is empty for: ${job_name} (${job_database_id}, ${job_url}), skipping..."
103-
continue
81+
82+
if [[ ${USE_GH_RUN_VIEW_LOGS} -eq 0 ]]; then
83+
# Since gh run view is unreliable, we will fetch the logs via API
84+
# instead, however, unfortunately the API does not provide the job
85+
# name in the log output.
86+
#
87+
# TODO(mafredri): This would be more reliably fetched from the following URL:
88+
# https://github.com/coder/coder/commit/${head_sha}/checks/${job_database_id}/logs/${job_step_number}
89+
# but it requires browser-level authentication(?).
90+
#
91+
# Example output:
92+
#
93+
# 2023-04-14T05:43:34.4763012Z ##[group]Run # Artifacts are not available after rerunning a job,
94+
# 2023-04-14T05:43:34.4763385Z # Artifacts are not available after rerunning a job,
95+
# 2023-04-14T05:43:34.4763815Z # so we need to print the test stats to the log.
96+
# 2023-04-14T05:43:34.4764149Z go run ./scripts/ci-report/main.go gotests.json | tee gotests_stats.json
97+
# 2023-04-14T05:43:34.4809056Z shell: /usr/bin/bash -e {0}
98+
# 2023-04-14T05:43:34.4809308Z ##[endgroup]
99+
# 2023-04-14T05:43:35.5934784Z {
100+
# 2023-04-14T05:43:35.5935419Z "packages": [
101+
# 2023-04-14T05:43:35.5936020Z {
102+
# 2023-04-14T05:43:35.5936585Z "name": "agent",
103+
# 2023-04-14T05:43:35.5937105Z "time": 17.044
104+
# 2023-04-14T05:43:35.5937631Z },
105+
gh api "/repos/coder/coder/actions/jobs/${job_database_id}/logs" >"${job_log}" || {
106+
# Sometimes gh fails to extract ZIP, etc. :'(
107+
rm -f "${job_log}"
108+
echo "Failed to fetch log for: ${job_name} (${job_database_id}, ${job_url}), skipping..."
109+
continue
110+
}
111+
112+
# Elaborate loop for finding the starting point for $job_step_name.
113+
# We check for the first occurrence of "##[group]" which contains
114+
# the go run command and then continue until we find the next
115+
# "##[group]". We then print everything in between.
116+
log_buffer=()
117+
found_step=0
118+
while read -r line; do
119+
if [[ ${found_step} -eq 1 ]] && [[ ${#log_buffer[@]} -eq 0 ]]; then
120+
if [[ ${line} == *"##[group]"* ]]; then
121+
break
122+
fi
123+
# Mimic output from gh run view.
124+
echo "${job_name}"$'\t'"${job_step_name}"$'\t'"${line}"
125+
fi
126+
if [[ ${found_step} -eq 0 ]] && [[ ${#log_buffer[@]} -eq 0 ]] && [[ ${line} != *"##[group]"* ]]; then
127+
continue
128+
fi
129+
if [[ ${line} == *"##[group]"* ]]; then
130+
log_buffer=("${line}")
131+
continue
132+
fi
133+
if [[ ${#log_buffer[@]} -gt 0 ]]; then
134+
log_buffer+=("${line}")
135+
fi
136+
if [[ ${line} == *"##[endgroup]"* ]]; then
137+
if [[ ${found_step} -eq 1 ]]; then
138+
for bufline in "${log_buffer[@]}"; do
139+
# Mimic output from gh run view.
140+
echo "${job_name}"$'\t'"${job_step_name}"$'\t'"${bufline}"
141+
done
142+
fi
143+
log_buffer=()
144+
continue
145+
fi
146+
# If line contains go run ./scripts/ci-report/main.go gotests.json
147+
if [[ ${line} == *"go run ./scripts/ci-report/main.go"* ]]; then
148+
found_step=1
149+
fi
150+
done <"${job_log}" >"${job_log}.parsed"
151+
mv "${job_log}.parsed" "${job_log}"
152+
else
153+
# Example log (partial).
154+
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4063489Z ##[group]Run # Artifacts are not available after rerunning a job,
155+
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4063872Z # Artifacts are not available after rerunning a job,
156+
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4064188Z # so we need to print the test stats to the log.
157+
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4064642Z go run ./scripts/ci-report/main.go gotests.json | tee gotests_stats.json
158+
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4110112Z shell: /usr/bin/bash -e {0}
159+
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4110364Z ##[endgroup]
160+
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3440469Z {
161+
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3441078Z "packages": [
162+
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3441448Z {
163+
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3442927Z "name": "agent",
164+
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3443311Z "time": 17.538
165+
# test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3444048Z },
166+
# ...
167+
gh run view --job "${job_database_id}" --log >"${job_log}" || {
168+
# Sometimes gh fails to extract ZIP, etc. :'(
169+
rm -f "${job_log}"
170+
echo "Failed to fetch log for: ${job_name} (${job_database_id}, ${job_url}), skipping..."
171+
continue
172+
}
104173
fi
105174
fi
106175

176+
log_lines="$(wc -l "${job_log}" | awk '{print $1}')"
177+
if [[ ${log_lines} -lt 7 ]]; then
178+
# Sanity check in case something went wrong, the ##[group]
179+
# and ##[endgroup] header is 6 lines and start of JSON ("{")
180+
# makes the 7th.
181+
rm -f "${job_log}"
182+
echo "Log is empty for: ${job_name} (${job_database_id}, ${job_url}), skipping..."
183+
continue
184+
fi
185+
107186
if ! job_stats="$(
108187
# Extract the stats job output (JSON) from the job log,
109188
# discarding the timestamp and non-JSON header.

0 commit comments

Comments
 (0)