Skip to content

Improve the GNU ci analysis: display the root test changes too #5728

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Dec 27, 2023
Merged

Improve the GNU ci analysis: display the root test changes too #5728

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
06c221c
try to compare the root results too
sylvestre Dec 26, 2023
5984526
Add debug info in the logs
sylvestre Dec 26, 2023
c2625d0
rename the variable for something more explicit
sylvestre Dec 26, 2023
14079a0
Improve intermittent doc
sylvestre Dec 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 77 additions & 52 deletions .github/workflows/GnuTests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ jobs:
## Compare test failures VS reference
have_new_failures=""
REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log'
ROOT_REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite-root.log'
REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json'
REPO_DEFAULT_BRANCH='${{ steps.vars.outputs.repo_default_branch }}'
path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}'
Expand All @@ -223,68 +224,92 @@ jobs:
rm -f ${COMMENT_LOG}
touch ${COMMENT_LOG}

if test -f "${REF_LOG_FILE}"; then
echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")"
REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort)
NEW_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort)
REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort)
NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort)
for LINE in ${REF_FAILING}
do
if ! grep -Fxq ${LINE}<<<"${NEW_FAILING}"; then
if ! grep ${LINE} ${IGNORE_INTERMITTENT}
compare_tests() {
local new_log_file=$1
local ref_log_file=$2
local test_type=$3 # "standard" or "root"

if test -f "${ref_log_file}"; then
echo "Reference ${test_type} test log SHA1/ID: $(sha1sum -- "${ref_log_file}") - ${test_type}"
REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort)
CURRENT_RUN_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort)
REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort)
CURRENT_RUN_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort)
echo "Detailled information:"
echo "REF_ERROR = ${REF_ERROR}"
echo "CURRENT_RUN_ERROR = ${CURRENT_RUN_ERROR}"
echo "REF_FAILING = ${REF_FAILING}"
echo "CURRENT_RUN_FAILING = ${CURRENT_RUN_FAILING}"

# Compare failing and error tests
for LINE in ${CURRENT_RUN_FAILING}
do
if ! grep -Fxq ${LINE}<<<"${REF_FAILING}"
then
MSG="Congrats! The gnu test ${LINE} is no longer failing!"
echo "::warning ::$MSG"
echo $MSG >> ${COMMENT_LOG}
else
MSG="Skipping an intermittent issue ${LINE}"
echo "::warning ::$MSG"
echo $MSG >> ${COMMENT_LOG}
echo ""
if ! grep ${LINE} ${IGNORE_INTERMITTENT}
then
MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?"
echo "::error ::$MSG"
echo $MSG >> ${COMMENT_LOG}
have_new_failures="true"
else
MSG="Skip an intermittent issue ${LINE} (fails in this run but passes in the 'main' branch)"
echo "::warning ::$MSG"
echo $MSG >> ${COMMENT_LOG}
echo ""
fi
fi
fi
done
for LINE in ${NEW_FAILING}
do
if ! grep -Fxq ${LINE}<<<"${REF_FAILING}"
then
if ! grep ${LINE} ${IGNORE_INTERMITTENT}
done

for LINE in ${REF_FAILING}
do
if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_FAILING}"
then
if ! grep ${LINE} ${IGNORE_INTERMITTENT}
then
MSG="Congrats! The gnu test ${LINE} is no longer failing!"
echo "::warning ::$MSG"
echo $MSG >> ${COMMENT_LOG}
else
MSG="Skipping an intermittent issue ${LINE} (passes in this run but fails in the 'main' branch)"
echo "::warning ::$MSG"
echo $MSG >> ${COMMENT_LOG}
echo ""
fi
fi
done

for LINE in ${CURRENT_RUN_ERROR}
do
if ! grep -Fxq ${LINE}<<<"${REF_ERROR}"
then
MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?"
MSG="GNU test error: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?"
echo "::error ::$MSG"
echo $MSG >> ${COMMENT_LOG}
have_new_failures="true"
else
MSG="Skip an intermittent issue ${LINE}"
fi
done

for LINE in ${REF_ERROR}
do
if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_ERROR}"
then
MSG="Congrats! The gnu test ${LINE} is no longer ERROR!"
echo "::warning ::$MSG"
echo $MSG >> ${COMMENT_LOG}
echo ""
fi
fi
done
for LINE in ${REF_ERROR}
do
if ! grep -Fxq ${LINE}<<<"${NEW_ERROR}"; then
MSG="Congrats! The gnu test ${LINE} is no longer ERROR!"
echo "::warning ::$MSG"
echo $MSG >> ${COMMENT_LOG}
fi
done
for LINE in ${NEW_ERROR}
do
if ! grep -Fxq ${LINE}<<<"${REF_ERROR}"
then
MSG="GNU test error: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?"
echo "::error ::$MSG"
echo $MSG >> ${COMMENT_LOG}
have_new_failures="true"
fi
done
done
else
echo "::warning ::Skipping ${test_type} test failure comparison; no prior reference test logs are available."
fi
}

# Compare standard tests
compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' "${REF_LOG_FILE}" "standard"

# Compare root tests
compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite-root.log' "${ROOT_REF_LOG_FILE}" "root"

else
echo "::warning ::Skipping test failure comparison; no prior reference test logs are available."
fi
if test -n "${have_new_failures}" ; then exit -1 ; fi
- name: Upload comparison log (for GnuComment workflow)
if: success() || failure() # run regardless of prior step success/failure
Expand Down