From 06c221c3f7d051486b0c3d4dbffc0dde134bc752 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Tue, 26 Dec 2023 01:02:30 +0100 Subject: [PATCH 1/4] try to compare the root results too --- .github/workflows/GnuTests.yml | 124 +++++++++++++++++++-------------- 1 file changed, 72 insertions(+), 52 deletions(-) diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 0799b2e6c14..97b8810f484 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -204,6 +204,7 @@ jobs: ## Compare test failures VS reference have_new_failures="" REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log' + ROOT_REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite-root.log' REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' REPO_DEFAULT_BRANCH='${{ steps.vars.outputs.repo_default_branch }}' path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' @@ -223,68 +224,87 @@ jobs: rm -f ${COMMENT_LOG} touch ${COMMENT_LOG} - if test -f "${REF_LOG_FILE}"; then - echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" - REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort) - NEW_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort) - REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort) - NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort) - for LINE in ${REF_FAILING} - do - if ! grep -Fxq ${LINE}<<<"${NEW_FAILING}"; then - if ! grep ${LINE} ${IGNORE_INTERMITTENT} + compare_tests() { + local new_log_file=$1 + local ref_log_file=$2 + local test_type=$3 # "standard" or "root" + + if test -f "${ref_log_file}"; then + echo "Reference ${test_type} test log SHA1/ID: $(sha1sum -- "${ref_log_file}")" + REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort) + NEW_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort) + REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort) + NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort) + + # Compare failing and error tests + for LINE in ${NEW_FAILING} + do + if ! grep -Fxq ${LINE}<<<"${REF_FAILING}" then - MSG="Congrats! The gnu test ${LINE} is no longer failing!" - echo "::warning ::$MSG" - echo $MSG >> ${COMMENT_LOG} - else - MSG="Skipping an intermittent issue ${LINE}" - echo "::warning ::$MSG" - echo $MSG >> ${COMMENT_LOG} - echo "" + if ! grep ${LINE} ${IGNORE_INTERMITTENT} + then + MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?" + echo "::error ::$MSG" + echo $MSG >> ${COMMENT_LOG} + have_new_failures="true" + else + MSG="Skip an intermittent issue ${LINE}" + echo "::warning ::$MSG" + echo $MSG >> ${COMMENT_LOG} + echo "" + fi fi - fi - done - for LINE in ${NEW_FAILING} - do - if ! grep -Fxq ${LINE}<<<"${REF_FAILING}" - then - if ! grep ${LINE} ${IGNORE_INTERMITTENT} + done + + for LINE in ${REF_FAILING} + do + if ! grep -Fxq ${LINE}<<<"${NEW_FAILING}" + then + if ! grep ${LINE} ${IGNORE_INTERMITTENT} + then + MSG="Congrats! The gnu test ${LINE} is no longer failing!" + echo "::warning ::$MSG" + echo $MSG >> ${COMMENT_LOG} + else + MSG="Skipping an intermittent issue ${LINE}" + echo "::warning ::$MSG" + echo $MSG >> ${COMMENT_LOG} + echo "" + fi + fi + done + + for LINE in ${NEW_ERROR} + do + if ! grep -Fxq ${LINE}<<<"${REF_ERROR}" then - MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?" + MSG="GNU test error: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?" echo "::error ::$MSG" echo $MSG >> ${COMMENT_LOG} have_new_failures="true" - else - MSG="Skip an intermittent issue ${LINE}" + fi + done + + for LINE in ${REF_ERROR} + do + if ! grep -Fxq ${LINE}<<<"${NEW_ERROR}" + then + MSG="Congrats! The gnu test ${LINE} is no longer ERROR!" echo "::warning ::$MSG" echo $MSG >> ${COMMENT_LOG} - echo "" fi - fi - done - for LINE in ${REF_ERROR} - do - if ! grep -Fxq ${LINE}<<<"${NEW_ERROR}"; then - MSG="Congrats! The gnu test ${LINE} is no longer ERROR!" - echo "::warning ::$MSG" - echo $MSG >> ${COMMENT_LOG} - fi - done - for LINE in ${NEW_ERROR} - do - if ! grep -Fxq ${LINE}<<<"${REF_ERROR}" - then - MSG="GNU test error: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?" - echo "::error ::$MSG" - echo $MSG >> ${COMMENT_LOG} - have_new_failures="true" - fi - done + done + else + echo "::warning ::Skipping ${test_type} test failure comparison; no prior reference test logs are available." + fi + } + + # Compare standard tests + compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' "${REF_LOG_FILE}" "standard" + + # Compare root tests + compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite-root.log' "${ROOT_REF_LOG_FILE}" "root" - else - echo "::warning ::Skipping test failure comparison; no prior reference test logs are available." - fi if test -n "${have_new_failures}" ; then exit -1 ; fi - name: Upload comparison log (for GnuComment workflow) if: success() || failure() # run regardless of prior step success/failure From 598452622a62f735092fa9e31e297d0f8a84af24 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Tue, 26 Dec 2023 12:25:01 +0100 Subject: [PATCH 2/4] Add debug info in the logs --- .github/workflows/GnuTests.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 97b8810f484..9877a8f8a53 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -230,12 +230,16 @@ jobs: local test_type=$3 # "standard" or "root" if test -f "${ref_log_file}"; then - echo "Reference ${test_type} test log SHA1/ID: $(sha1sum -- "${ref_log_file}")" + echo "Reference ${test_type} test log SHA1/ID: $(sha1sum -- "${ref_log_file}") - ${test_type}" REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort) NEW_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort) REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort) NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort) - + echo "Detailled information:" + echo "REF_ERROR = ${REF_ERROR}" + echo "NEW_ERROR = ${NEW_ERROR}" + echo "REF_FAILING = ${REF_FAILING}" + echo "NEW_FAILING = ${NEW_FAILING}" # Compare failing and error tests for LINE in ${NEW_FAILING} do From c2625d071e84cf188839cee17c2755c7de00ba6a Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Tue, 26 Dec 2023 15:15:21 +0100 Subject: [PATCH 3/4] rename the variable for something more explicit --- .github/workflows/GnuTests.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 9877a8f8a53..7bfc829d83c 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -232,16 +232,16 @@ jobs: if test -f "${ref_log_file}"; then echo "Reference ${test_type} test log SHA1/ID: $(sha1sum -- "${ref_log_file}") - ${test_type}" REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort) - NEW_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort) + CURRENT_RUN_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort) REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort) - NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort) + CURRENT_RUN_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort) echo "Detailled information:" echo "REF_ERROR = ${REF_ERROR}" - echo "NEW_ERROR = ${NEW_ERROR}" + echo "CURRENT_RUN_ERROR = ${CURRENT_RUN_ERROR}" echo "REF_FAILING = ${REF_FAILING}" - echo "NEW_FAILING = ${NEW_FAILING}" + echo "CURRENT_RUN_FAILING = ${CURRENT_RUN_FAILING}" # Compare failing and error tests - for LINE in ${NEW_FAILING} + for LINE in ${CURRENT_RUN_FAILING} do if ! grep -Fxq ${LINE}<<<"${REF_FAILING}" then @@ -262,7 +262,7 @@ jobs: for LINE in ${REF_FAILING} do - if ! grep -Fxq ${LINE}<<<"${NEW_FAILING}" + if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_FAILING}" then if ! grep ${LINE} ${IGNORE_INTERMITTENT} then @@ -278,7 +278,7 @@ jobs: fi done - for LINE in ${NEW_ERROR} + for LINE in ${CURRENT_RUN_ERROR} do if ! grep -Fxq ${LINE}<<<"${REF_ERROR}" then @@ -291,7 +291,7 @@ jobs: for LINE in ${REF_ERROR} do - if ! grep -Fxq ${LINE}<<<"${NEW_ERROR}" + if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_ERROR}" then MSG="Congrats! The gnu test ${LINE} is no longer ERROR!" echo "::warning ::$MSG" From 14079a0e7197b69d995504034787e6d91c242f16 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Tue, 26 Dec 2023 15:31:53 +0100 Subject: [PATCH 4/4] Improve intermittent doc --- .github/workflows/GnuTests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 7bfc829d83c..89fa5a7da50 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -240,6 +240,7 @@ jobs: echo "CURRENT_RUN_ERROR = ${CURRENT_RUN_ERROR}" echo "REF_FAILING = ${REF_FAILING}" echo "CURRENT_RUN_FAILING = ${CURRENT_RUN_FAILING}" + # Compare failing and error tests for LINE in ${CURRENT_RUN_FAILING} do @@ -252,7 +253,7 @@ jobs: echo $MSG >> ${COMMENT_LOG} have_new_failures="true" else - MSG="Skip an intermittent issue ${LINE}" + MSG="Skip an intermittent issue ${LINE} (fails in this run but passes in the 'main' branch)" echo "::warning ::$MSG" echo $MSG >> ${COMMENT_LOG} echo "" @@ -270,7 +271,7 @@ jobs: echo "::warning ::$MSG" echo $MSG >> ${COMMENT_LOG} else - MSG="Skipping an intermittent issue ${LINE}" + MSG="Skipping an intermittent issue ${LINE} (passes in this run but fails in the 'main' branch)" echo "::warning ::$MSG" echo $MSG >> ${COMMENT_LOG} echo ""