From 3c13d7f0c0de55bcbccf3ad282afe4330ac7ae5f Mon Sep 17 00:00:00 2001 From: Gguidini Date: Mon, 7 Apr 2025 13:32:33 +0200 Subject: [PATCH 1/3] feat(CI): add Codecov Test Analytics for flaky and failed tests ps.: the action that uploads the test results doesn't have a great time differentiating 'linux' from 'linux-arm64', so I had to add the matrix values for the `os:` option --- .github/workflows/tests.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2c033d69b126..a99343f2e8dd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -52,30 +52,39 @@ jobs: python-version: '3.11' extra-requirements: '-c requirements/testing/minver.txt' delete-font-cache: true + codecov-test-results-action-os: 'linux' - os: ubuntu-22.04 python-version: '3.11' CFLAGS: "-fno-lto" # Ensure that disabling LTO works. extra-requirements: '-r requirements/testing/extra.txt' + codecov-test-results-action-os: 'linux' - os: ubuntu-22.04-arm python-version: '3.12' + codecov-test-results-action-os: 'linux-arm64' - os: ubuntu-22.04 python-version: '3.13' + codecov-test-results-action-os: 'linux' - name-suffix: "Free-threaded" os: ubuntu-22.04 python-version: '3.13t' + codecov-test-results-action-os: 'linux' - os: ubuntu-24.04 python-version: '3.12' + codecov-test-results-action-os: 'linux' - os: macos-13 # This runner is on Intel chips. # merge numpy and pandas install in nighties test when this runner is dropped python-version: '3.10' + codecov-test-results-action-os: 'macos' - os: macos-14 # This runner is on M1 (arm64) chips. python-version: '3.12' # https://github.com/matplotlib/matplotlib/issues/29732 pygobject-ver: '<3.52.0' + codecov-test-results-action-os: 'macos' - os: macos-14 # This runner is on M1 (arm64) chips. python-version: '3.13' # https://github.com/matplotlib/matplotlib/issues/29732 pygobject-ver: '<3.52.0' + codecov-test-results-action-os: 'macos' steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -326,8 +335,15 @@ jobs: fi pytest -rfEsXR -n auto \ --maxfail=50 --timeout=300 --durations=25 \ - --cov-report=xml --cov=lib --log-level=DEBUG --color=yes + --cov-report=xml --cov=lib --log-level=DEBUG --color=yes \ + --junitxml=junit.xml -o junit_family=legacy + - name: Upload test results to Codecov + if: ${{ !cancelled() }} # Run even if tests fail + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + os: ${{ matrix.codecov-test-results-action-os }} - name: Cleanup non-failed image files if: failure() run: | From 887c1e937c265df4cfa2099836b54a3f2ac1cb52 Mon Sep 17 00:00:00 2001 From: Gguidini Date: Tue, 8 Apr 2025 09:50:28 +0200 Subject: [PATCH 2/3] move upload test results step alongside the codecov action step --- .github/workflows/tests.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a99343f2e8dd..ed0317dd9012 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -338,12 +338,6 @@ jobs: --cov-report=xml --cov=lib --log-level=DEBUG --color=yes \ --junitxml=junit.xml -o junit_family=legacy - - name: Upload test results to Codecov - if: ${{ !cancelled() }} # Run even if tests fail - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - os: ${{ matrix.codecov-test-results-action-os }} - name: Cleanup non-failed image files if: failure() run: | @@ -400,6 +394,14 @@ jobs: xcrun llvm-cov export -format="lcov" build/*/src/*.so \ -instr-profile default.profdata > info.lcov fi + + - name: Upload test results to Codecov + if: ${{ !cancelled() }} # Run even if tests fail + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + os: ${{ matrix.codecov-test-results-action-os }} + - name: Upload code coverage if: ${{ !cancelled() && github.event_name != 'schedule' }} uses: codecov/codecov-action@0565863a31f2c772f9f0395002a31e3f06189574 # v5.4.0 From bc70afb98830826cfd18ee24dc185bcc1af16a66 Mon Sep 17 00:00:00 2001 From: Gguidini Date: Thu, 10 Apr 2025 15:08:23 +0200 Subject: [PATCH 3/3] don't require an extra matrix var for test-results + add flags to test results --- .github/workflows/tests.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ed0317dd9012..afa7ea72fe30 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -52,39 +52,30 @@ jobs: python-version: '3.11' extra-requirements: '-c requirements/testing/minver.txt' delete-font-cache: true - codecov-test-results-action-os: 'linux' - os: ubuntu-22.04 python-version: '3.11' CFLAGS: "-fno-lto" # Ensure that disabling LTO works. extra-requirements: '-r requirements/testing/extra.txt' - codecov-test-results-action-os: 'linux' - os: ubuntu-22.04-arm python-version: '3.12' - codecov-test-results-action-os: 'linux-arm64' - os: ubuntu-22.04 python-version: '3.13' - codecov-test-results-action-os: 'linux' - name-suffix: "Free-threaded" os: ubuntu-22.04 python-version: '3.13t' - codecov-test-results-action-os: 'linux' - os: ubuntu-24.04 python-version: '3.12' - codecov-test-results-action-os: 'linux' - os: macos-13 # This runner is on Intel chips. # merge numpy and pandas install in nighties test when this runner is dropped python-version: '3.10' - codecov-test-results-action-os: 'macos' - os: macos-14 # This runner is on M1 (arm64) chips. python-version: '3.12' # https://github.com/matplotlib/matplotlib/issues/29732 pygobject-ver: '<3.52.0' - codecov-test-results-action-os: 'macos' - os: macos-14 # This runner is on M1 (arm64) chips. python-version: '3.13' # https://github.com/matplotlib/matplotlib/issues/29732 pygobject-ver: '<3.52.0' - codecov-test-results-action-os: 'macos' steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -400,7 +391,8 @@ jobs: uses: codecov/test-results-action@v1 with: token: ${{ secrets.CODECOV_TOKEN }} - os: ${{ matrix.codecov-test-results-action-os }} + flags: ${{matrix.os}},${{matrix.python-version}} + os: ${{ matrix.os == 'ubuntu-22.04-arm' && 'linux-arm64' || '' }} - name: Upload code coverage if: ${{ !cancelled() && github.event_name != 'schedule' }}