Skip to content

Commit 6b2f58d

Browse files
picnixzpicnixz
committed
faster CI
1 parent 17f6ef6 commit 6b2f58d

File tree

2 files changed

+4
-303
lines changed

2 files changed

+4
-303
lines changed

‎.github/workflows/build.yml

Lines changed: 4 additions & 275 deletions
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,6 @@ jobs:
4343
#
4444
uses: ./.github/workflows/reusable-context.yml
4545

46-
check-docs:
47-
name: Docs
48-
needs: build-context
49-
if: fromJSON(needs.build-context.outputs.run-docs)
50-
uses: ./.github/workflows/reusable-docs.yml
51-
5246
check-autoconf-regen:
5347
name: 'Check if Autoconf files are up to date'
5448
# Don't use ubuntu-latest but a specific version to make the job
@@ -160,15 +154,14 @@ jobs:
160154
needs: build-context
161155
if: fromJSON(needs.build-context.outputs.run-windows-tests)
162156
strategy:
163-
fail-fast: false
157+
fail-fast: true
164158
matrix:
165159
arch:
166160
- x64
167161
- Win32
168162
- arm64
169163
free-threading:
170164
- false
171-
- true
172165
exclude:
173166
# Skip Win32 on free-threaded builds
174167
- { arch: Win32, free-threading: true }
@@ -177,43 +170,24 @@ jobs:
177170
arch: ${{ matrix.arch }}
178171
free-threading: ${{ matrix.free-threading }}
179172

180-
build-windows-msi:
181-
name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category
182-
Windows MSI${{ '' }}
183-
needs: build-context
184-
if: fromJSON(needs.build-context.outputs.run-windows-msi)
185-
strategy:
186-
fail-fast: false
187-
matrix:
188-
arch:
189-
- x86
190-
- x64
191-
- arm64
192-
uses: ./.github/workflows/reusable-windows-msi.yml
193-
with:
194-
arch: ${{ matrix.arch }}
195-
196173
build-macos:
197174
name: >-
198175
macOS
199176
${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
200177
needs: build-context
201178
if: needs.build-context.outputs.run-tests == 'true'
202179
strategy:
203-
fail-fast: false
180+
fail-fast: true
204181
matrix:
205182
# Cirrus and macos-14 are M1, macos-13 is default GHA Intel.
206183
# macOS 13 only runs tests against the GIL-enabled CPython.
207184
# Cirrus used for upstream, macos-14 for forks.
208185
os:
209186
- ghcr.io/cirruslabs/macos-runner:sonoma
210-
- macos-14
211-
- macos-13
212187
is-fork: # only used for the exclusion trick
213188
- ${{ github.repository_owner != 'python' }}
214189
free-threading:
215190
- false
216-
- true
217191
exclude:
218192
- os: ghcr.io/cirruslabs/macos-runner:sonoma
219193
is-fork: true
@@ -235,17 +209,14 @@ jobs:
235209
needs: build-context
236210
if: needs.build-context.outputs.run-tests == 'true'
237211
strategy:
238-
fail-fast: false
212+
fail-fast: true
239213
matrix:
240214
bolt:
241215
- false
242-
- true
243216
free-threading:
244217
- false
245-
- true
246218
os:
247219
- ubuntu-24.04
248-
- ubuntu-24.04-arm
249220
exclude:
250221
# Do not test BOLT with free-threading, to conserve resources
251222
- bolt: true
@@ -267,7 +238,7 @@ jobs:
267238
needs: build-context
268239
if: needs.build-context.outputs.run-tests == 'true'
269240
strategy:
270-
fail-fast: false
241+
fail-fast: true
271242
matrix:
272243
os: [ubuntu-24.04]
273244
openssl_ver: [3.0.16, 3.1.8, 3.2.4, 3.3.3, 3.4.1]
@@ -330,203 +301,6 @@ jobs:
330301
with:
331302
config_hash: ${{ needs.build-context.outputs.config-hash }}
332303

333-
test-hypothesis:
334-
name: "Hypothesis tests on Ubuntu"
335-
runs-on: ubuntu-24.04
336-
timeout-minutes: 60
337-
needs: build-context
338-
if: needs.build-context.outputs.run-tests == 'true'
339-
env:
340-
OPENSSL_VER: 3.0.16
341-
PYTHONSTRICTEXTENSIONBUILD: 1
342-
steps:
343-
- uses: actions/checkout@v4
344-
with:
345-
persist-credentials: false
346-
- name: Register gcc problem matcher
347-
run: echo "::add-matcher::.github/problem-matchers/gcc.json"
348-
- name: Install dependencies
349-
run: sudo ./.github/workflows/posix-deps-apt.sh
350-
- name: Configure OpenSSL env vars
351-
run: |
352-
echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV"
353-
echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV"
354-
echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV"
355-
- name: 'Restore OpenSSL build'
356-
id: cache-openssl
357-
uses: actions/cache@v4
358-
with:
359-
path: ./multissl/openssl/${{ env.OPENSSL_VER }}
360-
key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
361-
- name: Install OpenSSL
362-
if: steps.cache-openssl.outputs.cache-hit != 'true'
363-
run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux
364-
- name: Add ccache to PATH
365-
run: |
366-
echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
367-
- name: Configure ccache action
368-
uses: hendrikmuhs/ccache-action@v1.2
369-
with:
370-
save: false
371-
- name: Setup directory envs for out-of-tree builds
372-
run: |
373-
echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV"
374-
echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV"
375-
- name: Create directories for read-only out-of-tree builds
376-
run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR"
377-
- name: Bind mount sources read-only
378-
run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR"
379-
- name: Runner image version
380-
run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
381-
- name: Restore config.cache
382-
uses: actions/cache@v4
383-
with:
384-
path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
385-
key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}
386-
- name: Configure CPython out-of-tree
387-
working-directory: ${{ env.CPYTHON_BUILDDIR }}
388-
run: |
389-
../cpython-ro-srcdir/configure \
390-
--config-cache \
391-
--with-pydebug \
392-
--enable-slower-safety \
393-
--with-openssl="$OPENSSL_DIR"
394-
- name: Build CPython out-of-tree
395-
working-directory: ${{ env.CPYTHON_BUILDDIR }}
396-
run: make -j4
397-
- name: Display build info
398-
working-directory: ${{ env.CPYTHON_BUILDDIR }}
399-
run: make pythoninfo
400-
- name: Remount sources writable for tests
401-
# some tests write to srcdir, lack of pyc files slows down testing
402-
run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw
403-
- name: Setup directory envs for out-of-tree builds
404-
run: |
405-
echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV"
406-
- name: "Create hypothesis venv"
407-
working-directory: ${{ env.CPYTHON_BUILDDIR }}
408-
run: |
409-
VENV_LOC=$(realpath -m .)/hypovenv
410-
VENV_PYTHON=$VENV_LOC/bin/python
411-
echo "HYPOVENV=${VENV_LOC}" >> "$GITHUB_ENV"
412-
echo "VENV_PYTHON=${VENV_PYTHON}" >> "$GITHUB_ENV"
413-
./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt"
414-
- name: 'Restore Hypothesis database'
415-
id: cache-hypothesis-database
416-
uses: actions/cache@v4
417-
with:
418-
path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/
419-
key: hypothesis-database-${{ github.head_ref || github.run_id }}
420-
restore-keys: |
421-
hypothesis-database-
422-
- name: "Run tests"
423-
working-directory: ${{ env.CPYTHON_BUILDDIR }}
424-
run: |
425-
# Most of the excluded tests are slow test suites with no property tests
426-
#
427-
# (GH-104097) test_sysconfig is skipped because it has tests that are
428-
# failing when executed from inside a virtual environment.
429-
"${VENV_PYTHON}" -m test \
430-
-W \
431-
--slowest \
432-
-j4 \
433-
--timeout 900 \
434-
-x test_asyncio \
435-
-x test_multiprocessing_fork \
436-
-x test_multiprocessing_forkserver \
437-
-x test_multiprocessing_spawn \
438-
-x test_concurrent_futures \
439-
-x test_socket \
440-
-x test_subprocess \
441-
-x test_signal \
442-
-x test_sysconfig
443-
- uses: actions/upload-artifact@v4
444-
if: always()
445-
with:
446-
name: hypothesis-example-db
447-
path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/
448-
449-
build-asan:
450-
name: 'Address sanitizer'
451-
runs-on: ${{ matrix.os }}
452-
timeout-minutes: 60
453-
needs: build-context
454-
if: needs.build-context.outputs.run-tests == 'true'
455-
strategy:
456-
fail-fast: false
457-
matrix:
458-
os: [ubuntu-24.04]
459-
env:
460-
OPENSSL_VER: 3.0.16
461-
PYTHONSTRICTEXTENSIONBUILD: 1
462-
ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
463-
steps:
464-
- uses: actions/checkout@v4
465-
with:
466-
persist-credentials: false
467-
- name: Runner image version
468-
run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
469-
- name: Restore config.cache
470-
uses: actions/cache@v4
471-
with:
472-
path: config.cache
473-
key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}
474-
- name: Register gcc problem matcher
475-
run: echo "::add-matcher::.github/problem-matchers/gcc.json"
476-
- name: Install dependencies
477-
run: sudo ./.github/workflows/posix-deps-apt.sh
478-
- name: Set up GCC-10 for ASAN
479-
uses: egor-tensin/setup-gcc@v1
480-
with:
481-
version: 10
482-
- name: Configure OpenSSL env vars
483-
run: |
484-
echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV"
485-
echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV"
486-
echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV"
487-
- name: 'Restore OpenSSL build'
488-
id: cache-openssl
489-
uses: actions/cache@v4
490-
with:
491-
path: ./multissl/openssl/${{ env.OPENSSL_VER }}
492-
key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
493-
- name: Install OpenSSL
494-
if: steps.cache-openssl.outputs.cache-hit != 'true'
495-
run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux
496-
- name: Add ccache to PATH
497-
run: |
498-
echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
499-
- name: Configure ccache action
500-
uses: hendrikmuhs/ccache-action@v1.2
501-
with:
502-
save: ${{ github.event_name == 'push' }}
503-
max-size: "200M"
504-
- name: Configure CPython
505-
run: ./configure --config-cache --with-address-sanitizer --without-pymalloc
506-
- name: Build CPython
507-
run: make -j4
508-
- name: Display build info
509-
run: make pythoninfo
510-
- name: Tests
511-
run: xvfb-run make ci
512-
513-
build-tsan:
514-
name: >-
515-
Thread sanitizer
516-
${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
517-
needs: build-context
518-
if: needs.build-context.outputs.run-tests == 'true'
519-
strategy:
520-
fail-fast: false
521-
matrix:
522-
free-threading:
523-
- false
524-
- true
525-
uses: ./.github/workflows/reusable-tsan.yml
526-
with:
527-
config_hash: ${{ needs.build-context.outputs.config-hash }}
528-
free-threading: ${{ matrix.free-threading }}
529-
530304
cross-build-linux:
531305
name: Cross build Linux
532306
runs-on: ubuntu-latest
@@ -567,66 +341,21 @@ jobs:
567341
run: |
568342
"$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed
569343
570-
# CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
571-
cifuzz:
572-
name: CIFuzz
573-
runs-on: ubuntu-latest
574-
timeout-minutes: 60
575-
needs: build-context
576-
if: needs.build-context.outputs.run-ci-fuzz == 'true'
577-
permissions:
578-
security-events: write
579-
strategy:
580-
fail-fast: false
581-
matrix:
582-
sanitizer: [address, undefined, memory]
583-
steps:
584-
- name: Build fuzzers (${{ matrix.sanitizer }})
585-
id: build
586-
uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
587-
with:
588-
oss-fuzz-project-name: cpython3
589-
sanitizer: ${{ matrix.sanitizer }}
590-
- name: Run fuzzers (${{ matrix.sanitizer }})
591-
uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
592-
with:
593-
fuzz-seconds: 600
594-
oss-fuzz-project-name: cpython3
595-
output-sarif: true
596-
sanitizer: ${{ matrix.sanitizer }}
597-
- name: Upload crash
598-
if: failure() && steps.build.outcome == 'success'
599-
uses: actions/upload-artifact@v4
600-
with:
601-
name: ${{ matrix.sanitizer }}-artifacts
602-
path: ./out/artifacts
603-
- name: Upload SARIF
604-
if: always() && steps.build.outcome == 'success'
605-
uses: github/codeql-action/upload-sarif@v3
606-
with:
607-
sarif_file: cifuzz-sarif/results.sarif
608-
checkout_path: cifuzz-sarif
609344
610345
all-required-green: # This job does nothing and is only used for the branch protection
611346
name: All required checks pass
612347
runs-on: ubuntu-latest
613348
timeout-minutes: 5
614349
needs:
615350
- build-context # Transitive dependency, needed to access `run-tests` value
616-
- check-docs
617351
- check-autoconf-regen
618352
- check-generated-files
619353
- build-windows
620-
- build-windows-msi
621354
- build-macos
622355
- build-ubuntu
623356
- build-ubuntu-ssltests
624357
- build-wasi
625-
- test-hypothesis
626-
- build-asan
627-
- build-tsan
628358
- cross-build-linux
629-
- cifuzz
630359
if: always()
631360

632361
steps:

0 commit comments

Comments
 (0)