From 97ffb3e136e52ec30cb0bce4b104f9236fec7ce5 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 28 Jul 2025 20:12:56 +0100 Subject: [PATCH 01/18] Add Android CI job --- .github/actions/build-android/action.yml | 49 ++++++++++++++++++++++++ .github/workflows/build.yml | 27 +++++++++++++ .github/zizmor.yml | 3 ++ .pre-commit-config.yaml | 2 +- Android/README.md | 9 +++-- 5 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 .github/actions/build-android/action.yml diff --git a/.github/actions/build-android/action.yml b/.github/actions/build-android/action.yml new file mode 100644 index 00000000000000..e811e9cb4f4146 --- /dev/null +++ b/.github/actions/build-android/action.yml @@ -0,0 +1,49 @@ +# This is coded as an action rather than a workflow so the release-tools +# repository can load it from a dynamically-chosen commit. Cross-repository +# workflow calls must have a Git reference hard-coded in the calling workflow, +# but actions can be run dynamically from the runner's filesystem. + +name: Build and test (Android) +description: Build and test (Android) + +inputs: + triplet: + description: Host triplet + required: true + +runs: + using: composite + + steps: + # Build Python, and package it into a release artifact. + - run: ./Android/android.py build ${{ inputs.triplet }} + - run: ./Android/android.py package ${{ inputs.triplet }} + - uses: actions/upload-artifact@v4 + with: + name: android + path: cross-build/${{ inputs.triplet }}/dist/* + if-no-files-found: error + + # Currently, GitHub Actions can only run the Android emulator on Linux, so + # all the remaining steps are conditional on that. + + # (https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). + - name: Enable KVM for Android emulator + if: runner.os == "Linux" + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + - name: Unpack release artifact + if: runner.os == "Linux" + run: | + mkdir $RUNNER_TEMP/android + tar -C $RUNNER_TEMP/android -xf cross-build/${{ inputs.triplet }}/dist/* + + - name: Tests + if: runner.os == "Linux" + run: | + $RUNNER_TEMP/android/android.py test --managed maxVersion -v -- \ + -uall --single-process --rerun -W diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f07f5e8040acf0..72b3bf0d44c87c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -397,6 +397,31 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py + build-android: + name: "Android" + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + arch: [aarch64, x86_64] + include: + # Use the same runs-on configuration as build-macos and build-ubuntu. + - arch: aarch64 + runs-on: ${{ github.repository_owner == 'python' && 'ghcr.io/cirruslabs/macos-runner:sonoma' || 'macos-14' }} + - arch: x86_64 + runs-on: ubuntu-24.04 + + runs-on: ${{ matrix.runs-on }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: ./.github/actions/build-android + with: + triplet: ${{ matrix.arch }}-linux-android + build-wasi: name: 'WASI' needs: build-context @@ -705,6 +730,7 @@ jobs: - build-ubuntu - build-ubuntu-ssltests-awslc - build-ubuntu-ssltests-openssl + - build-android - build-wasi - test-hypothesis - build-asan @@ -740,6 +766,7 @@ jobs: build-ubuntu, build-ubuntu-ssltests-awslc, build-ubuntu-ssltests-openssl, + build-android, build-wasi, test-hypothesis, build-asan, diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 9b42b47cc85545..208839df39674a 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -8,3 +8,6 @@ rules: config: policies: "*": ref-pin + obfuscation: + ignore: + - build.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 86410c46d1d707..c4098b8934dde9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -72,7 +72,7 @@ repos: - id: actionlint - repo: https://github.com/woodruffw/zizmor-pre-commit - rev: v1.6.0 + rev: v1.11.0 hooks: - id: zizmor diff --git a/Android/README.md b/Android/README.md index c42eb627006e6a..528de2f5d096b3 100644 --- a/Android/README.md +++ b/Android/README.md @@ -96,10 +96,13 @@ similar to the `Android` directory of the CPython source tree. ## Testing -The Python test suite can be run on Linux, macOS, or Windows: +The Python test suite can be run on Linux, macOS, or Windows. -* On Linux, the emulator needs access to the KVM virtualization interface, and - a DISPLAY environment variable pointing at an X server. Xvfb is acceptable. +On Linux, the emulator needs access to the KVM virtualization interface. This may +require adding your user to a group, or [changing your udev +rules](https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). +If the emulator fails to start, try running `$ANDROID_HOME/emulator/emulator +-accel-check`. The test suite can usually be run on a device with 2 GB of RAM, but this is borderline, so you may need to increase it to 4 GB. As of Android From bd7b6426823e69984c1564151234996baa41a3c1 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 28 Jul 2025 20:28:41 +0100 Subject: [PATCH 02/18] Add `shell` to each step of the composite action --- .github/actions/build-android/action.yml | 9 +++++++-- .github/zizmor.yml | 3 --- .pre-commit-config.yaml | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/actions/build-android/action.yml b/.github/actions/build-android/action.yml index e811e9cb4f4146..2d8c27a6f6fbeb 100644 --- a/.github/actions/build-android/action.yml +++ b/.github/actions/build-android/action.yml @@ -16,8 +16,10 @@ runs: steps: # Build Python, and package it into a release artifact. - - run: ./Android/android.py build ${{ inputs.triplet }} - - run: ./Android/android.py package ${{ inputs.triplet }} + - shell: bash + run: ./Android/android.py build ${{ inputs.triplet }} + - shell: bash + run: ./Android/android.py package ${{ inputs.triplet }} - uses: actions/upload-artifact@v4 with: name: android @@ -30,6 +32,7 @@ runs: # (https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). - name: Enable KVM for Android emulator if: runner.os == "Linux" + shell: bash run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ | sudo tee /etc/udev/rules.d/99-kvm4all.rules @@ -38,12 +41,14 @@ runs: - name: Unpack release artifact if: runner.os == "Linux" + shell: bash run: | mkdir $RUNNER_TEMP/android tar -C $RUNNER_TEMP/android -xf cross-build/${{ inputs.triplet }}/dist/* - name: Tests if: runner.os == "Linux" + shell: bash run: | $RUNNER_TEMP/android/android.py test --managed maxVersion -v -- \ -uall --single-process --rerun -W diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 208839df39674a..9b42b47cc85545 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -8,6 +8,3 @@ rules: config: policies: "*": ref-pin - obfuscation: - ignore: - - build.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c4098b8934dde9..e869741de3d855 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -72,9 +72,10 @@ repos: - id: actionlint - repo: https://github.com/woodruffw/zizmor-pre-commit - rev: v1.11.0 + rev: v1.6.0 hooks: - id: zizmor + exclude: ^.github/actions/ - repo: https://github.com/sphinx-contrib/sphinx-lint rev: v1.0.0 From 1d8dc2f14dc4635dfc1b672f3cc44fa164da9b39 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 28 Jul 2025 20:37:24 +0100 Subject: [PATCH 03/18] Use single quotes --- .github/actions/build-android/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/build-android/action.yml b/.github/actions/build-android/action.yml index 2d8c27a6f6fbeb..a3c2bc581fafe5 100644 --- a/.github/actions/build-android/action.yml +++ b/.github/actions/build-android/action.yml @@ -31,7 +31,7 @@ runs: # (https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). - name: Enable KVM for Android emulator - if: runner.os == "Linux" + if: runner.os == 'Linux' shell: bash run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ @@ -40,14 +40,14 @@ runs: sudo udevadm trigger --name-match=kvm - name: Unpack release artifact - if: runner.os == "Linux" + if: runner.os == 'Linux' shell: bash run: | mkdir $RUNNER_TEMP/android tar -C $RUNNER_TEMP/android -xf cross-build/${{ inputs.triplet }}/dist/* - name: Tests - if: runner.os == "Linux" + if: runner.os == 'Linux' shell: bash run: | $RUNNER_TEMP/android/android.py test --managed maxVersion -v -- \ From 51ee9235c11aa95cb84cbdf7f217041538eec05a Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 28 Jul 2025 20:46:20 +0100 Subject: [PATCH 04/18] Use different artifact names in each matrix job --- .github/actions/build-android/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-android/action.yml b/.github/actions/build-android/action.yml index a3c2bc581fafe5..08d081271afe36 100644 --- a/.github/actions/build-android/action.yml +++ b/.github/actions/build-android/action.yml @@ -4,7 +4,7 @@ # but actions can be run dynamically from the runner's filesystem. name: Build and test (Android) -description: Build and test (Android) +description: inputs: triplet: @@ -22,7 +22,7 @@ runs: run: ./Android/android.py package ${{ inputs.triplet }} - uses: actions/upload-artifact@v4 with: - name: android + name: ${{ inputs.triplet }} path: cross-build/${{ inputs.triplet }}/dist/* if-no-files-found: error From 0d1b6d1bf1bd4539534240087f77e42faa72e387 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 28 Jul 2025 21:29:49 +0100 Subject: [PATCH 05/18] Use similar arguments to `--fast-ci` --- .github/actions/build-android/action.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-android/action.yml b/.github/actions/build-android/action.yml index 08d081271afe36..2245a7601a0fe6 100644 --- a/.github/actions/build-android/action.yml +++ b/.github/actions/build-android/action.yml @@ -4,7 +4,7 @@ # but actions can be run dynamically from the runner's filesystem. name: Build and test (Android) -description: +description: Build and test (Android) inputs: triplet: @@ -49,6 +49,8 @@ runs: - name: Tests if: runner.os == 'Linux' shell: bash + # Arguments are similar to --fast-ci, but in single-process mode. run: | $RUNNER_TEMP/android/android.py test --managed maxVersion -v -- \ - -uall --single-process --rerun -W + --single-process --fail-env-changed --rerun --slowest --verbose3 \ + -u "all,-cpu" --timeout=600 From 67c7af31f6926e86f486080b7235b39a5d733c95 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 30 Jul 2025 14:56:30 +0100 Subject: [PATCH 06/18] Suppress `make install` output by default --- .pre-commit-config.yaml | 1 + Android/android.py | 42 +++++++++++++++++++++++++---------------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e869741de3d855..7f2e1b93669c95 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -75,6 +75,7 @@ repos: rev: v1.6.0 hooks: - id: zizmor + # Action files are misidentified as workflows. exclude: ^.github/actions/ - repo: https://github.com/sphinx-contrib/sphinx-lint diff --git a/Android/android.py b/Android/android.py index 75f73cd30993da..d507ad228d8852 100755 --- a/Android/android.py +++ b/Android/android.py @@ -247,7 +247,13 @@ def make_host_python(context): # flags to be duplicated. So we don't use the `host` argument here. os.chdir(host_dir) run(["make", "-j", str(os.cpu_count())]) - run(["make", "install", f"prefix={prefix_dir}"]) + + # The `make install` output is very verbose and rarely useful, so + # suppress it by default. + run( + ["make", "install", f"prefix={prefix_dir}"], + capture_output=not context.verbose, + ) def build_all(context): @@ -695,24 +701,31 @@ def parse_args(): parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand", required=True) + def add_parser(*args, **kwargs): + parser = subcommands.add_parser(*args, **kwargs) + parser.add_argument( + "-v", "--verbose", action="count", default=0, + help="Show verbose output. Use twice to be even more verbose.") + return parser + # Subcommands - build = subcommands.add_parser( + build = add_parser( "build", help="Run configure-build, make-build, configure-host and " "make-host") - configure_build = subcommands.add_parser( + configure_build = add_parser( "configure-build", help="Run `configure` for the build Python") - subcommands.add_parser( + add_parser( "make-build", help="Run `make` for the build Python") - configure_host = subcommands.add_parser( + configure_host = add_parser( "configure-host", help="Run `configure` for Android") - make_host = subcommands.add_parser( + make_host = add_parser( "make-host", help="Run `make` for Android") - subcommands.add_parser("clean", help="Delete all build directories") - subcommands.add_parser("build-testbed", help="Build the testbed app") - test = subcommands.add_parser("test", help="Run the testbed app") - package = subcommands.add_parser("package", help="Make a release package") - env = subcommands.add_parser("env", help="Print environment variables") + add_parser("clean", help="Delete all build directories") + add_parser("build-testbed", help="Build the testbed app") + test = add_parser("test", help="Run the testbed app") + package = add_parser("package", help="Make a release package") + env = add_parser("env", help="Print environment variables") # Common arguments for subcommand in build, configure_build, configure_host: @@ -733,11 +746,6 @@ def parse_args(): help="Extra arguments to pass to `configure`") # Test arguments - test.add_argument( - "-v", "--verbose", action="count", default=0, - help="Show Gradle output, and non-Python logcat messages. " - "Use twice to include high-volume messages which are rarely useful.") - device_group = test.add_mutually_exclusive_group(required=True) device_group.add_argument( "--connected", metavar="SERIAL", help="Run on a connected device. " @@ -803,6 +811,8 @@ def main(): def print_called_process_error(e): for stream_name in ["stdout", "stderr"]: content = getattr(e, stream_name) + if isinstance(content, bytes): + content = content.decode(*DECODE_ARGS) stream = getattr(sys, stream_name) if content: stream.write(content) From 16f1230f241d69353cb75cb5856bffd3fc0de9f4 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 30 Jul 2025 15:14:37 +0100 Subject: [PATCH 07/18] Strip debug information from release package by default --- Android/android.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Android/android.py b/Android/android.py index d507ad228d8852..a17f457c412ff6 100755 --- a/Android/android.py +++ b/Android/android.py @@ -677,6 +677,14 @@ def package(context): else: shutil.copy2(src, dst, follow_symlinks=False) + # Strip debug information. + if not context.debug: + run( + [android_env(context.host)["STRIP"]] + + glob(f"{temp_dir}/**/*.so", recursive=True), + log=False, + ) + dist_dir = subdir(context.host, "dist", create=True) package_path = shutil.make_archive( f"{dist_dir}/python-{version}-{context.host}", "gztar", temp_dir @@ -773,6 +781,11 @@ def add_parser(*args, **kwargs): "args", nargs="*", help=f"Arguments to add to sys.argv. " f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.") + # Package arguments. + package.add_argument( + "-g", action="store_true", default=False, dest="debug", + help="Include debug information") + return parser.parse_args() From 563ca5a1b281baa2fc8dd0958278a4d27c7c90a4 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 3 Aug 2025 22:19:42 +0100 Subject: [PATCH 08/18] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Russell Keith-Magee Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) --- .github/workflows/build.yml | 3 ++- Android/android.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 72b3bf0d44c87c..7f5fcd1b26b41d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -418,7 +418,8 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - uses: ./.github/actions/build-android + - name: Build and Test Android + uses: ./.github/actions/build-android with: triplet: ${{ matrix.arch }}-linux-android diff --git a/Android/android.py b/Android/android.py index a17f457c412ff6..db411192b260d6 100755 --- a/Android/android.py +++ b/Android/android.py @@ -680,8 +680,10 @@ def package(context): # Strip debug information. if not context.debug: run( - [android_env(context.host)["STRIP"]] - + glob(f"{temp_dir}/**/*.so", recursive=True), + [ + android_env(context.host)["STRIP"], + *glob(f"{temp_dir}/**/*.so", recursive=True), + ], log=False, ) From bd876f47f46baf8b0603950cf5f858930440c9fe Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 3 Aug 2025 22:41:52 +0100 Subject: [PATCH 09/18] Convert action into a single step with group markers --- .github/actions/build-android/action.yml | 82 ++++++++++++++---------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/.github/actions/build-android/action.yml b/.github/actions/build-android/action.yml index 2245a7601a0fe6..5b9ece0948d7b6 100644 --- a/.github/actions/build-android/action.yml +++ b/.github/actions/build-android/action.yml @@ -15,42 +15,58 @@ runs: using: composite steps: - # Build Python, and package it into a release artifact. + # The steps of a composite action are not clearly divided in the GitHub + # UI, so use a single step with ::group:: markers instead. - shell: bash - run: ./Android/android.py build ${{ inputs.triplet }} - - shell: bash - run: ./Android/android.py package ${{ inputs.triplet }} + run: | + echo "::group::Configure build Python" + ./Android/android.py configure-build + echo "::endgroup::" + + echo "::group::Compile build Python" + ./Android/android.py make-build + echo "::endgroup::" + + echo "::group::Configure host Python" + ./Android/android.py configure-host ${{ inputs.triplet }} + echo "::endgroup::" + + echo "::group::Compile host Python" + ./Android/android.py make-host ${{ inputs.triplet }} + echo "::endgroup::" + + echo "::group::Make release package" + ./Android/android.py package ${{ inputs.triplet }} + echo "::endgroup::" + + if [ "$RUNNER_OS" = "Linux" ] && [ "$RUNNER_ARCH" = "X64"]; then + # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/ + echo "::group::Enable KVM for Android emulator" + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + echo "::endgroup::" + + echo "::group::Unpack release artifact" + mkdir $RUNNER_TEMP/android + tar -C $RUNNER_TEMP/android -xf cross-build/${{ inputs.triplet }}/dist/* + echo "::endgroup::" + + echo "::group::Tests" + # Arguments are similar to --fast-ci, but in single-process mode. + $RUNNER_TEMP/android/android.py test --managed maxVersion -v -- \ + --single-process --fail-env-changed --rerun --slowest --verbose3 \ + -u "all,-cpu" --timeout=600 + echo "::endgroup::" + + else + echo "Skipping test: GitHub Actions currently only supports the " \ + "Android emulator on Linux x86_64." + fi + - uses: actions/upload-artifact@v4 with: name: ${{ inputs.triplet }} path: cross-build/${{ inputs.triplet }}/dist/* if-no-files-found: error - - # Currently, GitHub Actions can only run the Android emulator on Linux, so - # all the remaining steps are conditional on that. - - # (https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). - - name: Enable KVM for Android emulator - if: runner.os == 'Linux' - shell: bash - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ - | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - - name: Unpack release artifact - if: runner.os == 'Linux' - shell: bash - run: | - mkdir $RUNNER_TEMP/android - tar -C $RUNNER_TEMP/android -xf cross-build/${{ inputs.triplet }}/dist/* - - - name: Tests - if: runner.os == 'Linux' - shell: bash - # Arguments are similar to --fast-ci, but in single-process mode. - run: | - $RUNNER_TEMP/android/android.py test --managed maxVersion -v -- \ - --single-process --fail-env-changed --rerun --slowest --verbose3 \ - -u "all,-cpu" --timeout=600 From 6b9184c1b36476baa5ebc60fee1009d1f0401c2c Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 3 Aug 2025 23:17:14 +0100 Subject: [PATCH 10/18] Update zizmor --- .github/actions/build-android/action.yml | 12 +++++++----- .github/workflows/build.yml | 8 ++++---- .pre-commit-config.yaml | 4 +--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/actions/build-android/action.yml b/.github/actions/build-android/action.yml index 5b9ece0948d7b6..efb39b86bea257 100644 --- a/.github/actions/build-android/action.yml +++ b/.github/actions/build-android/action.yml @@ -18,6 +18,8 @@ runs: # The steps of a composite action are not clearly divided in the GitHub # UI, so use a single step with ::group:: markers instead. - shell: bash + env: + - triplet: ${{ inputs.triplet }} run: | echo "::group::Configure build Python" ./Android/android.py configure-build @@ -28,18 +30,18 @@ runs: echo "::endgroup::" echo "::group::Configure host Python" - ./Android/android.py configure-host ${{ inputs.triplet }} + ./Android/android.py configure-host $triplet echo "::endgroup::" echo "::group::Compile host Python" - ./Android/android.py make-host ${{ inputs.triplet }} + ./Android/android.py make-host $triplet echo "::endgroup::" echo "::group::Make release package" - ./Android/android.py package ${{ inputs.triplet }} + ./Android/android.py package $triplet echo "::endgroup::" - if [ "$RUNNER_OS" = "Linux" ] && [ "$RUNNER_ARCH" = "X64"]; then + if [ "$RUNNER_OS" = "Linux" ] && [ "$RUNNER_ARCH" = "X64" ]; then # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/ echo "::group::Enable KVM for Android emulator" echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ @@ -50,7 +52,7 @@ runs: echo "::group::Unpack release artifact" mkdir $RUNNER_TEMP/android - tar -C $RUNNER_TEMP/android -xf cross-build/${{ inputs.triplet }}/dist/* + tar -C $RUNNER_TEMP/android -xf cross-build/$triplet/dist/* echo "::endgroup::" echo "::group::Tests" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7f5fcd1b26b41d..3e29f0b7f3208e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -178,8 +178,8 @@ jobs: free-threading: ${{ matrix.free-threading }} build-windows-msi: - name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category - Windows MSI${{ '' }} + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: Windows MSI${{ '' }} # zizmor: ignore[obfuscation] needs: build-context if: fromJSON(needs.build-context.outputs.run-windows-msi) strategy: @@ -612,8 +612,8 @@ jobs: run: xvfb-run make ci build-san: - name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category - Sanitizers${{ '' }} + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: Sanitizers${{ '' }} # zizmor: ignore[obfuscation] needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7f2e1b93669c95..c4098b8934dde9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -72,11 +72,9 @@ repos: - id: actionlint - repo: https://github.com/woodruffw/zizmor-pre-commit - rev: v1.6.0 + rev: v1.11.0 hooks: - id: zizmor - # Action files are misidentified as workflows. - exclude: ^.github/actions/ - repo: https://github.com/sphinx-contrib/sphinx-lint rev: v1.0.0 From 55b91f7729218a226384dcd74ff1b951a902e4e9 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 3 Aug 2025 23:21:06 +0100 Subject: [PATCH 11/18] Fix syntax --- .github/actions/build-android/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build-android/action.yml b/.github/actions/build-android/action.yml index efb39b86bea257..accbdafae369ca 100644 --- a/.github/actions/build-android/action.yml +++ b/.github/actions/build-android/action.yml @@ -19,7 +19,7 @@ runs: # UI, so use a single step with ::group:: markers instead. - shell: bash env: - - triplet: ${{ inputs.triplet }} + triplet: ${{ inputs.triplet }} run: | echo "::group::Configure build Python" ./Android/android.py configure-build From 56bed397528f47b678474dd29578a28c2de4936a Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 4 Aug 2025 09:59:47 +0100 Subject: [PATCH 12/18] Cosmetic fixes --- .github/actions/build-android/action.yml | 5 ++++- .github/workflows/build.yml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-android/action.yml b/.github/actions/build-android/action.yml index accbdafae369ca..5c2e58671d4318 100644 --- a/.github/actions/build-android/action.yml +++ b/.github/actions/build-android/action.yml @@ -20,7 +20,10 @@ runs: - shell: bash env: triplet: ${{ inputs.triplet }} + # The `name` of a composite action step doesn't appear anywhere in the + # GitHub UI, so put it in a comment on the first line instead. run: | + # Build and test (Android) echo "::group::Configure build Python" ./Android/android.py configure-build echo "::endgroup::" @@ -63,7 +66,7 @@ runs: echo "::endgroup::" else - echo "Skipping test: GitHub Actions currently only supports the " \ + echo "Skipping test: GitHub Actions currently only supports the" \ "Android emulator on Linux x86_64." fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3e29f0b7f3208e..6c70a2ec2719c7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -418,7 +418,7 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - name: Build and Test Android + - name: Build and test (Android) uses: ./.github/actions/build-android with: triplet: ${{ matrix.arch }}-linux-android From f7f9f44f56f8ea5b65e1322944a05d114b6a6169 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 6 Aug 2025 10:48:59 +0100 Subject: [PATCH 13/18] Update to NDK version 27.3.13750724 --- Android/android-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Android/android-env.sh b/Android/android-env.sh index 7b381a013cf0ba..5859c0eac4a88f 100644 --- a/Android/android-env.sh +++ b/Android/android-env.sh @@ -24,7 +24,7 @@ fail() { # * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md # where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.: # https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md -ndk_version=27.2.12479018 +ndk_version=27.3.13750724 ndk=$ANDROID_HOME/ndk/$ndk_version if ! [ -e "$ndk" ]; then From 48070476471de09b89f2263f68f6f3cb66d964a9 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 9 Aug 2025 18:29:48 +0100 Subject: [PATCH 14/18] Revert "Update zizmor" This reverts commit 6b9184c1b36476baa5ebc60fee1009d1f0401c2c. --- .github/actions/build-android/action.yml | 10 ++++------ .github/workflows/build.yml | 8 ++++---- .pre-commit-config.yaml | 4 +++- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/actions/build-android/action.yml b/.github/actions/build-android/action.yml index 5c2e58671d4318..80e8996c10f8bd 100644 --- a/.github/actions/build-android/action.yml +++ b/.github/actions/build-android/action.yml @@ -18,8 +18,6 @@ runs: # The steps of a composite action are not clearly divided in the GitHub # UI, so use a single step with ::group:: markers instead. - shell: bash - env: - triplet: ${{ inputs.triplet }} # The `name` of a composite action step doesn't appear anywhere in the # GitHub UI, so put it in a comment on the first line instead. run: | @@ -33,15 +31,15 @@ runs: echo "::endgroup::" echo "::group::Configure host Python" - ./Android/android.py configure-host $triplet + ./Android/android.py configure-host ${{ inputs.triplet }} echo "::endgroup::" echo "::group::Compile host Python" - ./Android/android.py make-host $triplet + ./Android/android.py make-host ${{ inputs.triplet }} echo "::endgroup::" echo "::group::Make release package" - ./Android/android.py package $triplet + ./Android/android.py package ${{ inputs.triplet }} echo "::endgroup::" if [ "$RUNNER_OS" = "Linux" ] && [ "$RUNNER_ARCH" = "X64" ]; then @@ -55,7 +53,7 @@ runs: echo "::group::Unpack release artifact" mkdir $RUNNER_TEMP/android - tar -C $RUNNER_TEMP/android -xf cross-build/$triplet/dist/* + tar -C $RUNNER_TEMP/android -xf cross-build/${{ inputs.triplet }}/dist/* echo "::endgroup::" echo "::group::Tests" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6c70a2ec2719c7..3f3acaf3c4c083 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -178,8 +178,8 @@ jobs: free-threading: ${{ matrix.free-threading }} build-windows-msi: - # ${{ '' } is a hack to nest jobs under the same sidebar category. - name: Windows MSI${{ '' }} # zizmor: ignore[obfuscation] + name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category + Windows MSI${{ '' }} needs: build-context if: fromJSON(needs.build-context.outputs.run-windows-msi) strategy: @@ -612,8 +612,8 @@ jobs: run: xvfb-run make ci build-san: - # ${{ '' } is a hack to nest jobs under the same sidebar category. - name: Sanitizers${{ '' }} # zizmor: ignore[obfuscation] + name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category + Sanitizers${{ '' }} needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c4098b8934dde9..7f2e1b93669c95 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -72,9 +72,11 @@ repos: - id: actionlint - repo: https://github.com/woodruffw/zizmor-pre-commit - rev: v1.11.0 + rev: v1.6.0 hooks: - id: zizmor + # Action files are misidentified as workflows. + exclude: ^.github/actions/ - repo: https://github.com/sphinx-contrib/sphinx-lint rev: v1.0.0 From b3daf4b9a67f9f3956926ab735ad987582e8fdd1 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 9 Aug 2025 19:51:00 +0100 Subject: [PATCH 15/18] Convert action into an `android.py ci` command --- .github/actions/build-android/action.yml | 75 ------------------------ .github/workflows/build.yml | 6 +- .pre-commit-config.yaml | 2 - Android/README.md | 7 +-- Android/android.py | 74 +++++++++++++++++++++-- 5 files changed, 73 insertions(+), 91 deletions(-) delete mode 100644 .github/actions/build-android/action.yml diff --git a/.github/actions/build-android/action.yml b/.github/actions/build-android/action.yml deleted file mode 100644 index 80e8996c10f8bd..00000000000000 --- a/.github/actions/build-android/action.yml +++ /dev/null @@ -1,75 +0,0 @@ -# This is coded as an action rather than a workflow so the release-tools -# repository can load it from a dynamically-chosen commit. Cross-repository -# workflow calls must have a Git reference hard-coded in the calling workflow, -# but actions can be run dynamically from the runner's filesystem. - -name: Build and test (Android) -description: Build and test (Android) - -inputs: - triplet: - description: Host triplet - required: true - -runs: - using: composite - - steps: - # The steps of a composite action are not clearly divided in the GitHub - # UI, so use a single step with ::group:: markers instead. - - shell: bash - # The `name` of a composite action step doesn't appear anywhere in the - # GitHub UI, so put it in a comment on the first line instead. - run: | - # Build and test (Android) - echo "::group::Configure build Python" - ./Android/android.py configure-build - echo "::endgroup::" - - echo "::group::Compile build Python" - ./Android/android.py make-build - echo "::endgroup::" - - echo "::group::Configure host Python" - ./Android/android.py configure-host ${{ inputs.triplet }} - echo "::endgroup::" - - echo "::group::Compile host Python" - ./Android/android.py make-host ${{ inputs.triplet }} - echo "::endgroup::" - - echo "::group::Make release package" - ./Android/android.py package ${{ inputs.triplet }} - echo "::endgroup::" - - if [ "$RUNNER_OS" = "Linux" ] && [ "$RUNNER_ARCH" = "X64" ]; then - # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/ - echo "::group::Enable KVM for Android emulator" - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ - | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - echo "::endgroup::" - - echo "::group::Unpack release artifact" - mkdir $RUNNER_TEMP/android - tar -C $RUNNER_TEMP/android -xf cross-build/${{ inputs.triplet }}/dist/* - echo "::endgroup::" - - echo "::group::Tests" - # Arguments are similar to --fast-ci, but in single-process mode. - $RUNNER_TEMP/android/android.py test --managed maxVersion -v -- \ - --single-process --fail-env-changed --rerun --slowest --verbose3 \ - -u "all,-cpu" --timeout=600 - echo "::endgroup::" - - else - echo "Skipping test: GitHub Actions currently only supports the" \ - "Android emulator on Linux x86_64." - fi - - - uses: actions/upload-artifact@v4 - with: - name: ${{ inputs.triplet }} - path: cross-build/${{ inputs.triplet }}/dist/* - if-no-files-found: error diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3f3acaf3c4c083..ee78aa3eaabc42 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -418,10 +418,8 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - name: Build and test (Android) - uses: ./.github/actions/build-android - with: - triplet: ${{ matrix.arch }}-linux-android + - name: Build and test + run: ./Android/android.py ci ${{ matrix.arch }}-linux-android build-wasi: name: 'WASI' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7f2e1b93669c95..86410c46d1d707 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -75,8 +75,6 @@ repos: rev: v1.6.0 hooks: - id: zizmor - # Action files are misidentified as workflows. - exclude: ^.github/actions/ - repo: https://github.com/sphinx-contrib/sphinx-lint rev: v1.0.0 diff --git a/Android/README.md b/Android/README.md index 528de2f5d096b3..9f71aeb934f386 100644 --- a/Android/README.md +++ b/Android/README.md @@ -99,10 +99,9 @@ similar to the `Android` directory of the CPython source tree. The Python test suite can be run on Linux, macOS, or Windows. On Linux, the emulator needs access to the KVM virtualization interface. This may -require adding your user to a group, or [changing your udev -rules](https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). -If the emulator fails to start, try running `$ANDROID_HOME/emulator/emulator --accel-check`. +require adding your user to a group, or changing your udev rules. On GitHub +Actions, the test script will do this automatically using the commands shown +[here](https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). The test suite can usually be run on a device with 2 GB of RAM, but this is borderline, so you may need to increase it to 4 GB. As of Android diff --git a/Android/android.py b/Android/android.py index db411192b260d6..b1265f7a39a335 100755 --- a/Android/android.py +++ b/Android/android.py @@ -3,6 +3,7 @@ import asyncio import argparse import os +import platform import re import shlex import shutil @@ -272,6 +273,16 @@ def clean_all(context): clean(host) +def setup_ci(): + # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/ + if "GITHUB_ACTIONS" in os.environ and platform.system() == "Linux": + Path("/etc/udev/rules.d/99-kvm4all.rules").write_text( + 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n' + ) + run(["sudo", "udevadm", "control", "--reload-rules"]) + run(["sudo", "udevadm", "trigger", "--name-match=kvm"]) + + def setup_sdk(): sdkmanager = android_home / ( "cmdline-tools/latest/bin/sdkmanager" @@ -584,6 +595,7 @@ async def gradle_task(context): async def run_testbed(context): + setup_ci() setup_sdk() setup_testbed() @@ -692,6 +704,53 @@ def package(context): f"{dist_dir}/python-{version}-{context.host}", "gztar", temp_dir ) print(f"Wrote {package_path}") + return package_path + + +def ci(context): + for step in [ + configure_build_python, + make_build_python, + configure_host_python, + make_host_python, + package, + ]: + caption = ( + step.__name__.replace("_", " ") + .capitalize() + .replace("python", "Python") + ) + print(f"::group::{caption}") + result = step(context) + if step is package: + package_path = result + print("::endgroup::") + + if ( + "GITHUB_ACTIONS" in os.environ + and (platform.system(), platform.machine()) != ("Linux", "x86_64") + ): + print( + "Skipping test: GitHub Actions does not supports the Android " + "emulator on this platform." + ) + else: + with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: + print("::group::Tests") + # Prove the package is self-contained by using it to run the tests. + shutil.unpack_archive(package_path, temp_dir) + + # Arguments are similar to --fast-ci, but in single-process mode. + launcher_args = ["--managed", "maxVersion", "-v"] + test_args = [ + "--single-process", "--fail-env-changed", "--rerun", "--slowest", + "--verbose3", "-u", "all,-cpu", "--timeout=600" + ] + run( + ["./android.py", "test", *launcher_args, "--", *test_args], + cwd=temp_dir + ) + print("::endgroup::") def env(context): @@ -735,15 +794,16 @@ def add_parser(*args, **kwargs): add_parser("build-testbed", help="Build the testbed app") test = add_parser("test", help="Run the testbed app") package = add_parser("package", help="Make a release package") + ci = add_parser("ci", help="Run build, package and test") env = add_parser("env", help="Print environment variables") # Common arguments - for subcommand in build, configure_build, configure_host: + for subcommand in [build, configure_build, configure_host, ci]: subcommand.add_argument( "--clean", action="store_true", default=False, dest="clean", help="Delete the relevant build directories first") - host_commands = [build, configure_host, make_host, package] + host_commands = [build, configure_host, make_host, package, ci] if in_source_tree: host_commands.append(env) for subcommand in host_commands: @@ -751,7 +811,7 @@ def add_parser(*args, **kwargs): "host", metavar="HOST", choices=HOSTS, help="Host triplet: choices=[%(choices)s]") - for subcommand in build, configure_build, configure_host: + for subcommand in [build, configure_build, configure_host, ci]: subcommand.add_argument("args", nargs="*", help="Extra arguments to pass to `configure`") @@ -784,9 +844,10 @@ def add_parser(*args, **kwargs): f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.") # Package arguments. - package.add_argument( - "-g", action="store_true", default=False, dest="debug", - help="Include debug information") + for subcommand in [package, ci]: + subcommand.add_argument( + "-g", action="store_true", default=False, dest="debug", + help="Include debug information in package") return parser.parse_args() @@ -811,6 +872,7 @@ def main(): "build-testbed": build_testbed, "test": run_testbed, "package": package, + "ci": ci, "env": env, } From c9700e47b6bf98f6ca2e60ecdbee15ba228dcc7c Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 9 Aug 2025 20:03:46 +0100 Subject: [PATCH 16/18] Use `sudo` when updating udev rules --- Android/android.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/Android/android.py b/Android/android.py index b1265f7a39a335..9633ad36471f93 100755 --- a/Android/android.py +++ b/Android/android.py @@ -276,8 +276,10 @@ def clean_all(context): def setup_ci(): # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/ if "GITHUB_ACTIONS" in os.environ and platform.system() == "Linux": - Path("/etc/udev/rules.d/99-kvm4all.rules").write_text( - 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n' + run( + ["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"], + input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n', + text=True, ) run(["sudo", "udevadm", "control", "--reload-rules"]) run(["sudo", "udevadm", "trigger", "--name-match=kvm"]) @@ -691,13 +693,8 @@ def package(context): # Strip debug information. if not context.debug: - run( - [ - android_env(context.host)["STRIP"], - *glob(f"{temp_dir}/**/*.so", recursive=True), - ], - log=False, - ) + so_files = glob(f"{temp_dir}/**/*.so", recursive=True) + run([android_env(context.host)["STRIP"], *so_files], log=False) dist_dir = subdir(context.host, "dist", create=True) package_path = shutil.make_archive( @@ -731,7 +728,7 @@ def ci(context): and (platform.system(), platform.machine()) != ("Linux", "x86_64") ): print( - "Skipping test: GitHub Actions does not supports the Android " + "Skipping tests: GitHub Actions does not support the Android " "emulator on this platform." ) else: From e8e2a9e0521f30d25fd9d110810fa2bd26656689 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 11 Aug 2025 13:25:21 +0100 Subject: [PATCH 17/18] Adjust matrix generation --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ee78aa3eaabc42..7539a91d6c08e5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -398,14 +398,13 @@ jobs: run: ./python Lib/test/ssltests.py build-android: - name: "Android" + name: Android (${{matrix.arch}}) needs: build-context if: needs.build-context.outputs.run-tests == 'true' timeout-minutes: 60 strategy: fail-fast: false matrix: - arch: [aarch64, x86_64] include: # Use the same runs-on configuration as build-macos and build-ubuntu. - arch: aarch64 From 7e87375eeff06e03a66209e506ce83c3f70ec57f Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 11 Aug 2025 13:32:33 +0100 Subject: [PATCH 18/18] Fix whitespace --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7539a91d6c08e5..faaaa9dc2ee730 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -398,7 +398,7 @@ jobs: run: ./python Lib/test/ssltests.py build-android: - name: Android (${{matrix.arch}}) + name: Android (${{ matrix.arch }}) needs: build-context if: needs.build-context.outputs.run-tests == 'true' timeout-minutes: 60