Skip to content

Commit d69d8da

Browse files
eendebakpteendebakpt
authored
Merge branch 'main' into deepcopy_atomic_types
2 parents 4bc74d3 + c408c36 commit d69d8da

File tree

384 files changed

+11775
-5333
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

384 files changed

+11775
-5333
lines changed

‎.devcontainer/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM docker.io/library/fedora:37
1+
FROM docker.io/library/fedora:40
22

33
ENV CC=clang
44

‎.github/CODEOWNERS

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,18 @@ Lib/test/support/interpreters/ @ericsnowcurrently
243243
Modules/_xx*interp*module.c @ericsnowcurrently
244244
Lib/test/test_interpreters/ @ericsnowcurrently
245245

246+
# Android
247+
**/*Android* @mhsmith
248+
**/*android* @mhsmith
249+
250+
# iOS (but not termios)
251+
**/iOS* @freakboy3742
252+
**/ios* @freakboy3742
253+
**/*_iOS* @freakboy3742
254+
**/*_ios* @freakboy3742
255+
**/*-iOS* @freakboy3742
256+
**/*-ios* @freakboy3742
257+
246258
# WebAssembly
247259
/Tools/wasm/ @brettcannon
248260

‎.github/workflows/build.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ jobs:
137137
uses: actions/cache@v4
138138
with:
139139
path: config.cache
140+
# Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python
140141
key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }}
141142
- name: Install Dependencies
142143
run: sudo ./.github/workflows/posix-deps-apt.sh

‎.github/workflows/jit.yml

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ on:
1212
- 'Python/optimizer*.c'
1313
workflow_dispatch:
1414

15+
permissions:
16+
contents: read
17+
1518
concurrency:
1619
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
1720
cancel-in-progress: true
@@ -38,7 +41,7 @@ jobs:
3841
- true
3942
- false
4043
llvm:
41-
- 16
44+
- 18
4245
include:
4346
- target: i686-pc-windows-msvc/msvc
4447
architecture: Win32
@@ -73,13 +76,13 @@ jobs:
7376
runner: ubuntu-latest
7477
compiler: gcc
7578
# These fail because of emulation, not because of the JIT:
76-
exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection
79+
exclude: test_pathlib test_posixpath test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection
7780
- target: aarch64-unknown-linux-gnu/clang
7881
architecture: aarch64
7982
runner: ubuntu-latest
8083
compiler: clang
8184
# These fail because of emulation, not because of the JIT:
82-
exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection
85+
exclude: test_pathlib test_posixpath test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection
8386
env:
8487
CC: ${{ matrix.compiler }}
8588
steps:
@@ -91,6 +94,7 @@ jobs:
9194
- name: Native Windows
9295
if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
9396
run: |
97+
choco upgrade llvm -y
9498
choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
9599
./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
96100
./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
@@ -99,27 +103,31 @@ jobs:
99103
- name: Emulated Windows
100104
if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
101105
run: |
106+
choco upgrade llvm -y
102107
choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
103108
./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
104109
105110
- name: Native macOS
106111
if: runner.os == 'macOS'
107112
run: |
113+
brew update
108114
brew install llvm@${{ matrix.llvm }}
109115
SDKROOT="$(xcrun --show-sdk-path)" \
110116
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
111117
make all --jobs 4
112118
./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
113119
120+
# --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
114121
- name: Native Linux
115122
if: runner.os == 'Linux' && matrix.architecture == 'x86_64'
116123
run: |
117124
sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
118125
export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
119-
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
126+
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }}
120127
make all --jobs 4
121128
./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
122129
130+
# --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
123131
- name: Emulated Linux
124132
if: runner.os == 'Linux' && matrix.architecture != 'x86_64'
125133
run: |
@@ -136,6 +144,6 @@ jobs:
136144
CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \
137145
CPP="$CC --preprocess" \
138146
HOSTRUNNER=qemu-${{ matrix.architecture }} \
139-
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
147+
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
140148
make all --jobs 4
141149
./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3

‎.github/workflows/reusable-docs.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ jobs:
7474
- name: 'Set up Python'
7575
uses: actions/setup-python@v5
7676
with:
77-
python-version: '3.11' # known to work with Sphinx 4.2
77+
python-version: '3.12' # known to work with Sphinx 6.2.1
7878
cache: 'pip'
7979
cache-dependency-path: 'Doc/requirements-oldest-sphinx.txt'
8080
- name: 'Install build dependencies'

‎.github/workflows/reusable-ubuntu.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ jobs:
1414
timeout-minutes: 60
1515
runs-on: ubuntu-20.04
1616
env:
17+
FORCE_COLOR: 1
1718
OPENSSL_VER: 3.0.13
1819
PYTHONSTRICTEXTENSIONBUILD: 1
1920
steps:

‎.github/workflows/reusable-wasi.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,8 @@ jobs:
5050
uses: actions/cache@v4
5151
with:
5252
path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache
53-
key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
53+
# Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python
54+
key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}-${{ env.pythonLocation }}
5455
- name: "Configure build Python"
5556
run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug
5657
- name: "Make build Python"
@@ -59,7 +60,8 @@ jobs:
5960
uses: actions/cache@v4
6061
with:
6162
path: ${{ env.CROSS_BUILD_WASI }}/config.cache
62-
key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-wasi-sdk-${{ env.WASI_SDK_VERSION }}-${{ inputs.config_hash }}
63+
# Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python
64+
key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-wasi-sdk-${{ env.WASI_SDK_VERSION }}-${{ inputs.config_hash }}-${{ env.pythonLocation }}
6365
- name: "Configure host"
6466
# `--with-pydebug` inferred from configure-build-python
6567
run: python3 Tools/wasm/wasi.py configure-host -- --config-cache

‎Android/README.md

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,25 @@ you don't already have the SDK, here's how to install it:
2222
`android-sdk/cmdline-tools/latest`.
2323
* `export ANDROID_HOME=/path/to/android-sdk`
2424

25+
The `android.py` script also requires the following commands to be on the `PATH`:
26+
27+
* `curl`
28+
* `java`
29+
* `tar`
30+
* `unzip`
31+
2532

2633
## Building
2734

28-
Building for Android requires doing a cross-build where you have a "build"
29-
Python to help produce an Android build of CPython. This procedure has been
30-
tested on Linux and macOS.
35+
Python can be built for Android on any POSIX platform supported by the Android
36+
development tools, which currently means Linux or macOS. This involves doing a
37+
cross-build where you use a "build" Python (for your development machine) to
38+
help produce a "host" Python for Android.
39+
40+
First, make sure you have all the usual tools and libraries needed to build
41+
Python for your development machine. The only Android tool you need to install
42+
is the command line tools package above: the build script will download the
43+
rest.
3144

3245
The easiest way to do a build is to use the `android.py` script. You can either
3346
have it perform the entire build process from start to finish in one step, or
@@ -43,9 +56,10 @@ The discrete steps for building via `android.py` are:
4356
./android.py make-host HOST
4457
```
4558

46-
To see the possible values of HOST, run `./android.py configure-host --help`.
59+
`HOST` identifies which architecture to build. To see the possible values, run
60+
`./android.py configure-host --help`.
4761

48-
Or to do it all in a single command, run:
62+
To do all steps in a single command, run:
4963

5064
```sh
5165
./android.py build HOST
@@ -62,3 +76,22 @@ call. For example, if you want a pydebug build that also caches the results from
6276
```sh
6377
./android.py build HOST -- -C --with-pydebug
6478
```
79+
80+
81+
## Testing
82+
83+
To run the Python test suite on Android:
84+
85+
* Install Android Studio, if you don't already have it.
86+
* Follow the instructions in the previous section to build all supported
87+
architectures.
88+
* Run `./android.py setup-testbed` to download the Gradle wrapper.
89+
* Open the `testbed` directory in Android Studio.
90+
* In the *Device Manager* dock, connect a device or start an emulator.
91+
Then select it from the drop-down list in the toolbar.
92+
* Click the "Run" button in the toolbar.
93+
* The testbed app displays nothing on screen while running. To see its output,
94+
open the [Logcat window](https://developer.android.com/studio/debug/logcat).
95+
96+
To run specific tests, or pass any other arguments to the test suite, edit the
97+
command line in testbed/app/src/main/python/main.py.

‎Android/android-env.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,12 @@ done
6161
export CFLAGS=""
6262
export LDFLAGS="-Wl,--build-id=sha1 -Wl,--no-rosegment"
6363

64+
# Unlike Linux, Android does not implicitly use a dlopened library to resolve
65+
# relocations in subsequently-loaded libraries, even if RTLD_GLOBAL is used
66+
# (https://github.com/android/ndk/issues/1244). So any library that fails to
67+
# build with this flag, would also fail to load at runtime.
68+
LDFLAGS="$LDFLAGS -Wl,--no-undefined"
69+
6470
# Many packages get away with omitting -lm on Linux, but Android is stricter.
6571
LDFLAGS="$LDFLAGS -lm"
6672

‎Android/android.py

Lines changed: 40 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@
77
import subprocess
88
import sys
99
import sysconfig
10-
from os.path import relpath
10+
from os.path import basename, relpath
1111
from pathlib import Path
12+
from tempfile import TemporaryDirectory
1213

1314
SCRIPT_NAME = Path(__file__).name
1415
CHECKOUT = Path(__file__).resolve().parent.parent
@@ -102,11 +103,17 @@ def unpack_deps(host):
102103
for name_ver in ["bzip2-1.0.8-1", "libffi-3.4.4-2", "openssl-3.0.13-1",
103104
"sqlite-3.45.1-0", "xz-5.4.6-0"]:
104105
filename = f"{name_ver}-{host}.tar.gz"
105-
run(["wget", f"{deps_url}/{name_ver}/{filename}"])
106+
download(f"{deps_url}/{name_ver}/{filename}")
106107
run(["tar", "-xf", filename])
107108
os.remove(filename)
108109

109110

111+
def download(url, target_dir="."):
112+
out_path = f"{target_dir}/{basename(url)}"
113+
run(["curl", "-Lf", "-o", out_path, url])
114+
return out_path
115+
116+
110117
def configure_host_python(context):
111118
host_dir = subdir(context.host, clean=context.clean)
112119

@@ -160,6 +167,30 @@ def clean_all(context):
160167
delete_if_exists(CROSS_BUILD_DIR)
161168

162169

170+
# To avoid distributing compiled artifacts without corresponding source code,
171+
# the Gradle wrapper is not included in the CPython repository. Instead, we
172+
# extract it from the Gradle release.
173+
def setup_testbed(context):
174+
ver_long = "8.7.0"
175+
ver_short = ver_long.removesuffix(".0")
176+
testbed_dir = CHECKOUT / "Android/testbed"
177+
178+
for filename in ["gradlew", "gradlew.bat"]:
179+
out_path = download(
180+
f"https://raw.githubusercontent.com/gradle/gradle/v{ver_long}/{filename}",
181+
testbed_dir)
182+
os.chmod(out_path, 0o755)
183+
184+
with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir:
185+
os.chdir(temp_dir)
186+
bin_zip = download(
187+
f"https://services.gradle.org/distributions/gradle-{ver_short}-bin.zip")
188+
outer_jar = f"gradle-{ver_short}/lib/plugins/gradle-wrapper-{ver_short}.jar"
189+
run(["unzip", bin_zip, outer_jar])
190+
run(["unzip", "-o", "-d", f"{testbed_dir}/gradle/wrapper", outer_jar,
191+
"gradle-wrapper.jar"])
192+
193+
163194
def main():
164195
parser = argparse.ArgumentParser()
165196
subcommands = parser.add_subparsers(dest="subcommand")
@@ -173,8 +204,11 @@ def main():
173204
help="Run `configure` for Android")
174205
make_host = subcommands.add_parser("make-host",
175206
help="Run `make` for Android")
176-
clean = subcommands.add_parser("clean", help="Delete files and directories "
177-
"created by this script")
207+
subcommands.add_parser(
208+
"clean", help="Delete the cross-build directory")
209+
subcommands.add_parser(
210+
"setup-testbed", help="Download the testbed Gradle wrapper")
211+
178212
for subcommand in build, configure_build, configure_host:
179213
subcommand.add_argument(
180214
"--clean", action="store_true", default=False, dest="clean",
@@ -194,7 +228,8 @@ def main():
194228
"configure-host": configure_host_python,
195229
"make-host": make_host_python,
196230
"build": build_all,
197-
"clean": clean_all}
231+
"clean": clean_all,
232+
"setup-testbed": setup_testbed}
198233
dispatch[context.subcommand](context)
199234

200235

0 commit comments

Comments
 (0)