diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8e09808f08bba2..2720ac30e1cc6a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "image": "ghcr.io/python/devcontainer:2025.05.29.15334414373", + "image": "ghcr.io/python/devcontainer:latest", "onCreateCommand": [ // Install common tooling. "dnf", diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 02a7b5d45b4627..57022de2b0e5ca 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,17 +4,23 @@ # It uses the same pattern rule for gitignore file # https://git-scm.com/docs/gitignore#_pattern_format -# GitHub +# Azure Pipelines +.azure-pipelines/ @AA-Turner + +# GitHub & related scripts .github/** @ezio-melotti @hugovk @AA-Turner +Tools/build/compute-changes.py @AA-Turner +Tools/build/verify_ensurepip_wheels.py @AA-Turner # pre-commit -.pre-commit-config.yaml @hugovk @AlexWaygood +.pre-commit-config.yaml @hugovk .ruff.toml @hugovk @AlexWaygood @AA-Turner -# Build system -configure* @erlend-aasland @corona10 -Makefile.pre.in @erlend-aasland -Modules/Setup* @erlend-aasland +# Build system (autotools) +configure* @erlend-aasland @corona10 @AA-Turner +Makefile.pre.in @erlend-aasland @AA-Turner +Modules/Setup* @erlend-aasland @AA-Turner +Tools/build/regen-configure.sh @AA-Turner # argparse **/*argparse* @savannahostrowski @@ -67,6 +73,7 @@ Doc/make.bat @AA-Turner @hugovk Doc/requirements.txt @AA-Turner @hugovk Doc/_static/** @AA-Turner @hugovk Doc/tools/** @AA-Turner @hugovk +.readthedocs.yml @AA-Turner # runtime state/lifecycle **/*pylifecycle* @ericsnowcurrently @ZeroIntensity @@ -155,6 +162,10 @@ Doc/c-api/module.rst @ericsnowcurrently **/*importlib/resources/* @jaraco @warsaw @FFY00 **/*importlib/metadata/* @jaraco @warsaw +# Calendar +Lib/calendar.py @AA-Turner +Lib/test/test_calendar.py @AA-Turner + # Dates and times **/*datetime* @pganssle @abalkin **/*str*time* @pganssle @abalkin @@ -205,6 +216,11 @@ Lib/test/test_ast/ @eclips4 @tomasr8 # multiprocessing **/*multiprocessing* @gpshead +# pydoc +Lib/pydoc.py @AA-Turner +Lib/pydoc_data/ @AA-Turner +Lib/test/test_pydoc/ @AA-Turner + # SQLite 3 **/*sqlite* @berkerpeksag @erlend-aasland @@ -217,6 +233,11 @@ Lib/test/test_ast/ @eclips4 @tomasr8 **/*pdb* @gaogaotiantian **/*bdb* @gaogaotiantian +# types +Lib/test/test_types.py @AA-Turner +Lib/types.py @AA-Turner +Modules/_typesmodule.c @AA-Turner + # Limited C API & stable ABI Tools/build/stable_abi.py @encukou Misc/stable_abi.toml @encukou @@ -234,6 +255,11 @@ Doc/c-api/stable.rst @encukou /Tools/msi/ @python/windows-team /Tools/nuget/ @python/windows-team +# Zstandard +Lib/compression/zstd/ @AA-Turner +Lib/test/test_zstd.py @AA-Turner +Modules/_zstd/ @AA-Turner + # Misc **/*itertools* @rhettinger **/*collections* @rhettinger @@ -266,6 +292,9 @@ Doc/c-api/stable.rst @encukou **/*cjkcodecs* @corona10 +# Patchcheck +Tools/patchcheck/ @AA-Turner + # macOS /Mac/ @python/macos-team **/*osx_support* @python/macos-team @@ -277,9 +306,9 @@ Doc/c-api/stable.rst @encukou **/*zipfile/_path/* @jaraco # Argument Clinic -/Tools/clinic/** @erlend-aasland -/Lib/test/test_clinic.py @erlend-aasland -Doc/howto/clinic.rst @erlend-aasland +/Tools/clinic/** @erlend-aasland @AA-Turner +/Lib/test/test_clinic.py @erlend-aasland @AA-Turner +Doc/howto/clinic.rst @erlend-aasland @AA-Turner # Subinterpreters **/*interpreteridobject.* @ericsnowcurrently @@ -323,6 +352,7 @@ Lib/test/test_configparser.py @jaraco # Doc sections Doc/reference/ @willingc @AA-Turner +Doc/whatsnew/ @AA-Turner **/*weakref* @kumaraditya303 @@ -336,7 +366,7 @@ Modules/_xxtestfuzz/ @ammaraskar # t-strings **/*interpolationobject* @lysnikolaou **/*templateobject* @lysnikolaou -**/*templatelib* @lysnikolaou +**/*templatelib* @lysnikolaou @AA-Turner **/*tstring* @lysnikolaou # Remote debugging @@ -346,3 +376,6 @@ Modules/_remote_debugging_module.c @pablogsal @ambv @1st1 # gettext **/*gettext* @tomasr8 + +# Internal Docs +InternalDocs/ @AA-Turner diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 2ef9cdc191481e..5b86302bdd1ec4 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -4,7 +4,7 @@ Contributing to Python Build Status ------------ -- `Buildbot status overview `_ +- `Buildbot status overview `_ - `GitHub Actions status `_ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f07f5e8040acf0..56f222cd94ab6f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -178,8 +178,8 @@ jobs: free-threading: ${{ matrix.free-threading }} build-windows-msi: - name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category - Windows MSI${{ '' }} + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: Windows MSI${{ '' }} # zizmor: ignore[obfuscation] needs: build-context if: fromJSON(needs.build-context.outputs.run-windows-msi) strategy: @@ -397,6 +397,29 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py + build-android: + name: Android (${{ matrix.arch }}) + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + # Use the same runs-on configuration as build-macos and build-ubuntu. + - arch: aarch64 + runs-on: ${{ github.repository_owner == 'python' && 'ghcr.io/cirruslabs/macos-runner:sonoma' || 'macos-14' }} + - arch: x86_64 + runs-on: ubuntu-24.04 + + runs-on: ${{ matrix.runs-on }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Build and test + run: ./Android/android.py ci ${{ matrix.arch }}-linux-android + build-wasi: name: 'WASI' needs: build-context @@ -586,8 +609,8 @@ jobs: run: xvfb-run make ci build-san: - name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category - Sanitizers${{ '' }} + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: Sanitizers${{ '' }} # zizmor: ignore[obfuscation] needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: @@ -705,6 +728,7 @@ jobs: - build-ubuntu - build-ubuntu-ssltests-awslc - build-ubuntu-ssltests-openssl + - build-android - build-wasi - test-hypothesis - build-asan @@ -740,6 +764,7 @@ jobs: build-ubuntu, build-ubuntu-ssltests-awslc, build-ubuntu-ssltests-openssl, + build-android, build-wasi, test-hypothesis, build-asan, diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 95133c1338b682..0413d5f905d771 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -13,9 +13,12 @@ on: - "Lib/test/libregrtest/**" - "Lib/tomllib/**" - "Misc/mypy/**" + - "Tools/build/mypy.ini" + - "Tools/build/check_extension_modules.py" - "Tools/build/compute-changes.py" - "Tools/build/deepfreeze.py" - "Tools/build/generate_sbom.py" + - "Tools/build/generate_stdlib_module_names.py" - "Tools/build/generate-build-details.py" - "Tools/build/verify_ensurepip_wheels.py" - "Tools/build/update_file.py" diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 7b9dc4818577eb..65154aae4c41d5 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -75,18 +75,6 @@ jobs: --fail-if-regression \ --fail-if-improved \ --fail-if-new-news-nit - - name: 'Build EPUB documentation' - continue-on-error: true - run: | - set -Eeuo pipefail - make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet" epub - pip install epubcheck - epubcheck Doc/build/epub/Python.epub &> Doc/epubcheck.txt - - name: 'Check for fatal errors in EPUB' - if: github.event_name == 'pull_request' - continue-on-error: true # until gh-136155 is fixed - run: | - python Doc/tools/check-epub.py # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: @@ -114,3 +102,30 @@ jobs: # Use "xvfb-run" since some doctest tests open GUI windows - name: 'Run documentation doctest' run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="--fail-on-warning" doctest + + check-epub: + name: 'Check EPUB' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: 'Set up Python' + uses: actions/setup-python@v5 + with: + python-version: '3' + cache: 'pip' + cache-dependency-path: 'Doc/requirements.txt' + - name: 'Install build dependencies' + run: | + make -C Doc/ venv + python -m pip install epubcheck + - name: 'Build EPUB documentation' + run: make -C Doc/ PYTHON=../python epub + - name: 'Run epubcheck' + continue-on-error: true + run: epubcheck Doc/build/epub/Python.epub &> Doc/epubcheck.txt + - run: cat Doc/epubcheck.txt + - name: 'Check for fatal errors in EPUB' + run: python Doc/tools/check-epub.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 86410c46d1d707..d101f5c37e60b1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.8 + rev: v0.12.8 hooks: - id: ruff name: Run Ruff (lint) on Doc/ @@ -42,7 +42,7 @@ repos: exclude: ^Tools/c-analyzer/cpython/_parser.py - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-case-conflict - id: check-merge-conflict @@ -60,7 +60,7 @@ repos: files: '^\.github/CODEOWNERS|\.(gram)$' - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.33.0 + rev: 0.33.2 hooks: - id: check-dependabot - id: check-github-workflows @@ -72,7 +72,7 @@ repos: - id: actionlint - repo: https://github.com/woodruffw/zizmor-pre-commit - rev: v1.6.0 + rev: v1.11.0 hooks: - id: zizmor diff --git a/.well-known/funding-manifest-urls b/.well-known/funding-manifest-urls new file mode 100644 index 00000000000000..d56ca9eb52f004 --- /dev/null +++ b/.well-known/funding-manifest-urls @@ -0,0 +1 @@ +https://www.python.org/funding.json diff --git a/Android/README.md b/Android/README.md index c42eb627006e6a..9f71aeb934f386 100644 --- a/Android/README.md +++ b/Android/README.md @@ -96,10 +96,12 @@ similar to the `Android` directory of the CPython source tree. ## Testing -The Python test suite can be run on Linux, macOS, or Windows: +The Python test suite can be run on Linux, macOS, or Windows. -* On Linux, the emulator needs access to the KVM virtualization interface, and - a DISPLAY environment variable pointing at an X server. Xvfb is acceptable. +On Linux, the emulator needs access to the KVM virtualization interface. This may +require adding your user to a group, or changing your udev rules. On GitHub +Actions, the test script will do this automatically using the commands shown +[here](https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). The test suite can usually be run on a device with 2 GB of RAM, but this is borderline, so you may need to increase it to 4 GB. As of Android diff --git a/Android/android-env.sh b/Android/android-env.sh index 7b381a013cf0ba..5859c0eac4a88f 100644 --- a/Android/android-env.sh +++ b/Android/android-env.sh @@ -24,7 +24,7 @@ fail() { # * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md # where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.: # https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md -ndk_version=27.2.12479018 +ndk_version=27.3.13750724 ndk=$ANDROID_HOME/ndk/$ndk_version if ! [ -e "$ndk" ]; then diff --git a/Android/android.py b/Android/android.py index 75f73cd30993da..85874ad9b60f3d 100755 --- a/Android/android.py +++ b/Android/android.py @@ -3,6 +3,7 @@ import asyncio import argparse import os +import platform import re import shlex import shutil @@ -187,7 +188,7 @@ def unpack_deps(host, prefix_dir): os.chdir(prefix_dir) deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.15-4", - "sqlite-3.49.1-0", "xz-5.4.6-1", "zstd-1.5.7-1"]: + "sqlite-3.50.4-0", "xz-5.4.6-1", "zstd-1.5.7-1"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") shutil.unpack_archive(filename) @@ -247,7 +248,13 @@ def make_host_python(context): # flags to be duplicated. So we don't use the `host` argument here. os.chdir(host_dir) run(["make", "-j", str(os.cpu_count())]) - run(["make", "install", f"prefix={prefix_dir}"]) + + # The `make install` output is very verbose and rarely useful, so + # suppress it by default. + run( + ["make", "install", f"prefix={prefix_dir}"], + capture_output=not context.verbose, + ) def build_all(context): @@ -266,6 +273,18 @@ def clean_all(context): clean(host) +def setup_ci(): + # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/ + if "GITHUB_ACTIONS" in os.environ and platform.system() == "Linux": + run( + ["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"], + input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n', + text=True, + ) + run(["sudo", "udevadm", "control", "--reload-rules"]) + run(["sudo", "udevadm", "trigger", "--name-match=kvm"]) + + def setup_sdk(): sdkmanager = android_home / ( "cmdline-tools/latest/bin/sdkmanager" @@ -578,6 +597,7 @@ async def gradle_task(context): async def run_testbed(context): + setup_ci() setup_sdk() setup_testbed() @@ -671,11 +691,63 @@ def package(context): else: shutil.copy2(src, dst, follow_symlinks=False) + # Strip debug information. + if not context.debug: + so_files = glob(f"{temp_dir}/**/*.so", recursive=True) + run([android_env(context.host)["STRIP"], *so_files], log=False) + dist_dir = subdir(context.host, "dist", create=True) package_path = shutil.make_archive( f"{dist_dir}/python-{version}-{context.host}", "gztar", temp_dir ) print(f"Wrote {package_path}") + return package_path + + +def ci(context): + for step in [ + configure_build_python, + make_build_python, + configure_host_python, + make_host_python, + package, + ]: + caption = ( + step.__name__.replace("_", " ") + .capitalize() + .replace("python", "Python") + ) + print(f"::group::{caption}") + result = step(context) + if step is package: + package_path = result + print("::endgroup::") + + if ( + "GITHUB_ACTIONS" in os.environ + and (platform.system(), platform.machine()) != ("Linux", "x86_64") + ): + print( + "Skipping tests: GitHub Actions does not support the Android " + "emulator on this platform." + ) + else: + with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: + print("::group::Tests") + # Prove the package is self-contained by using it to run the tests. + shutil.unpack_archive(package_path, temp_dir) + + # Arguments are similar to --fast-ci, but in single-process mode. + launcher_args = ["--managed", "maxVersion", "-v"] + test_args = [ + "--single-process", "--fail-env-changed", "--rerun", "--slowest", + "--verbose3", "-u", "all,-cpu", "--timeout=600" + ] + run( + ["./android.py", "test", *launcher_args, "--", *test_args], + cwd=temp_dir + ) + print("::endgroup::") def env(context): @@ -695,32 +767,40 @@ def parse_args(): parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand", required=True) + def add_parser(*args, **kwargs): + parser = subcommands.add_parser(*args, **kwargs) + parser.add_argument( + "-v", "--verbose", action="count", default=0, + help="Show verbose output. Use twice to be even more verbose.") + return parser + # Subcommands - build = subcommands.add_parser( + build = add_parser( "build", help="Run configure-build, make-build, configure-host and " "make-host") - configure_build = subcommands.add_parser( + configure_build = add_parser( "configure-build", help="Run `configure` for the build Python") - subcommands.add_parser( + add_parser( "make-build", help="Run `make` for the build Python") - configure_host = subcommands.add_parser( + configure_host = add_parser( "configure-host", help="Run `configure` for Android") - make_host = subcommands.add_parser( + make_host = add_parser( "make-host", help="Run `make` for Android") - subcommands.add_parser("clean", help="Delete all build directories") - subcommands.add_parser("build-testbed", help="Build the testbed app") - test = subcommands.add_parser("test", help="Run the testbed app") - package = subcommands.add_parser("package", help="Make a release package") - env = subcommands.add_parser("env", help="Print environment variables") + add_parser("clean", help="Delete all build directories") + add_parser("build-testbed", help="Build the testbed app") + test = add_parser("test", help="Run the testbed app") + package = add_parser("package", help="Make a release package") + ci = add_parser("ci", help="Run build, package and test") + env = add_parser("env", help="Print environment variables") # Common arguments - for subcommand in build, configure_build, configure_host: + for subcommand in [build, configure_build, configure_host, ci]: subcommand.add_argument( "--clean", action="store_true", default=False, dest="clean", help="Delete the relevant build directories first") - host_commands = [build, configure_host, make_host, package] + host_commands = [build, configure_host, make_host, package, ci] if in_source_tree: host_commands.append(env) for subcommand in host_commands: @@ -728,16 +808,11 @@ def parse_args(): "host", metavar="HOST", choices=HOSTS, help="Host triplet: choices=[%(choices)s]") - for subcommand in build, configure_build, configure_host: + for subcommand in [build, configure_build, configure_host, ci]: subcommand.add_argument("args", nargs="*", help="Extra arguments to pass to `configure`") # Test arguments - test.add_argument( - "-v", "--verbose", action="count", default=0, - help="Show Gradle output, and non-Python logcat messages. " - "Use twice to include high-volume messages which are rarely useful.") - device_group = test.add_mutually_exclusive_group(required=True) device_group.add_argument( "--connected", metavar="SERIAL", help="Run on a connected device. " @@ -765,6 +840,12 @@ def parse_args(): "args", nargs="*", help=f"Arguments to add to sys.argv. " f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.") + # Package arguments. + for subcommand in [package, ci]: + subcommand.add_argument( + "-g", action="store_true", default=False, dest="debug", + help="Include debug information in package") + return parser.parse_args() @@ -788,6 +869,7 @@ def main(): "build-testbed": build_testbed, "test": run_testbed, "package": package, + "ci": ci, "env": env, } @@ -803,6 +885,8 @@ def main(): def print_called_process_error(e): for stream_name in ["stdout", "stderr"]: content = getattr(e, stream_name) + if isinstance(content, bytes): + content = content.decode(*DECODE_ARGS) stream = getattr(sys, stream_name) if content: stream.write(content) diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 16bd79475dc1e6..aa91b85d07fc7e 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -3,92 +3,24 @@ .. _complexobjects: Complex Number Objects ----------------------- +====================== .. index:: pair: object; complex number -Python's complex number objects are implemented as two distinct types when -viewed from the C API: one is the Python object exposed to Python programs, and -the other is a C structure which represents the actual complex number value. -The API provides functions for working with both. - - -Complex Numbers as C Structures -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Note that the functions which accept these structures as parameters and return -them as results do so *by value* rather than dereferencing them through -pointers. This is consistent throughout the API. - - -.. c:type:: Py_complex - - The C structure which corresponds to the value portion of a Python complex - number object. Most of the functions for dealing with complex number objects - use structures of this type as input or output values, as appropriate. - - .. c:member:: double real - double imag - - The structure is defined as:: - - typedef struct { - double real; - double imag; - } Py_complex; - - -.. c:function:: Py_complex _Py_c_sum(Py_complex left, Py_complex right) - - Return the sum of two complex numbers, using the C :c:type:`Py_complex` - representation. - - -.. c:function:: Py_complex _Py_c_diff(Py_complex left, Py_complex right) - - Return the difference between two complex numbers, using the C - :c:type:`Py_complex` representation. - - -.. c:function:: Py_complex _Py_c_neg(Py_complex num) - - Return the negation of the complex number *num*, using the C - :c:type:`Py_complex` representation. - - -.. c:function:: Py_complex _Py_c_prod(Py_complex left, Py_complex right) - - Return the product of two complex numbers, using the C :c:type:`Py_complex` - representation. - - -.. c:function:: Py_complex _Py_c_quot(Py_complex dividend, Py_complex divisor) - - Return the quotient of two complex numbers, using the C :c:type:`Py_complex` - representation. - - If *divisor* is null, this method returns zero and sets - :c:data:`errno` to :c:macro:`!EDOM`. - - -.. c:function:: Py_complex _Py_c_pow(Py_complex num, Py_complex exp) - - Return the exponentiation of *num* by *exp*, using the C :c:type:`Py_complex` - representation. - - If *num* is null and *exp* is not a positive real number, - this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`. - - Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. +.. c:type:: PyComplexObject -Complex Numbers as Python Objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + This subtype of :c:type:`PyObject` represents a Python complex number object. + .. c:member:: Py_complex cval -.. c:type:: PyComplexObject + The complex number value, using the C :c:type:`Py_complex` representation. - This subtype of :c:type:`PyObject` represents a Python complex number object. + .. deprecated-removed:: next 3.20 + Use :c:func:`PyComplex_AsCComplex` and + :c:func:`PyComplex_FromCComplex` to convert a + Python complex number to/from the C :c:type:`Py_complex` + representation. .. c:var:: PyTypeObject PyComplex_Type @@ -109,12 +41,6 @@ Complex Numbers as Python Objects :c:type:`PyComplexObject`. This function always succeeds. -.. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) - - Create a new Python complex number object from a C :c:type:`Py_complex` value. - Return ``NULL`` with an exception set on error. - - .. c:function:: PyObject* PyComplex_FromDoubles(double real, double imag) Return a new :c:type:`PyComplexObject` object from *real* and *imag*. @@ -153,6 +79,29 @@ Complex Numbers as Python Objects .. versionchanged:: 3.13 Use :meth:`~object.__complex__` if available. + +.. c:type:: Py_complex + + This C structure defines export format for a Python complex + number object. + + .. c:member:: double real + double imag + + The structure is defined as:: + + typedef struct { + double real; + double imag; + } Py_complex; + + +.. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) + + Create a new Python complex number object from a C :c:type:`Py_complex` value. + Return ``NULL`` with an exception set on error. + + .. c:function:: Py_complex PyComplex_AsCComplex(PyObject *op) Return the :c:type:`Py_complex` value of the complex number *op*. @@ -169,3 +118,82 @@ Complex Numbers as Python Objects .. versionchanged:: 3.8 Use :meth:`~object.__index__` if available. + + +Complex Numbers as C Structures +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The API also provides functions for working with complex numbers, using the +:c:type:`Py_complex` representation. Note that the functions which accept +these structures as parameters and return them as results do so *by value* +rather than dereferencing them through pointers. + +Please note, that these functions are :term:`soft deprecated` since Python +3.15. Avoid using this API in a new code to do complex arithmetic: either use +the `Number Protocol `_ API or use native complex types, like +:c:expr:`double complex`. + + +.. c:function:: Py_complex _Py_c_sum(Py_complex left, Py_complex right) + + Return the sum of two complex numbers, using the C :c:type:`Py_complex` + representation. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_diff(Py_complex left, Py_complex right) + + Return the difference between two complex numbers, using the C + :c:type:`Py_complex` representation. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_neg(Py_complex num) + + Return the negation of the complex number *num*, using the C + :c:type:`Py_complex` representation. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_prod(Py_complex left, Py_complex right) + + Return the product of two complex numbers, using the C :c:type:`Py_complex` + representation. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_quot(Py_complex dividend, Py_complex divisor) + + Return the quotient of two complex numbers, using the C :c:type:`Py_complex` + representation. + + If *divisor* is null, this method returns zero and sets + :c:data:`errno` to :c:macro:`!EDOM`. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_pow(Py_complex num, Py_complex exp) + + Return the exponentiation of *num* by *exp*, using the C :c:type:`Py_complex` + representation. + + If *num* is null and *exp* is not a positive real number, + this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`. + + Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. + + .. deprecated:: 3.15 + + +.. c:function:: double _Py_c_abs(Py_complex num) + + Return the absolute value of the complex number *num*. + + Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. + + .. deprecated:: 3.15 diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index 00f8cb887dc7eb..b5fe93573a1456 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -51,7 +51,7 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. Hash function definition used by :c:func:`PyHash_GetFuncDef`. - .. c::member:: Py_hash_t (*const hash)(const void *, Py_ssize_t) + .. c:member:: Py_hash_t (*const hash)(const void *, Py_ssize_t) Hash function. diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 2d0bda76697e81..31e2cd57034467 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -372,6 +372,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Set *\*value* to a signed C :c:expr:`int32_t` or :c:expr:`int64_t` representation of *obj*. + If *obj* is not an instance of :c:type:`PyLongObject`, first call its + :meth:`~object.__index__` method (if present) to convert it to a + :c:type:`PyLongObject`. + If the *obj* value is out of range, raise an :exc:`OverflowError`. Set *\*value* and return ``0`` on success. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 55f0d0f9fb7ff8..78599e704b1317 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -197,7 +197,7 @@ Object Protocol in favour of using :c:func:`PyObject_DelAttr`, but there are currently no plans to remove it. - The function must not be called with ``NULL`` *v* and an an exception set. + The function must not be called with a ``NULL`` *v* and an exception set. This case can arise from forgetting ``NULL`` checks and would delete the attribute. @@ -214,7 +214,7 @@ Object Protocol If *v* is ``NULL``, the attribute is deleted, but this feature is deprecated in favour of using :c:func:`PyObject_DelAttrString`. - The function must not be called with ``NULL`` *v* and an an exception set. + The function must not be called with a ``NULL`` *v* and an exception set. This case can arise from forgetting ``NULL`` checks and would delete the attribute. diff --git a/Doc/conf.py b/Doc/conf.py index 1c1f36e5bc0737..35e0b3eaeafe94 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -567,14 +567,11 @@ 'image': '_static/og-image.png', 'line_color': '#3776ab', } -if 'builder_html' in tags: # noqa: F821 - ogp_custom_meta_tags = [ - '', - ] - if 'create-social-cards' not in tags: # noqa: F821 - # Define a static preview image when not creating social cards - ogp_image = '_static/og-image.png' - ogp_custom_meta_tags += [ - '', - '', - ] +ogp_custom_meta_tags = ('',) +if 'create-social-cards' not in tags: # noqa: F821 + # Define a static preview image when not creating social cards + ogp_image = '_static/og-image.png' + ogp_custom_meta_tags += ( + '', + '', + ) diff --git a/Doc/deprecations/c-api-pending-removal-in-3.20.rst b/Doc/deprecations/c-api-pending-removal-in-3.20.rst new file mode 100644 index 00000000000000..82f975d6ed4020 --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.20.rst @@ -0,0 +1,7 @@ +Pending removal in Python 3.20 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* The ``cval`` field in :c:type:`PyComplexObject` (:gh:`128813`). + Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` + to convert a Python complex number to/from the C :c:type:`Py_complex` + representation. diff --git a/Doc/deprecations/index.rst b/Doc/deprecations/index.rst index d064f2bec42c22..c6e05c176b2aa1 100644 --- a/Doc/deprecations/index.rst +++ b/Doc/deprecations/index.rst @@ -18,4 +18,6 @@ C API deprecations .. include:: c-api-pending-removal-in-3.18.rst +.. include:: c-api-pending-removal-in-3.20.rst + .. include:: c-api-pending-removal-in-future.rst diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index a89a69043c0f9f..17c6fb224265ca 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -75,12 +75,37 @@ the module and a copyright notice if you like). See :ref:`arg-parsing-string-and-buffers` for a description of this macro. All user-visible symbols defined by :file:`Python.h` have a prefix of ``Py`` or -``PY``, except those defined in standard header files. For convenience, and -since they are used extensively by the Python interpreter, ``"Python.h"`` -includes a few standard header files: ````, ````, -````, and ````. If the latter header file does not exist on -your system, it declares the functions :c:func:`malloc`, :c:func:`free` and -:c:func:`realloc` directly. +``PY``, except those defined in standard header files. + +.. tip:: + + For backward compatibility, :file:`Python.h` includes several standard header files. + C extensions should include the standard headers that they use, + and should not rely on these implicit includes. + If using the limited C API version 3.13 or newer, the implicit includes are: + + * ```` + * ```` (on Windows) + * ```` + * ```` + * ```` + * ```` + * ```` + * ```` (if present) + + If :c:macro:`Py_LIMITED_API` is not defined, or is set to version 3.12 or older, + the headers below are also included: + + * ```` + * ```` (on POSIX) + + If :c:macro:`Py_LIMITED_API` is not defined, or is set to version 3.10 or older, + the headers below are also included: + + * ```` + * ```` + * ```` + * ```` The next thing we add to our module file is the C function that will be called when the Python expression ``spam.system(string)`` is evaluated (we'll see diff --git a/Doc/glossary.rst b/Doc/glossary.rst index b7bd547d38fd1e..69e8857af894e4 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -435,6 +435,11 @@ Glossary with :term:`abstract base classes `.) Instead, it typically employs :func:`hasattr` tests or :term:`EAFP` programming. + dunder + An informal short-hand for "double underscore", used when talking about a + :term:`special method`. For example, ``__init__`` is often pronounced + "dunder init". + EAFP Easier to ask for forgiveness than permission. This common Python coding style assumes the existence of valid keys or attributes and catches @@ -462,6 +467,7 @@ Glossary core and with user code. f-string + f-strings String literals prefixed with ``f`` or ``F`` are commonly called "f-strings" which is short for :ref:`formatted string literals `. See also :pep:`498`. @@ -1323,6 +1329,7 @@ Glossary See also :term:`borrowed reference`. t-string + t-strings String literals prefixed with ``t`` or ``T`` are commonly called "t-strings" which is short for :ref:`template string literals `. @@ -1472,6 +1479,11 @@ Glossary A computer defined entirely in software. Python's virtual machine executes the :term:`bytecode` emitted by the bytecode compiler. + walrus operator + A light-hearted way to refer to the :ref:`assignment expression + ` operator ``:=`` because it looks a bit like a + walrus if you turn your head. + Zen of Python Listing of Python design principles and philosophies that are helpful in understanding and using the language. The listing can be found by typing diff --git a/Doc/howto/a-conceptual-overview-of-asyncio.rst b/Doc/howto/a-conceptual-overview-of-asyncio.rst new file mode 100644 index 00000000000000..d68f7cc6921fc9 --- /dev/null +++ b/Doc/howto/a-conceptual-overview-of-asyncio.rst @@ -0,0 +1,606 @@ +.. _a-conceptual-overview-of-asyncio: + +**************************************** +A Conceptual Overview of :mod:`!asyncio` +**************************************** + +This :ref:`HOWTO ` article seeks to help you build a sturdy mental +model of how :mod:`asyncio` fundamentally works, helping you understand the +how and why behind the recommended patterns. + +You might be curious about some key :mod:`!asyncio` concepts. +You'll be comfortably able to answer these questions by the end of this +article: + +- What's happening behind the scenes when an object is awaited? +- How does :mod:`!asyncio` differentiate between a task which doesn't need + CPU-time (such as a network request or file read) as opposed to a task that + does (such as computing n-factorial)? +- How to write an asynchronous variant of an operation, such as + an async sleep or database request. + +.. seealso:: + + * The `guide `_ that inspired this HOWTO article, by Alexander Nordin. + * This in-depth `YouTube tutorial series `_ on + ``asyncio`` created by Python core team member, Łukasz Langa. + * `500 Lines or Less: A Web Crawler With asyncio Coroutines `_ by A. + Jesse Jiryu Davis and Guido van Rossum. + +-------------------------------------------- +A conceptual overview part 1: the high-level +-------------------------------------------- + +In part 1, we'll cover the main, high-level building blocks of :mod:`!asyncio`: +the event loop, coroutine functions, coroutine objects, tasks and ``await``. + +========== +Event Loop +========== + +Everything in :mod:`!asyncio` happens relative to the event loop. +It's the star of the show. +It's like an orchestra conductor. +It's behind the scenes managing resources. +Some power is explicitly granted to it, but a lot of its ability to get things +done comes from the respect and cooperation of its worker bees. + +In more technical terms, the event loop contains a collection of jobs to be run. +Some jobs are added directly by you, and some indirectly by :mod:`!asyncio`. +The event loop takes a job from its backlog of work and invokes it (or "gives +it control"), similar to calling a function, and then that job runs. +Once it pauses or completes, it returns control to the event loop. +The event loop will then select another job from its pool and invoke it. +You can *roughly* think of the collection of jobs as a queue: jobs are added and +then processed one at a time, generally (but not always) in order. +This process repeats indefinitely with the event loop cycling endlessly +onwards. +If there are no more jobs pending execution, the event loop is smart enough to +rest and avoid needlessly wasting CPU cycles, and will come back when there's +more work to be done. + +Effective execution relies on jobs sharing well and cooperating; a greedy job +could hog control and leave the other jobs to starve, rendering the overall +event loop approach rather useless. + +:: + + import asyncio + + # This creates an event loop and indefinitely cycles through + # its collection of jobs. + event_loop = asyncio.new_event_loop() + event_loop.run_forever() + +===================================== +Asynchronous functions and coroutines +===================================== + +This is a basic, boring Python function:: + + def hello_printer(): + print( + "Hi, I am a lowly, simple printer, though I have all I " + "need in life -- \nfresh paper and my dearly beloved octopus " + "partner in crime." + ) + +Calling a regular function invokes its logic or body:: + + >>> hello_printer() + Hi, I am a lowly, simple printer, though I have all I need in life -- + fresh paper and my dearly beloved octopus partner in crime. + +The :ref:`async def `, as opposed to just a plain ``def``, makes +this an asynchronous function (or "coroutine function"). +Calling it creates and returns a :ref:`coroutine ` object. + +:: + + async def loudmouth_penguin(magic_number: int): + print( + "I am a super special talking penguin. Far cooler than that printer. " + f"By the way, my lucky number is: {magic_number}." + ) + +Calling the async function, ``loudmouth_penguin``, does not execute the print statement; +instead, it creates a coroutine object:: + + >>> loudmouth_penguin(magic_number=3) + + +The terms "coroutine function" and "coroutine object" are often conflated +as coroutine. +That can be confusing! +In this article, coroutine specifically refers to a coroutine object, or more +precisely, an instance of :data:`types.CoroutineType` (native coroutine). +Note that coroutines can also exist as instances of +:class:`collections.abc.Coroutine` -- a distinction that matters for type +checking. + +A coroutine represents the function's body or logic. +A coroutine has to be explicitly started; again, merely creating the coroutine +does not start it. +Notably, the coroutine can be paused and resumed at various points within the +function's body. +That pausing and resuming ability is what allows for asynchronous behavior! + +Coroutines and coroutine functions were built by leveraging the functionality +of :term:`generators ` and +:term:`generator functions `. +Recall, a generator function is a function that :keyword:`yield`\s, like this +one:: + + def get_random_number(): + # This would be a bad random number generator! + print("Hi") + yield 1 + print("Hello") + yield 7 + print("Howdy") + yield 4 + ... + +Similar to a coroutine function, calling a generator function does not run it. +Instead, it creates a generator object:: + + >>> get_random_number() + + +You can proceed to the next ``yield`` of a generator by using the +built-in function :func:`next`. +In other words, the generator runs, then pauses. +For example:: + + >>> generator = get_random_number() + >>> next(generator) + Hi + 1 + >>> next(generator) + Hello + 7 + +===== +Tasks +===== + +Roughly speaking, :ref:`tasks ` are coroutines (not coroutine +functions) tied to an event loop. +A task also maintains a list of callback functions whose importance will become +clear in a moment when we discuss :keyword:`await`. +The recommended way to create tasks is via :func:`asyncio.create_task`. + +Creating a task automatically schedules it for execution (by adding a +callback to run it in the event loop's to-do list, that is, collection of jobs). + +Since there's only one event loop (in each thread), :mod:`!asyncio` takes care of +associating the task with the event loop for you. As such, there's no need +to specify the event loop. + +:: + + coroutine = loudmouth_penguin(magic_number=5) + # This creates a Task object and schedules its execution via the event loop. + task = asyncio.create_task(coroutine) + +Earlier, we manually created the event loop and set it to run forever. +In practice, it's recommended to use (and common to see) :func:`asyncio.run`, +which takes care of managing the event loop and ensuring the provided +coroutine finishes before advancing. +For example, many async programs follow this setup:: + + import asyncio + + async def main(): + # Perform all sorts of wacky, wild asynchronous things... + ... + + if __name__ == "__main__": + asyncio.run(main()) + # The program will not reach the following print statement until the + # coroutine main() finishes. + print("coroutine main() is done!") + +It's important to be aware that the task itself is not added to the event loop, +only a callback to the task is. +This matters if the task object you created is garbage collected before it's +called by the event loop. +For example, consider this program: + +.. code-block:: + :linenos: + + async def hello(): + print("hello!") + + async def main(): + asyncio.create_task(hello()) + # Other asynchronous instructions which run for a while + # and cede control to the event loop... + ... + + asyncio.run(main()) + +Because there's no reference to the task object created on line 5, it *might* +be garbage collected before the event loop invokes it. +Later instructions in the coroutine ``main()`` hand control back to the event +loop so it can invoke other jobs. +When the event loop eventually tries to run the task, it might fail and +discover the task object does not exist! +This can also happen even if a coroutine keeps a reference to a task but +completes before that task finishes. +When the coroutine exits, local variables go out of scope and may be subject +to garbage collection. +In practice, ``asyncio`` and Python's garbage collector work pretty hard to +ensure this sort of thing doesn't happen. +But that's no reason to be reckless! + +===== +await +===== + +:keyword:`await` is a Python keyword that's commonly used in one of two +different ways:: + + await task + await coroutine + +In a crucial way, the behavior of ``await`` depends on the type of object +being awaited. + +Awaiting a task will cede control from the current task or coroutine to +the event loop. +In the process of relinquishing control, a few important things happen. +We'll use the following code example to illustrate:: + + async def plant_a_tree(): + dig_the_hole_task = asyncio.create_task(dig_the_hole()) + await dig_the_hole_task + + # Other instructions associated with planting a tree. + ... + +In this example, imagine the event loop has passed control to the start of the +coroutine ``plant_a_tree()``. +As seen above, the coroutine creates a task and then awaits it. +The ``await dig_the_hole_task`` instruction adds a callback (which will resume +``plant_a_tree()``) to the ``dig_the_hole_task`` object's list of callbacks. +And then, the instruction cedes control to the event loop. +Some time later, the event loop will pass control to ``dig_the_hole_task`` +and the task will finish whatever it needs to do. +Once the task finishes, it will add its various callbacks to the event loop, +in this case, a call to resume ``plant_a_tree()``. + +Generally speaking, when the awaited task finishes (``dig_the_hole_task``), +the original task or coroutine (``plant_a_tree()``) is added back to the event +loops to-do list to be resumed. + +This is a basic, yet reliable mental model. +In practice, the control handoffs are slightly more complex, but not by much. +In part 2, we'll walk through the details that make this possible. + +**Unlike tasks, awaiting a coroutine does not hand control back to the event +loop!** +Wrapping a coroutine in a task first, then awaiting that would cede +control. +The behavior of ``await coroutine`` is effectively the same as invoking a +regular, synchronous Python function. +Consider this program:: + + import asyncio + + async def coro_a(): + print("I am coro_a(). Hi!") + + async def coro_b(): + print("I am coro_b(). I sure hope no one hogs the event loop...") + + async def main(): + task_b = asyncio.create_task(coro_b()) + num_repeats = 3 + for _ in range(num_repeats): + await coro_a() + await task_b + + asyncio.run(main()) + +The first statement in the coroutine ``main()`` creates ``task_b`` and schedules +it for execution via the event loop. +Then, ``coro_a()`` is repeatedly awaited. Control never cedes to the +event loop which is why we see the output of all three ``coro_a()`` +invocations before ``coro_b()``'s output: + +.. code-block:: none + + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_b(). I sure hope no one hogs the event loop... + +If we change ``await coro_a()`` to ``await asyncio.create_task(coro_a())``, the +behavior changes. +The coroutine ``main()`` cedes control to the event loop with that statement. +The event loop then proceeds through its backlog of work, calling ``task_b`` +and then the task which wraps ``coro_a()`` before resuming the coroutine +``main()``. + +.. code-block:: none + + I am coro_b(). I sure hope no one hogs the event loop... + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_a(). Hi! + +This behavior of ``await coroutine`` can trip a lot of people up! +That example highlights how using only ``await coroutine`` could +unintentionally hog control from other tasks and effectively stall the event +loop. +:func:`asyncio.run` can help you detect such occurences via the +``debug=True`` flag which accordingly enables +:ref:`debug mode `. +Among other things, it will log any coroutines that monopolize execution for +100ms or longer. + +The design intentionally trades off some conceptual clarity around usage of +``await`` for improved performance. +Each time a task is awaited, control needs to be passed all the way up the +call stack to the event loop. +That might sound minor, but in a large program with many ``await``'s and a deep +callstack that overhead can add up to a meaningful performance drag. + +------------------------------------------------ +A conceptual overview part 2: the nuts and bolts +------------------------------------------------ + +Part 2 goes into detail on the mechanisms :mod:`!asyncio` uses to manage +control flow. +This is where the magic happens. +You'll come away from this section knowing what ``await`` does behind the scenes +and how to make your own asynchronous operators. + +================================ +The inner workings of coroutines +================================ + +:mod:`!asyncio` leverages four components to pass around control. + +:meth:`coroutine.send(arg) ` is the method used to start or +resume a coroutine. +If the coroutine was paused and is now being resumed, the argument ``arg`` +will be sent in as the return value of the ``yield`` statement which originally +paused it. +If the coroutine is being used for the first time (as opposed to being resumed) +``arg`` must be ``None``. + +.. code-block:: + :linenos: + + class Rock: + def __await__(self): + value_sent_in = yield 7 + print(f"Rock.__await__ resuming with value: {value_sent_in}.") + return value_sent_in + + async def main(): + print("Beginning coroutine main().") + rock = Rock() + print("Awaiting rock...") + value_from_rock = await rock + print(f"Coroutine received value: {value_from_rock} from rock.") + return 23 + + coroutine = main() + intermediate_result = coroutine.send(None) + print(f"Coroutine paused and returned intermediate value: {intermediate_result}.") + + print(f"Resuming coroutine and sending in value: 42.") + try: + coroutine.send(42) + except StopIteration as e: + returned_value = e.value + print(f"Coroutine main() finished and provided value: {returned_value}.") + +:ref:`yield `, like usual, pauses execution and returns control +to the caller. +In the example above, the ``yield``, on line 3, is called by +``... = await rock`` on line 11. +More broadly speaking, ``await`` calls the :meth:`~object.__await__` method of +the given object. +``await`` also does one more very special thing: it propagates (or "passes +along") any ``yield``\ s it receives up the call-chain. +In this case, that's back to ``... = coroutine.send(None)`` on line 16. + +The coroutine is resumed via the ``coroutine.send(42)`` call on line 21. +The coroutine picks back up from where it ``yield``\ ed (or paused) on line 3 +and executes the remaining statements in its body. +When a coroutine finishes, it raises a :exc:`StopIteration` exception with the +return value attached in the :attr:`~StopIteration.value` attribute. + +That snippet produces this output: + +.. code-block:: none + + Beginning coroutine main(). + Awaiting rock... + Coroutine paused and returned intermediate value: 7. + Resuming coroutine and sending in value: 42. + Rock.__await__ resuming with value: 42. + Coroutine received value: 42 from rock. + Coroutine main() finished and provided value: 23. + +It's worth pausing for a moment here and making sure you followed the various +ways that control flow and values were passed. A lot of important ideas were +covered and it's worth ensuring your understanding is firm. + +The only way to yield (or effectively cede control) from a coroutine is to +``await`` an object that ``yield``\ s in its ``__await__`` method. +That might sound odd to you. You might be thinking: + + 1. What about a ``yield`` directly within the coroutine function? The + coroutine function becomes an + :ref:`async generator function `, a + different beast entirely. + + 2. What about a :ref:`yield from ` within the coroutine function to a (plain) + generator? + That causes the error: ``SyntaxError: yield from not allowed in a coroutine.`` + This was intentionally designed for the sake of simplicity -- mandating only + one way of using coroutines. + Initially ``yield`` was barred as well, but was re-accepted to allow for + async generators. + Despite that, ``yield from`` and ``await`` effectively do the same thing. + +======= +Futures +======= + +A :ref:`future ` is an object meant to represent a +computation's status and result. +The term is a nod to the idea of something still to come or not yet happened, +and the object is a way to keep an eye on that something. + +A future has a few important attributes. One is its state which can be either +"pending", "cancelled" or "done". +Another is its result, which is set when the state transitions to done. +Unlike a coroutine, a future does not represent the actual computation to be +done; instead, it represents the status and result of that computation, kind of +like a status light (red, yellow or green) or indicator. + +:class:`asyncio.Task` subclasses :class:`asyncio.Future` in order to gain +these various capabilities. +The prior section said tasks store a list of callbacks, which wasn't entirely +accurate. +It's actually the ``Future`` class that implements this logic, which ``Task`` +inherits. + +Futures may also be used directly (not via tasks). +Tasks mark themselves as done when their coroutine is complete. +Futures are much more versatile and will be marked as done when you say so. +In this way, they're the flexible interface for you to make your own conditions +for waiting and resuming. + +======================== +A homemade asyncio.sleep +======================== + +We'll go through an example of how you could leverage a future to create your +own variant of asynchronous sleep (``async_sleep``) which mimics +:func:`asyncio.sleep`. + +This snippet registers a few tasks with the event loop and then awaits a +coroutine wrapped in a task: ``async_sleep(3)``. +We want that task to finish only after three seconds have elapsed, but without +preventing other tasks from running. + +:: + + async def other_work(): + print("I like work. Work work.") + + async def main(): + # Add a few other tasks to the event loop, so there's something + # to do while asynchronously sleeping. + work_tasks = [ + asyncio.create_task(other_work()), + asyncio.create_task(other_work()), + asyncio.create_task(other_work()) + ] + print( + "Beginning asynchronous sleep at time: " + f"{datetime.datetime.now().strftime("%H:%M:%S")}." + ) + await asyncio.create_task(async_sleep(3)) + print( + "Done asynchronous sleep at time: " + f"{datetime.datetime.now().strftime("%H:%M:%S")}." + ) + # asyncio.gather effectively awaits each task in the collection. + await asyncio.gather(*work_tasks) + + +Below, we use a future to enable custom control over when that task will be +marked as done. +If :meth:`future.set_result() ` (the method +responsible for marking that future as done) is never called, then this task +will never finish. +We've also enlisted the help of another task, which we'll see in a moment, that +will monitor how much time has elapsed and, accordingly, call +``future.set_result()``. + +:: + + async def async_sleep(seconds: float): + future = asyncio.Future() + time_to_wake = time.time() + seconds + # Add the watcher-task to the event loop. + watcher_task = asyncio.create_task(_sleep_watcher(future, time_to_wake)) + # Block until the future is marked as done. + await future + +Below, we'll use a rather bare object, ``YieldToEventLoop()``, to ``yield`` +from ``__await__`` in order to cede control to the event loop. +This is effectively the same as calling ``asyncio.sleep(0)``, but this approach +offers more clarity, not to mention it's somewhat cheating to use +``asyncio.sleep`` when showcasing how to implement it! + +As usual, the event loop cycles through its tasks, giving them control +and receiving control back when they pause or finish. +The ``watcher_task``, which runs the coroutine ``_sleep_watcher(...)``, will +be invoked once per full cycle of the event loop. +On each resumption, it'll check the time and if not enough has elapsed, then +it'll pause once again and hand control back to the event loop. +Eventually, enough time will have elapsed, and ``_sleep_watcher(...)`` will +mark the future as done, and then itself finish too by breaking out of the +infinite ``while`` loop. +Given this helper task is only invoked once per cycle of the event loop, +you'd be correct to note that this asynchronous sleep will sleep *at least* +three seconds, rather than exactly three seconds. +Note this is also of true of ``asyncio.sleep``. + +:: + + class YieldToEventLoop: + def __await__(self): + yield + + async def _sleep_watcher(future, time_to_wake): + while True: + if time.time() >= time_to_wake: + # This marks the future as done. + future.set_result(None) + break + else: + await YieldToEventLoop() + +Here is the full program's output: + +.. code-block:: none + + $ python custom-async-sleep.py + Beginning asynchronous sleep at time: 14:52:22. + I like work. Work work. + I like work. Work work. + I like work. Work work. + Done asynchronous sleep at time: 14:52:25. + +You might feel this implementation of asynchronous sleep was unnecessarily +convoluted. +And, well, it was. +The example was meant to showcase the versatility of futures with a simple +example that could be mimicked for more complex needs. +For reference, you could implement it without futures, like so:: + + async def simpler_async_sleep(seconds): + time_to_wake = time.time() + seconds + while True: + if time.time() >= time_to_wake: + return + else: + await YieldToEventLoop() + +But, that's all for now. Hopefully you're ready to more confidently dive into +some async programming or check out advanced topics in the +:mod:`rest of the documentation `. diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 6441b7aed1eda8..7713aede6d564a 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -990,9 +990,9 @@ Supported ``_sunder_`` names from the final class - :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for an enum member; may be overridden -- :meth:`~EnumType._add_alias_` -- adds a new name as an alias to an existing +- :meth:`~Enum._add_alias_` -- adds a new name as an alias to an existing member. -- :meth:`~EnumType._add_value_alias_` -- adds a new value as an alias to an +- :meth:`~Enum._add_value_alias_` -- adds a new value as an alias to an existing member. See `MultiValueEnum`_ for an example. .. note:: diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index f350141004c2db..81fc7e63f35bd7 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -1,3 +1,5 @@ +.. _how-tos: + *************** Python HOWTOs *************** @@ -11,6 +13,7 @@ Python Library Reference. :maxdepth: 1 :hidden: + a-conceptual-overview-of-asyncio.rst cporting.rst curses.rst descriptor.rst @@ -38,6 +41,7 @@ Python Library Reference. General: +* :ref:`a-conceptual-overview-of-asyncio` * :ref:`annotations-howto` * :ref:`argparse-tutorial` * :ref:`descriptorhowto` diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index b24459b5c6346f..319b2c81505f48 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -289,9 +289,9 @@ Literals * ``conversion`` is an integer: * -1: no formatting - * 115 (``ord('s')``): ``!s`` string formatting - * 114 (``ord('r')``): ``!r`` repr formatting - * 97 (``ord('a')``): ``!a`` ASCII formatting + * 97 (``ord('a')``): ``!a`` :func:`ASCII ` formatting + * 114 (``ord('r')``): ``!r`` :func:`repr` formatting + * 115 (``ord('s')``): ``!s`` :func:`string ` formatting * ``format_spec`` is a :class:`JoinedStr` node representing the formatting of the value, or ``None`` if no format was specified. Both @@ -325,14 +325,18 @@ Literals Constant(value='.3')]))])) -.. class:: TemplateStr(values) +.. class:: TemplateStr(values, /) - A t-string, comprising a series of :class:`Interpolation` and :class:`Constant` - nodes. + .. versionadded:: 3.14 + + Node representing a template string literal, comprising a series of + :class:`Interpolation` and :class:`Constant` nodes. + These nodes may be any order, and do not need to be interleaved. .. doctest:: - >>> print(ast.dump(ast.parse('t"{name} finished {place:ordinal}"', mode='eval'), indent=4)) + >>> expr = ast.parse('t"{name} finished {place:ordinal}"', mode='eval') + >>> print(ast.dump(expr, indent=4)) Expression( body=TemplateStr( values=[ @@ -349,28 +353,28 @@ Literals values=[ Constant(value='ordinal')]))])) - .. versionadded:: 3.14 - +.. class:: Interpolation(value, str, conversion, format_spec=None) -.. class:: Interpolation(value, str, conversion, format_spec) + .. versionadded:: 3.14 - Node representing a single interpolation field in a t-string. + Node representing a single interpolation field in a template string literal. * ``value`` is any expression node (such as a literal, a variable, or a function call). + This has the same meaning as ``FormattedValue.value``. * ``str`` is a constant containing the text of the interpolation expression. * ``conversion`` is an integer: * -1: no conversion - * 115: ``!s`` string conversion - * 114: ``!r`` repr conversion - * 97: ``!a`` ascii conversion + * 97 (``ord('a')``): ``!a`` :func:`ASCII ` conversion + * 114 (``ord('r')``): ``!r`` :func:`repr` conversion + * 115 (``ord('s')``): ``!s`` :func:`string ` conversion + This has the same meaning as ``FormattedValue.conversion``. * ``format_spec`` is a :class:`JoinedStr` node representing the formatting of the value, or ``None`` if no format was specified. Both ``conversion`` and ``format_spec`` can be set at the same time. - - .. versionadded:: 3.14 + This has the same meaning as ``FormattedValue.format_spec``. .. class:: List(elts, ctx) diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 32771ba72e0002..4b69e569523c58 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -75,6 +75,7 @@ Future Functions Deprecation warning is emitted if *future* is not a Future-like object and *loop* is not specified and there is no running event loop. +.. _asyncio-future-obj: Future Object ============= diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index b19ffa8213a971..f825ae92ec7471 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -1193,6 +1193,7 @@ Introspection .. versionadded:: 3.4 +.. _asyncio-task-obj: Task Object =========== diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 7d368dae49dc1d..444db01390d922 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -29,6 +29,11 @@ database connection libraries, distributed task queues, etc. asyncio is often a perfect fit for IO-bound and high-level **structured** network code. +.. seealso:: + + :ref:`a-conceptual-overview-of-asyncio` + Explanation of the fundamentals of asyncio. + asyncio provides a set of **high-level** APIs to: * :ref:`run Python coroutines ` concurrently and diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index f96f2f8281f450..5932012c535b56 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1251,7 +1251,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | iso8859_3 | iso-8859-3, latin3, L3 | Esperanto, Maltese | +-----------------+--------------------------------+--------------------------------+ -| iso8859_4 | iso-8859-4, latin4, L4 | Baltic languages | +| iso8859_4 | iso-8859-4, latin4, L4 | Northern Europe | +-----------------+--------------------------------+--------------------------------+ | iso8859_5 | iso-8859-5, cyrillic | Belarusian, Bulgarian, | | | | Macedonian, Russian, Serbian | diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 09f596101b4d1e..d8dac24c8ab532 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -700,14 +700,10 @@ compiler does it. It is possible to override this behavior entirely by specifyi :attr:`~Structure._layout_` class attribute in the subclass definition; see the attribute documentation for details. -It is possible to specify the maximum alignment for the fields by setting -the :attr:`~Structure._pack_` class attribute to a positive integer. -This matches what ``#pragma pack(n)`` does in MSVC. - -It is also possible to set a minimum alignment for how the subclass itself is packed in the -same way ``#pragma align(n)`` works in MSVC. -This can be achieved by specifying a :attr:`~Structure._align_` class attribute -in the subclass definition. +It is possible to specify the maximum alignment for the fields and/or for the +structure itself by setting the class attributes :attr:`~Structure._pack_` +and/or :attr:`~Structure._align_`, respectively. +See the attribute documentation for details. :mod:`ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the @@ -2792,11 +2788,18 @@ fields, or any other data types containing pointer type fields. .. attribute:: _pack_ An optional small integer that allows overriding the alignment of - structure fields in the instance. :attr:`_pack_` must already be defined - when :attr:`_fields_` is assigned, otherwise it will have no effect. - Setting this attribute to 0 is the same as not setting it at all. + structure fields in the instance. + + This is only implemented for the MSVC-compatible memory layout + (see :attr:`_layout_`). - This is only implemented for the MSVC-compatible memory layout. + Setting :attr:`!_pack_` to 0 is the same as not setting it at all. + Otherwise, the value must be a positive power of two. + The effect is equivalent to ``#pragma pack(N)`` in C, except + :mod:`ctypes` may allow larger *n* than what the compiler accepts. + + :attr:`!_pack_` must already be defined + when :attr:`_fields_` is assigned, otherwise it will have no effect. .. deprecated-removed:: 3.14 3.19 @@ -2809,9 +2812,22 @@ fields, or any other data types containing pointer type fields. .. attribute:: _align_ - An optional small integer that allows overriding the alignment of + An optional small integer that allows increasing the alignment of the structure when being packed or unpacked to/from memory. - Setting this attribute to 0 is the same as not setting it at all. + + The value must not be negative. + The effect is equivalent to ``__attribute__((aligned(N)))`` on GCC + or ``#pragma align(N)`` on MSVC, except :mod:`ctypes` may allow + values that the compiler would reject. + + :attr:`!_align_` can only *increase* a structure's alignment + requirements. Setting it to 0 or 1 has no effect. + + Using values that are not powers of two is discouraged and may lead to + surprising behavior. + + :attr:`!_align_` must already be defined + when :attr:`_fields_` is assigned, otherwise it will have no effect. .. versionadded:: 3.13 diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 16ed3215bc2c1a..7010f99c54da0a 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -2383,7 +2383,7 @@ locations where different offsets are used in different days of the year or where historical changes have been made to civil time. -.. class:: timezone(offset, name=None) +.. class:: timezone(offset[, name]) The *offset* argument must be specified as a :class:`timedelta` object representing the difference between the local time and UTC. It must diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index ce948a6860f02c..c55ecac340972b 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -278,7 +278,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. emu -.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n') +.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n', *, color=False) Compare *a* and *b* (lists of strings); return a delta (a :term:`generator` generating the delta lines) in unified diff format. @@ -297,6 +297,10 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. For inputs that do not have trailing newlines, set the *lineterm* argument to ``""`` so that the output will be uniformly newline free. + Set *color* to ``True`` to enable output in color, similar to + :program:`git diff --color`. Even if enabled, it can be + :ref:`controlled using environment variables `. + The unified diff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for *fromfile*, *tofile*, *fromfiledate*, and *tofiledate*. The modification times are normally @@ -319,6 +323,10 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. See :ref:`difflib-interface` for a more detailed example. + .. versionchanged:: next + Added the *color* parameter. + + .. function:: diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'', fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\n') Compare *a* and *b* (lists of bytes objects) using *dfunc*; yield a @@ -351,9 +359,9 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. seealso:: - `Pattern Matching: The Gestalt Approach `_ + `Pattern Matching: The Gestalt Approach `_ Discussion of a similar algorithm by John W. Ratcliff and D. E. Metzener. This - was published in `Dr. Dobb's Journal `_ in July, 1988. + was published in Dr. Dobb's Journal in July, 1988. .. _sequence-matcher: diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index ac8a911c40a860..7360f4aa804724 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1122,8 +1122,8 @@ iterations of the loop. .. opcode:: BUILD_TEMPLATE - Constructs a new :class:`~string.templatelib.Template` from a tuple - of strings and a tuple of interpolations and pushes the resulting instance + Constructs a new :class:`~string.templatelib.Template` instance from a tuple + of strings and a tuple of interpolations and pushes the resulting object onto the stack:: interpolations = STACK.pop() @@ -1135,8 +1135,8 @@ iterations of the loop. .. opcode:: BUILD_INTERPOLATION (format) - Constructs a new :class:`~string.templatelib.Interpolation` from a - value and its source expression and pushes the resulting instance onto the + Constructs a new :class:`~string.templatelib.Interpolation` instance from a + value and its source expression and pushes the resulting object onto the stack. If no conversion or format specification is present, ``format`` is set to diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 2cfc2f4962979f..8875669ffccc37 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -175,6 +175,10 @@ Data Types final *enum*, as well as creating the enum members, properly handling duplicates, providing iteration over the enum class, etc. + .. versionadded:: 3.11 + + Before 3.11 ``EnumType`` was called ``EnumMeta``, which is still available as an alias. + .. method:: EnumType.__call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) This method is called in two different ways: @@ -206,7 +210,7 @@ Data Types >>> Color.RED.value in Color True - .. versionchanged:: 3.12 + .. versionchanged:: 3.12 Before Python 3.12, a ``TypeError`` is raised if a non-Enum-member is used in a containment check. @@ -251,20 +255,6 @@ Data Types >>> list(reversed(Color)) [, , ] - .. method:: EnumType._add_alias_ - - Adds a new name as an alias to an existing member. Raises a - :exc:`NameError` if the name is already assigned to a different member. - - .. method:: EnumType._add_value_alias_ - - Adds a new value as an alias to an existing member. Raises a - :exc:`ValueError` if the value is already linked with a different member. - - .. versionadded:: 3.11 - - Before 3.11 ``EnumType`` was called ``EnumMeta``, which is still available as an alias. - .. class:: Enum @@ -470,6 +460,30 @@ Data Types .. versionchanged:: 3.12 Added :ref:`enum-dataclass-support` + .. method:: Enum._add_alias_ + + Adds a new name as an alias to an existing member:: + + >>> Color.RED._add_alias_("ERROR") + >>> Color.ERROR + + + Raises a :exc:`NameError` if the name is already assigned to a different member. + + .. versionadded:: 3.13 + + .. method:: Enum._add_value_alias_ + + Adds a new value as an alias to an existing member:: + + >>> Color.RED._add_value_alias_(42) + >>> Color(42) + + + Raises a :exc:`ValueError` if the value is already linked with a different member. + + .. versionadded:: 3.13 + .. class:: IntEnum @@ -879,10 +893,6 @@ Once all the members are created it is no longer used. Supported ``_sunder_`` names """""""""""""""""""""""""""" -- :meth:`~EnumType._add_alias_` -- adds a new name as an alias to an existing - member. -- :meth:`~EnumType._add_value_alias_` -- adds a new value as an alias to an - existing member. - :attr:`~Enum._name_` -- name of the member - :attr:`~Enum._value_` -- value of the member; can be set in ``__new__`` - :meth:`~Enum._missing_` -- a lookup function used when a value is not found; @@ -903,6 +913,11 @@ Supported ``_sunder_`` names For :class:`Flag` classes the next value chosen will be the next highest power-of-two. +- :meth:`~Enum._add_alias_` -- adds a new name as an alias to an existing + member. +- :meth:`~Enum._add_value_alias_` -- adds a new value as an alias to an + existing member. + - While ``_sunder_`` names are generally reserved for the further development of the :class:`Enum` class and can not be used, some are explicitly allowed: diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index 8796056b4b8722..d6d1c7a461c51c 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -14,8 +14,8 @@ The :mod:`fractions` module provides support for rational number arithmetic. -A Fraction instance can be constructed from a pair of integers, from -another rational number, or from a string. +A Fraction instance can be constructed from a pair of rational numbers, from +a single number, or from a string. .. index:: single: as_integer_ratio() @@ -25,8 +25,8 @@ another rational number, or from a string. The first version requires that *numerator* and *denominator* are instances of :class:`numbers.Rational` and returns a new :class:`Fraction` instance - with value equal to ``numerator/denominator`` where the denominator is positive. - If *denominator* is ``0``, it raises a :exc:`ZeroDivisionError`. + with a value equal to ``numerator/denominator``. + If *denominator* is zero, it raises a :exc:`ZeroDivisionError`. The second version requires that *number* is an instance of :class:`numbers.Rational` or has the :meth:`!as_integer_ratio` method @@ -125,7 +125,8 @@ another rational number, or from a string. .. attribute:: denominator - Denominator of the Fraction in lowest term. + Denominator of the Fraction in lowest terms. + Guaranteed to be positive. .. method:: as_integer_ratio() diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 80bd1275973f8d..9bd7fddc79981e 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1220,9 +1220,9 @@ are always available. They are listed here in alphabetical order. Added the *strict* parameter. -.. function:: max(iterable, *, key=None) - max(iterable, *, default, key=None) - max(arg1, arg2, *args, key=None) +.. function:: max(iterable, /, *, key=None) + max(iterable, /, *, default, key=None) + max(arg1, arg2, /, *args, key=None) Return the largest item in an iterable or the largest of two or more arguments. @@ -1258,9 +1258,9 @@ are always available. They are listed here in alphabetical order. :ref:`typememoryview` for more information. -.. function:: min(iterable, *, key=None) - min(iterable, *, default, key=None) - min(arg1, arg2, *args, key=None) +.. function:: min(iterable, /, *, key=None) + min(iterable, /, *, default, key=None) + min(arg1, arg2, /, *args, key=None) Return the smallest item in an iterable or the smallest of two or more arguments. @@ -1422,38 +1422,10 @@ are always available. They are listed here in alphabetical order. *errors* is an optional string that specifies how encoding and decoding errors are to be handled—this cannot be used in binary mode. - A variety of standard error handlers are available - (listed under :ref:`error-handlers`), though any - error handling name that has been registered with + A variety of standard error handlers are available, + though any error handling name that has been registered with :func:`codecs.register_error` is also valid. The standard names - include: - - * ``'strict'`` to raise a :exc:`ValueError` exception if there is - an encoding error. The default value of ``None`` has the same - effect. - - * ``'ignore'`` ignores errors. Note that ignoring encoding errors - can lead to data loss. - - * ``'replace'`` causes a replacement marker (such as ``'?'``) to be inserted - where there is malformed data. - - * ``'surrogateescape'`` will represent any incorrect bytes as low - surrogate code units ranging from U+DC80 to U+DCFF. - These surrogate code units will then be turned back into - the same bytes when the ``surrogateescape`` error handler is used - when writing data. This is useful for processing files in an - unknown encoding. - - * ``'xmlcharrefreplace'`` is only supported when writing to a file. - Characters not supported by the encoding are replaced with the - appropriate XML character reference :samp:`&#{nnn};`. - - * ``'backslashreplace'`` replaces malformed data by Python's backslashed - escape sequences. - - * ``'namereplace'`` (also only supported when writing) - replaces unsupported characters with ``\N{...}`` escape sequences. + can be found in :ref:`error-handlers`. .. index:: single: universal newlines; open() built-in function @@ -1562,13 +1534,19 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.11 The ``'U'`` mode has been removed. -.. function:: ord(c) +.. function:: ord(character, /) - Given a string representing one Unicode character, return an integer - representing the Unicode code point of that character. For example, + Return the ordinal value of a character. + + If the argument is a one-character string, return the Unicode code point + of that character. For example, ``ord('a')`` returns the integer ``97`` and ``ord('€')`` (Euro sign) returns ``8364``. This is the inverse of :func:`chr`. + If the argument is a :class:`bytes` or :class:`bytearray` object of + length 1, return its single byte value. + For example, ``ord(b'a')`` returns the integer ``97``. + .. function:: pow(base, exp, mod=None) @@ -1577,7 +1555,7 @@ are always available. They are listed here in alphabetical order. ``pow(base, exp) % mod``). The two-argument form ``pow(base, exp)`` is equivalent to using the power operator: ``base**exp``. - The arguments must have numeric types. With mixed operand types, the + When arguments are builtin numeric types with mixed operand types, the coercion rules for binary arithmetic operators apply. For :class:`int` operands, the result has the same type as the operands (after coercion) unless the second argument is negative; in that case, all arguments are @@ -1729,8 +1707,8 @@ are always available. They are listed here in alphabetical order. .. _func-range: -.. class:: range(stop) - range(start, stop, step=1) +.. class:: range(stop, /) + range(start, stop, step=1, /) :noindex: Rather than being a function, :class:`range` is actually an immutable diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 4f374be778d6b3..ddf503af82d988 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -393,6 +393,8 @@ ABC hierarchy:: .. deprecated:: 3.7 This ABC is deprecated in favour of supporting resource loading through :class:`importlib.resources.abc.TraversableResources`. + This class exists for backwards compatibility only with other ABCs in + this module. .. method:: get_data(path) :abstractmethod: diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 426e3a06e1ef11..d48ea04077f366 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -34,12 +34,17 @@ The :mod:`locale` module defines the following exception and functions: If *locale* is given and not ``None``, :func:`setlocale` modifies the locale setting for the *category*. The available categories are listed in the data - description below. *locale* may be a string, or an iterable of two strings - (language code and encoding). If it's an iterable, it's converted to a locale - name using the locale aliasing engine. An empty string specifies the user's + description below. *locale* may be a :ref:`string `, or a pair, + language code and encoding. An empty string specifies the user's default settings. If the modification of the locale fails, the exception :exc:`Error` is raised. If successful, the new locale setting is returned. + If *locale* is a pair, it is converted to a locale name using + the locale aliasing engine. + The language code has the same format as a :ref:`locale name `, + but without encoding and ``@``-modifier. + The language code and encoding can be ``None``. + If *locale* is omitted or ``None``, the current setting for *category* is returned. @@ -345,22 +350,26 @@ The :mod:`locale` module defines the following exception and functions: ``'LANG'``. The GNU gettext search path contains ``'LC_ALL'``, ``'LC_CTYPE'``, ``'LANG'`` and ``'LANGUAGE'``, in that order. - Except for the code ``'C'``, the language code corresponds to :rfc:`1766`. - *language code* and *encoding* may be ``None`` if their values cannot be + The language code has the same format as a :ref:`locale name `, + but without encoding and ``@``-modifier. + The language code and encoding may be ``None`` if their values cannot be determined. + The "C" locale is represented as ``(None, None)``. .. deprecated-removed:: 3.11 3.15 .. function:: getlocale(category=LC_CTYPE) - Returns the current setting for the given locale category as sequence containing - *language code*, *encoding*. *category* may be one of the :const:`!LC_\*` values - except :const:`LC_ALL`. It defaults to :const:`LC_CTYPE`. + Returns the current setting for the given locale category as a tuple containing + the language code and encoding. *category* may be one of the :const:`!LC_\*` + values except :const:`LC_ALL`. It defaults to :const:`LC_CTYPE`. - Except for the code ``'C'``, the language code corresponds to :rfc:`1766`. - *language code* and *encoding* may be ``None`` if their values cannot be + The language code has the same format as a :ref:`locale name `, + but without encoding and ``@``-modifier. + The language code and encoding may be ``None`` if their values cannot be determined. + The "C" locale is represented as ``(None, None)``. .. function:: getpreferredencoding(do_setlocale=True) @@ -615,6 +624,61 @@ whose high bit is set (i.e., non-ASCII bytes) are never converted or considered part of a character class such as letter or whitespace. +.. _locale_name: + +Locale names +------------ + +The format of the locale name is platform dependent, and the set of supported +locales can depend on the system configuration. + +On Posix platforms, it usually has the format [1]_: + +.. productionlist:: locale_name + : language ["_" territory] ["." charset] ["@" modifier] + +where *language* is a two- or three-letter language code from `ISO 639`_, +*territory* is a two-letter country or region code from `ISO 3166`_, +*charset* is a locale encoding, and *modifier* is a script name, +a language subtag, a sort order identifier, or other locale modifier +(for example, "latin", "valencia", "stroke" and "euro"). + +On Windows, several formats are supported. [2]_ [3]_ +A subset of `IETF BCP 47`_ tags: + +.. productionlist:: locale_name + : language ["-" script] ["-" territory] ["." charset] + : language ["-" script] "-" territory "-" modifier + +where *language* and *territory* have the same meaning as in Posix, +*script* is a four-letter script code from `ISO 15924`_, +and *modifier* is a language subtag, a sort order identifier +or custom modifier (for example, "valencia", "stroke" or "x-python"). +Both hyphen (``'-'``) and underscore (``'_'``) separators are supported. +Only UTF-8 encoding is allowed for BCP 47 tags. + +Windows also supports locale names in the format: + +.. productionlist:: locale_name + : language ["_" territory] ["." charset] + +where *language* and *territory* are full names, such as "English" and +"United States", and *charset* is either a code page number (for example, "1252") +or UTF-8. +Only the underscore separator is supported in this format. + +The "C" locale is supported on all platforms. + +.. _ISO 639: https://www.iso.org/iso-639-language-code +.. _ISO 3166: https://www.iso.org/iso-3166-country-codes.html +.. _IETF BCP 47: https://www.rfc-editor.org/info/bcp47 +.. _ISO 15924: https://www.unicode.org/iso15924/ + +.. [1] `IEEE Std 1003.1-2024; 8.2 Internationalization Variables `_ +.. [2] `UCRT Locale names, Languages, and Country/Region strings `_ +.. [3] `Locale Names `_ + + .. _embedding-locale: For extension writers and programs that embed Python diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 681d0b76f2a14b..57b35017072c97 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -69,11 +69,11 @@ The numeric tower .. attribute:: numerator - Abstract. + Abstract. The numerator of this rational number. .. attribute:: denominator - Abstract. + Abstract. The denominator of this rational number. .. class:: Integral diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index a14af6d3d88df2..d317ead66f9bcb 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -509,7 +509,7 @@ Module constants .. data:: SQLITE_KEYWORDS - A :class:`tuple` containing all sqlite3 keywords. + A :class:`tuple` containing all SQLite keywords. This constant is only available if Python was compiled with SQLite 3.24.0 or greater. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 90683c0b00d78a..a81a6704142dcc 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2673,9 +2673,10 @@ For example: The formatting operations described here exhibit a variety of quirks that lead to a number of common errors (such as failing to display tuples and - dictionaries correctly). Using the newer :ref:`formatted string literals - `, the :meth:`str.format` interface, or :ref:`template strings - ($-strings) ` may help avoid these errors. + dictionaries correctly). + + Using :ref:`formatted string literals `, the :meth:`str.format` + interface, or :class:`string.Template` may help avoid these errors. Each of these alternatives provides their own trade-offs and benefits of simplicity, flexibility, and/or extensibility. diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 83e8ee2722ed8a..6336a0ec47b91e 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -200,7 +200,7 @@ syntax for format strings (although in the case of :class:`Formatter`, subclasses can define their own format string syntax). The syntax is related to that of :ref:`formatted string literals ` and :ref:`template string literals `, but it is less sophisticated -and, in particular, does not support arbitrary expressions. +and, in particular, does not support arbitrary expressions in interpolations. .. index:: single: {} (curly brackets); in string formatting @@ -799,13 +799,15 @@ Template strings ($-strings) .. note:: - The feature described here was introduced in Python 2.4. It is unrelated - to, and should not be confused with, the newer - :ref:`template strings ` and - :ref:`t-string literal syntax ` introduced in Python 3.14. - T-string literals evaluate to instances of a different - :class:`~string.templatelib.Template` class, found in the - :mod:`string.templatelib` module. + The feature described here was introduced in Python 2.4; + a simple templating method based upon regular expressions. + It predates :meth:`str.format`, :ref:`formatted string literals `, + and :ref:`template string literals `. + + It is unrelated to template string literals (t-strings), + which were introduced in Python 3.14. + These evaluate to :class:`string.templatelib.Template` objects, + found in the :mod:`string.templatelib` module. Template strings provide simpler string substitutions as described in :pep:`292`. A primary use case for template strings is for diff --git a/Doc/library/string.templatelib.rst b/Doc/library/string.templatelib.rst index 31b90d75f411f0..19daf352bdc5fc 100644 --- a/Doc/library/string.templatelib.rst +++ b/Doc/library/string.templatelib.rst @@ -11,8 +11,8 @@ .. seealso:: * :ref:`Format strings ` - * :ref:`T-string literal syntax ` - + * :ref:`Template string literal (t-string) syntax ` + * :pep:`750` .. _template-strings: @@ -21,278 +21,311 @@ Template strings .. versionadded:: 3.14 -Template strings are a formatting mechanism that allows for deep control over -how strings are processed. You can create templates using -:ref:`t-string literal syntax `, which is identical to -:ref:`f-string syntax ` but uses a ``t`` instead of an ``f``. -While f-strings evaluate to ``str``, t-strings create a :class:`Template` -instance that gives you access to the static and interpolated (in curly braces) -parts of a string *before* they are combined. - - -.. _templatelib-template: +Template strings are a mechanism for custom string processing. +They have the full flexibility of Python's :ref:`f-strings`, +but return a :class:`Template` instance that gives access +to the static and interpolated (in curly braces) parts of a string +*before* they are combined. -Template --------- +To write a t-string, use a ``'t'`` prefix instead of an ``'f'``, like so: -The :class:`!Template` class describes the contents of a template string. +.. code-block:: pycon -:class:`!Template` instances are immutable: their attributes cannot be -reassigned. + >>> pi = 3.14 + >>> t't-strings are new in Python {pi!s}!' + Template( + strings=('t-strings are new in Python ', '.'), + interpolations=(Interpolation(3.14, 'pi', 's', ''),) + ) -.. class:: Template(*args) +Types +----- - Create a new :class:`!Template` object. +.. class:: Template - :param args: A mix of strings and :class:`Interpolation` instances in any order. - :type args: str | Interpolation + The :class:`!Template` class describes the contents of a template string. + It is immutable, meaning that attributes of a template cannot be reassigned. The most common way to create a :class:`!Template` instance is to use the - :ref:`t-string literal syntax `. This syntax is identical to that of - :ref:`f-strings ` except that it uses a ``t`` instead of an ``f``: + :ref:`template string literal syntax `. + This syntax is identical to that of :ref:`f-strings `, + except that it uses a ``t`` prefix in place of an ``f``: - >>> name = "World" - >>> template = t"Hello {name}!" + >>> cheese = 'Red Leicester' + >>> template = t"We're fresh out of {cheese}, sir." >>> type(template) - Templates ars stored as sequences of literal :attr:`~Template.strings` + Templates are stored as sequences of literal :attr:`~Template.strings` and dynamic :attr:`~Template.interpolations`. - A :attr:`~Template.values` attribute holds the interpolation values: + A :attr:`~Template.values` attribute holds the values of the interpolations: + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' >>> template.strings - ('Hello ', '!') + ('Ah! We do have ', '.') >>> template.interpolations - (Interpolation('World', ...),) + (Interpolation('Camembert', ...),) >>> template.values - ('World',) + ('Camembert',) The :attr:`!strings` tuple has one more element than :attr:`!interpolations` and :attr:`!values`; the interpolations “belong” between the strings. - This may be easier to understand when tuples are aligned:: + This may be easier to understand when tuples are aligned - template.strings: ('Hello ', '!') - template.values: ( 'World', ) + .. code-block:: python - While literal syntax is the most common way to create :class:`!Template` - instances, it is also possible to create them directly using the constructor: + template.strings: ('Ah! We do have ', '.') + template.values: ( 'Camembert', ) - >>> from string.templatelib import Interpolation, Template - >>> name = "World" - >>> template = Template("Hello, ", Interpolation(name, "name"), "!") - >>> list(template) - ['Hello, ', Interpolation('World', 'name', None, ''), '!'] + .. rubric:: Attributes - If two or more consecutive strings are passed, they will be concatenated - into a single value in the :attr:`~Template.strings` attribute. For example, - the following code creates a :class:`Template` with a single final string: + .. attribute:: strings + :type: tuple[str, ...] - >>> from string.templatelib import Template - >>> template = Template("Hello ", "World", "!") - >>> template.strings - ('Hello World!',) + A :class:`tuple` of the static strings in the template. - If two or more consecutive interpolations are passed, they will be treated - as separate interpolations and an empty string will be inserted between them. - For example, the following code creates a template with empty placeholders - in the :attr:`~Template.strings` attribute: + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.strings + ('Ah! We do have ', '.') - >>> from string.templatelib import Interpolation, Template - >>> template = Template(Interpolation("World", "name"), Interpolation("!", "punctuation")) - >>> template.strings - ('', '', '') + Empty strings *are* included in the tuple: - .. attribute:: strings - :type: tuple[str, ...] + >>> response = 'We do have ' + >>> cheese = 'Camembert' + >>> template = t'Ah! {response}{cheese}.' + >>> template.strings + ('Ah! ', '', '.') - A :ref:`tuple ` of the static strings in the template. + The ``strings`` tuple is never empty, and always contains one more + string than the ``interpolations`` and ``values`` tuples: - >>> name = "World" - >>> t"Hello {name}!".strings - ('Hello ', '!') + >>> t''.strings + ('',) + >>> t''.values + () + >>> t'{'cheese'}'.strings + ('', '') + >>> t'{'cheese'}'.values + ('cheese',) - Empty strings *are* included in the tuple: + .. attribute:: interpolations + :type: tuple[Interpolation, ...] - >>> name = "World" - >>> t"Hello {name}{name}!".strings - ('Hello ', '', '!') + A :class:`tuple` of the interpolations in the template. - The ``strings`` tuple is never empty, and always contains one more - string than the ``interpolations`` and ``values`` tuples: + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.interpolations + (Interpolation('Camembert', 'cheese', None, ''),) - >>> t"".strings - ('',) - >>> t"".values - () - >>> t"{'cheese'}".strings - ('', '') - >>> t"{'cheese'}".values - ('cheese',) + The ``interpolations`` tuple may be empty and always contains one fewer + values than the ``strings`` tuple: - .. attribute:: interpolations - :type: tuple[Interpolation, ...] + >>> t'Red Leicester'.interpolations + () - A tuple of the interpolations in the template. + .. attribute:: values + :type: tuple[object, ...] - >>> name = "World" - >>> t"Hello {name}!".interpolations - (Interpolation('World', 'name', None, ''),) + A tuple of all interpolated values in the template. - The ``interpolations`` tuple may be empty and always contains one fewer - values than the ``strings`` tuple: + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.values + ('Camembert',) - >>> t"Hello!".interpolations - () + The ``values`` tuple always has the same length as the + ``interpolations`` tuple. It is always equivalent to + ``tuple(i.value for i in template.interpolations)``. - .. attribute:: values - :type: tuple[Any, ...] + .. rubric:: Methods - A tuple of all interpolated values in the template. + .. method:: __new__(*args: str | Interpolation) - >>> name = "World" - >>> t"Hello {name}!".values - ('World',) + While literal syntax is the most common way to create a :class:`!Template`, + it is also possible to create them directly using the constructor: - The ``values`` tuple always has the same length as the - ``interpolations`` tuple. It is equivalent to - ``tuple(i.value for i in template.interpolations)``. + >>> from string.templatelib import Interpolation, Template + >>> cheese = 'Camembert' + >>> template = Template( + ... 'Ah! We do have ', Interpolation(cheese, 'cheese'), '.' + ... ) + >>> list(template) + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, ''), '.'] - .. describe:: iter(template) + If multiple strings are passed consecutively, they will be concatenated + into a single value in the :attr:`~Template.strings` attribute. For example, + the following code creates a :class:`Template` with a single final string: - Iterate over the template, yielding each string and - :class:`Interpolation` in order. + >>> from string.templatelib import Template + >>> template = Template('Ah! We do have ', 'Camembert', '.') + >>> template.strings + ('Ah! We do have Camembert.',) - >>> name = "World" - >>> list(t"Hello {name}!") - ['Hello ', Interpolation('World', 'name', None, ''), '!'] + If multiple interpolations are passed consecutively, they will be treated + as separate interpolations and an empty string will be inserted between them. + For example, the following code creates a template with empty placeholders + in the :attr:`~Template.strings` attribute: - Empty strings are *not* included in the iteration: + >>> from string.templatelib import Interpolation, Template + >>> template = Template( + ... Interpolation('Camembert', 'cheese'), + ... Interpolation('.', 'punctuation'), + ... ) + >>> template.strings + ('', '', '') - >>> name = "World" - >>> list(t"Hello {name}{name}") - ['Hello ', Interpolation('World', 'name', None, ''), Interpolation('World', 'name', None, '')] + .. describe:: iter(template) - .. describe:: template + other - template += other + Iterate over the template, yielding each non-empty string and + :class:`Interpolation` in the correct order: - Concatenate this template with another, returning a new - :class:`!Template` instance: + >>> cheese = 'Camembert' + >>> list(t'Ah! We do have {cheese}.') + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, ''), '.'] - >>> name = "World" - >>> list(t"Hello " + t"there {name}!") - ['Hello there ', Interpolation('World', 'name', None, ''), '!'] + .. caution:: - Concatenation between a :class:`!Template` and a ``str`` is *not* supported. - This is because it is ambiguous whether the string should be treated as - a static string or an interpolation. If you want to concatenate a - :class:`!Template` with a string, you should either wrap the string - directly in a :class:`!Template` (to treat it as a static string) or use - an :class:`!Interpolation` (to treat it as dynamic): + Empty strings are **not** included in the iteration: - >>> from string.templatelib import Template, Interpolation - >>> template = t"Hello " - >>> # Treat "there " as a static string - >>> template += Template("there ") - >>> # Treat name as an interpolation - >>> name = "World" - >>> template += Template(Interpolation(name, "name")) - >>> list(template) - ['Hello there ', Interpolation('World', 'name', None, '')] + >>> response = 'We do have ' + >>> cheese = 'Camembert' + >>> list(t'Ah! {response}{cheese}.') # doctest: +NORMALIZE_WHITESPACE + ['Ah! ', + Interpolation('We do have ', 'response', None, ''), + Interpolation('Camembert', 'cheese', None, ''), + '.'] + .. describe:: template + other + template += other -.. class:: Interpolation(value, expression="", conversion=None, format_spec="") + Concatenate this template with another, returning a new + :class:`!Template` instance: - Create a new :class:`!Interpolation` object. + >>> cheese = 'Camembert' + >>> list(t'Ah! ' + t'We do have {cheese}.') + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, ''), '.'] - :param value: The evaluated, in-scope result of the interpolation. - :type value: object + Concatenating a :class:`!Template` and a ``str`` is **not** supported. + This is because it is unclear whether the string should be treated as + a static string or an interpolation. + If you want to concatenate a :class:`!Template` with a string, + you should either wrap the string directly in a :class:`!Template` + (to treat it as a static string) + or use an :class:`!Interpolation` (to treat it as dynamic): - :param expression: The text of a valid Python expression, or an empty string. - :type expression: str + >>> from string.templatelib import Interpolation, Template + >>> template = t'Ah! ' + >>> # Treat 'We do have ' as a static string + >>> template += Template('We do have ') + >>> # Treat cheese as an interpolation + >>> cheese = 'Camembert' + >>> template += Template(Interpolation(cheese, 'cheese')) + >>> list(template) + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, '')] - :param conversion: The optional :ref:`conversion ` to be used, one of r, s, and a. - :type conversion: ``Literal["a", "r", "s"] | None`` - :param format_spec: An optional, arbitrary string used as the :ref:`format specification ` to present the value. - :type format_spec: str +.. class:: Interpolation The :class:`!Interpolation` type represents an expression inside a template string. + It is immutable, meaning that attributes of an interpolation cannot be reassigned. - :class:`!Interpolation` instances are immutable: their attributes cannot be - reassigned. + Interpolations support pattern matching, allowing you to match against + their attributes with the :ref:`match statement `: + + >>> from string.templatelib import Interpolation + >>> interpolation = t'{1. + 2.:.2f}'.interpolations[0] + >>> interpolation + Interpolation(3.0, '1. + 2.', None, '.2f') + >>> match interpolation: + ... case Interpolation(value, expression, conversion, format_spec): + ... print(value, expression, conversion, format_spec, sep=' | ') + ... + 3.0 | 1. + 2. | None | .2f + + .. rubric:: Attributes .. attribute:: value + :type: object - :returns: The evaluated value of the interpolation. - :type: object + The evaluated value of the interpolation. - >>> t"{1 + 2}".interpolations[0].value - 3 + >>> t'{1 + 2}'.interpolations[0].value + 3 .. attribute:: expression + :type: str - :returns: The text of a valid Python expression, or an empty string. - :type: str + The text of a valid Python expression, or an empty string. - The :attr:`~Interpolation.expression` is the original text of the - interpolation's Python expression, if the interpolation was created - from a t-string literal. Developers creating interpolations manually - should either set this to an empty string or choose a suitable valid - Python expression. + The :attr:`.expression` is the original text of the + interpolation's Python expression, if the interpolation was created + from a t-string literal. Developers creating interpolations manually + should either set this to an empty string or choose a suitable valid + Python expression. - >>> t"{1 + 2}".interpolations[0].expression - '1 + 2' + >>> t'{1 + 2}'.interpolations[0].expression + '1 + 2' .. attribute:: conversion + :type: typing.Literal['a', 'r', 's'] | None - :returns: The conversion to apply to the value, or ``None``. - :type: ``Literal["a", "r", "s"] | None`` + The conversion to apply to the value, or ``None``. - The :attr:`!Interpolation.conversion` is the optional conversion to apply - to the value: + The :attr:`!conversion` is the optional conversion to apply + to the value: - >>> t"{1 + 2!a}".interpolations[0].conversion - 'a' + >>> t'{1 + 2!a}'.interpolations[0].conversion + 'a' - .. note:: + .. note:: Unlike f-strings, where conversions are applied automatically, the expected behavior with t-strings is that code that *processes* the :class:`!Template` will decide how to interpret and whether to apply - the :attr:`!Interpolation.conversion`. + the :attr:`!conversion`. + For convenience, the :func:`convert` function can be used to mimic + f-string conversion semantics. .. attribute:: format_spec + :type: str - :returns: The format specification to apply to the value. - :type: str + The format specification to apply to the value. - The :attr:`!Interpolation.format_spec` is an optional, arbitrary string - used as the format specification to present the value: + The :attr:`!format_spec` is an optional, arbitrary string + used as the format specification to present the value: - >>> t"{1 + 2:.2f}".interpolations[0].format_spec - '.2f' + >>> t'{1 + 2:.2f}'.interpolations[0].format_spec + '.2f' - .. note:: + .. note:: Unlike f-strings, where format specifications are applied automatically via the :func:`format` protocol, the expected behavior with - t-strings is that code that *processes* the :class:`!Template` will + t-strings is that code that *processes* the interpolation will decide how to interpret and whether to apply the format specification. - As a result, :attr:`!Interpolation.format_spec` values in - :class:`!Template` instances can be arbitrary strings, even those that - do not necessarily conform to the rules of Python's :func:`format` - protocol. + As a result, :attr:`!format_spec` values in interpolations + can be arbitrary strings, + including those that do not conform to the :func:`format` protocol. - Interpolations support pattern matching, allowing you to match against - their attributes with the :ref:`match statement `: + .. rubric:: Methods - >>> from string.templatelib import Interpolation - >>> interpolation = Interpolation(3.0, "1 + 2", None, ".2f") - >>> match interpolation: - ... case Interpolation(value, expression, conversion, format_spec): - ... print(value, expression, conversion, format_spec) - ... - 3.0 1 + 2 None .2f + .. method:: __new__(value: object, \ + expression: str, \ + conversion: typing.Literal['a', 'r', 's'] | None = None, \ + format_spec: str = '') + + Create a new :class:`!Interpolation` object from component parts. + + :param value: The evaluated, in-scope result of the interpolation. + :param expression: The text of a valid Python expression, + or an empty string. + :param conversion: The :ref:`conversion ` to be used, + one of ``None``, ``'a'``, ``'r'``, or ``'s'``. + :param format_spec: An optional, arbitrary string used as the + :ref:`format specification ` to present the value. Helper functions @@ -306,8 +339,8 @@ Helper functions Three conversion flags are currently supported: - * ``'s'`` which calls :func:`str` on the value, - * ``'r'`` which calls :func:`repr`, and - * ``'a'`` which calls :func:`ascii`. + * ``'s'`` which calls :func:`str` on the value (like ``!s``), + * ``'r'`` which calls :func:`repr` (like ``!r``), and + * ``'a'`` which calls :func:`ascii` (like ``!a``). If the conversion flag is ``None``, *obj* is returned unchanged. diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 52f0af31c68726..771e0f2709a4aa 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -2152,11 +2152,16 @@ always available. Unless explicitly noted otherwise, all variables are read-only The default hook formats :attr:`!err_msg` and :attr:`!object` as: ``f'{err_msg}: {object!r}'``; use "Exception ignored in" error message - if :attr:`!err_msg` is ``None``. + if :attr:`!err_msg` is ``None``. Similar to the :mod:`traceback` module, + this adds color to exceptions by default. This can be disabled using + :ref:`environment variables `. :func:`sys.unraisablehook` can be overridden to control how unraisable exceptions are handled. + .. versionchanged:: next + Exceptions are now printed with colorful text. + .. seealso:: :func:`excepthook` which handles uncaught exceptions. diff --git a/Doc/library/time.rst b/Doc/library/time.rst index df9be68bf4f69d..b05c0a312dbe34 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -935,7 +935,7 @@ These constants are used as parameters for :func:`clock_getres` and .. data:: CLOCK_TAI - `International Atomic Time `_ + `International Atomic Time `_ The system must have a current leap second table in order for this to give the correct answer. PTP or NTP software can maintain a leap second table. diff --git a/Doc/library/tulip_coro.dia b/Doc/library/tulip_coro.dia deleted file mode 100644 index 70a33e3c00cf6e..00000000000000 Binary files a/Doc/library/tulip_coro.dia and /dev/null differ diff --git a/Doc/library/tulip_coro.png b/Doc/library/tulip_coro.png deleted file mode 100644 index aad41c93015d09..00000000000000 Binary files a/Doc/library/tulip_coro.png and /dev/null differ diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 16013763dae210..9dbac8ce75d489 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -45,15 +45,15 @@ provides backports of these new features to older versions of Python. .. seealso:: - `"Typing cheat sheet" `_ + `Typing cheat sheet `_ A quick overview of type hints (hosted at the mypy docs) - "Type System Reference" section of `the mypy docs `_ + Type System Reference section of `the mypy docs `_ The Python typing system is standardised via PEPs, so this reference should broadly apply to most Python type checkers. (Some parts may still be specific to mypy.) - `"Static Typing with Python" `_ + `Static Typing with Python `_ Type-checker-agnostic documentation written by the community detailing type system features, useful typing related tools and typing best practices. @@ -64,7 +64,7 @@ Specification for the Python Type System ======================================== The canonical, up-to-date specification of the Python type system can be -found at `"Specification for the Python type system" `_. +found at `Specification for the Python type system `_. .. _type-aliases: @@ -2573,7 +2573,7 @@ types. at runtime as soon as the class has been created. Monkey-patching attributes onto a runtime-checkable protocol will still work, but will have no impact on :func:`isinstance` checks comparing objects to the - protocol. See :ref:`"What's new in Python 3.12" ` + protocol. See :ref:`What's new in Python 3.12 ` for more details. diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 091562cc9aef98..91f90a0726aa93 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -639,7 +639,7 @@ the *new_callable* argument to :func:`patch`. This is either ``None`` (if the mock hasn't been called), or the arguments that the mock was last called with. This will be in the form of a tuple: the first member, which can also be accessed through - the ``args`` property, is any ordered arguments the mock was + the ``args`` property, is any positional arguments the mock was called with (or an empty tuple) and the second member, which can also be accessed through the ``kwargs`` property, is any keyword arguments (or an empty dictionary). diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index e514b98fc5d553..5f796578eaa64e 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -837,7 +837,7 @@ The following attribute and methods should only be used by classes derived from 1. a :class:`Request` object, #. a file-like object with the HTTP error body, #. the three-digit code of the error, as a string, - #. the user-visible explanation of the code, as as string, and + #. the user-visible explanation of the code, as a string, and #. the headers of the error, as a mapping object. Return values and exceptions raised should be the same as those of diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 00bafd1be4bd0c..05e061cc697778 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -458,7 +458,7 @@ Available Functions lower.one_way(**kw) This makes the warning refer to both the ``example.lower.one_way()`` and - ``package.higher.another_way()`` call sites only from calling code living + ``example.higher.another_way()`` call sites only from calling code living outside of ``example`` package. *source*, if supplied, is the destroyed object which emitted a diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 7ac4d8587ce7d5..4e49a49c08167a 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -386,7 +386,7 @@ have ambiguous semantics. It is not possible to mix :keyword:`except` and :keyword:`!except*` in the same :keyword:`try`. -:keyword:`break`, :keyword:`continue` and :keyword:`return` +The :keyword:`break`, :keyword:`continue`, and :keyword:`return` statements cannot appear in an :keyword:`!except*` clause. diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index cf241829b71120..e320eedfa67a27 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -599,9 +599,9 @@ The allowed prefixes are: See the linked sections for details on each type. -Prefixes are case-insensitive (for example, ``B`` works the same as ``b``). -The ``r`` prefix can be combined with ``f``, ``t`` or ``b``, so ``fr``, -``rf``, ``tr``, ``rt``, ``br`` and ``rb`` are also valid prefixes. +Prefixes are case-insensitive (for example, '``B``' works the same as '``b``'). +The '``r``' prefix can be combined with '``f``', '``t``' or '``b``', so '``fr``', +'``rf``', '``tr``', '``rt``', '``br``', and '``rb``' are also valid prefixes. .. versionadded:: 3.3 The ``'rb'`` prefix of raw bytes literals has been added as a synonym @@ -661,7 +661,7 @@ quote. Escape sequences ---------------- -Unless an ``'r'`` or ``'R'`` prefix is present, escape sequences in string and +Unless an '``r``' or '``R``' prefix is present, escape sequences in string and bytes literals are interpreted according to rules similar to those used by Standard C. The recognized escape sequences are: @@ -852,7 +852,7 @@ unrecognized escapes. Bytes literals -------------- -:dfn:`Bytes literals` are always prefixed with ``'b'`` or ``'B'``; they produce an +:dfn:`Bytes literals` are always prefixed with '``b``' or '``B``'; they produce an instance of the :class:`bytes` type instead of the :class:`str` type. They may only contain ASCII characters; bytes with a numeric value of 128 or greater must be expressed with escape sequences (typically @@ -878,8 +878,8 @@ Similarly, a zero byte must be expressed using an escape sequence (typically Raw string literals ------------------- -Both string and bytes literals may optionally be prefixed with a letter ``'r'`` -or ``'R'``; such constructs are called :dfn:`raw string literals` +Both string and bytes literals may optionally be prefixed with a letter '``r``' +or '``R``'; such constructs are called :dfn:`raw string literals` and :dfn:`raw bytes literals` respectively and treat backslashes as literal characters. As a result, in raw string literals, :ref:`escape sequences ` @@ -923,7 +923,7 @@ f-strings .. versionadded:: 3.6 A :dfn:`formatted string literal` or :dfn:`f-string` is a string literal -that is prefixed with ``f`` or ``F``. These strings may contain +that is prefixed with '``f``' or '``F``'. These strings may contain replacement fields, which are expressions delimited by curly braces ``{}``. While other string literals always have a constant value, formatted strings are really expressions evaluated at run time. @@ -1089,37 +1089,37 @@ t-strings .. versionadded:: 3.14 A :dfn:`template string literal` or :dfn:`t-string` is a string literal -that is prefixed with ``t`` or ``T``. These strings follow the same -syntax and evaluation rules as :ref:`formatted string literals `, with -the following differences: +that is prefixed with '``t``' or '``T``'. +These strings follow the same syntax and evaluation rules as +:ref:`formatted string literals `, with the following differences: -- Rather than evaluating to a ``str`` object, t-strings evaluate to a - :class:`~string.templatelib.Template` object from the - :mod:`string.templatelib` module. +* Rather than evaluating to a ``str`` object, template string literals evaluate + to a :class:`string.templatelib.Template` object. -- The :func:`format` protocol is not used. Instead, the format specifier and - conversions (if any) are passed to a new :class:`~string.templatelib.Interpolation` - object that is created for each evaluated expression. It is up to code that - processes the resulting :class:`~string.templatelib.Template` object to - decide how to handle format specifiers and conversions. +* The :func:`format` protocol is not used. + Instead, the format specifier and conversions (if any) are passed to + a new :class:`~string.templatelib.Interpolation` object that is created + for each evaluated expression. + It is up to code that processes the resulting :class:`~string.templatelib.Template` + object to decide how to handle format specifiers and conversions. -- Format specifiers containing nested replacement fields are evaluated eagerly, +* Format specifiers containing nested replacement fields are evaluated eagerly, prior to being passed to the :class:`~string.templatelib.Interpolation` object. For instance, an interpolation of the form ``{amount:.{precision}f}`` will - evaluate the expression ``{precision}`` before setting the ``format_spec`` - attribute of the resulting :class:`!Interpolation` object; if ``precision`` - is (for example) ``2``, the resulting format specifier will be ``'.2f'``. - -- When the equal sign ``'='`` is provided in an interpolation expression, the - resulting :class:`~string.templatelib.Template` object will have the expression - text along with a ``'='`` character placed in its - :attr:`~string.templatelib.Template.strings` attribute. The - :attr:`~string.templatelib.Template.interpolations` attribute will also - contain an ``Interpolation`` instance for the expression. By default, the - :attr:`~string.templatelib.Interpolation.conversion` attribute will be set to - ``'r'`` (that is, :func:`repr`), unless there is a conversion explicitly - specified (in which case it overrides the default) or a format specifier is - provided (in which case, the ``conversion`` defaults to ``None``). + evaluate the inner expression ``{precision}`` to determine the value of the + ``format_spec`` attribute. + If ``precision`` were to be ``2``, the resulting format specifier + would be ``'.2f'``. + +* When the equals sign ``'='`` is provided in an interpolation expression, + the text of the expression is appended to the literal string that precedes + the relevant interpolation. + This includes the equals sign and any surrounding whitespace. + The :class:`!Interpolation` instance for the expression will be created as + normal, except that :attr:`~string.templatelib.Interpolation.conversion` will + be set to '``r``' (:func:`repr`) by default. + If an explicit conversion or format specifier are provided, + this will override the default behaviour. .. _numbers: diff --git a/Doc/requirements.txt b/Doc/requirements.txt index a2960ea9aa0203..7b7286429a1041 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -11,7 +11,7 @@ sphinx~=8.2.0 blurb -sphinxext-opengraph~=0.10.0 +sphinxext-opengraph~=0.12.0 sphinx-notfound-page~=1.0.0 # The theme used by the documentation is stored separately, so we need diff --git a/Doc/tools/check-epub.py b/Doc/tools/check-epub.py index 693dc239c8ad58..6a10096c117542 100644 --- a/Doc/tools/check-epub.py +++ b/Doc/tools/check-epub.py @@ -1,24 +1,30 @@ -import sys from pathlib import Path +CPYTHON_ROOT = Path( + __file__, # cpython/Doc/tools/check-epub.py + '..', # cpython/Doc/tools + '..', # cpython/Doc + '..', # cpython +).resolve() +EPUBCHECK_PATH = CPYTHON_ROOT / 'Doc' / 'epubcheck.txt' -def main() -> int: - wrong_directory_msg = "Must run this script from the repo root" - if not Path("Doc").exists() or not Path("Doc").is_dir(): - raise RuntimeError(wrong_directory_msg) - - with Path("Doc/epubcheck.txt").open(encoding="UTF-8") as f: - messages = [message.split(" - ") for message in f.read().splitlines()] - fatal_errors = [message for message in messages if message[0] == "FATAL"] +def main() -> int: + lines = EPUBCHECK_PATH.read_text(encoding='utf-8').splitlines() + fatal_errors = [line for line in lines if line.startswith('FATAL')] if fatal_errors: - print("\nError: must not contain fatal errors:\n") - for error in fatal_errors: - print(" - ".join(error)) + err_count = len(fatal_errors) + s = 's' * (err_count != 1) + print() + print(f'Error: epubcheck reported {err_count} fatal error{s}:') + print() + print('\n'.join(fatal_errors)) + return 1 - return len(fatal_errors) + print('Success: no fatal errors found.') + return 0 -if __name__ == "__main__": - sys.exit(main()) +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index ea546c6a29df44..a00f06cf46c41a 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -95,11 +95,11 @@ Some examples:: >>> repr((x, y, ('spam', 'eggs'))) "(32.5, 40000, ('spam', 'eggs'))" -The :mod:`string` module also contains support for so-called -:ref:`$-strings ` that offer yet another way to -substitute values into strings, using placeholders like ``$x`` and replacing -them with values from a dictionary. This syntax is easy to use, although -it offers much less control of the formatting. +The :mod:`string` module contains support for a simple templating approach +based upon regular expressions, via :class:`string.Template`. +This offers yet another way to substitute values into strings, +using placeholders like ``$x`` and replacing them with values from a dictionary. +This syntax is easy to use, although it offers much less control for formatting. .. index:: single: formatted string literal diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 6c00a0fcb83fee..9f01b52f1aff3b 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -183,7 +183,7 @@ Other language changes compatibility between versions of Python, ensure that an explicit ``encoding`` argument is always provided. The :ref:`opt-in encoding warning ` can be used to identify code that may be affected by this change. - The special special ``encoding='locale'`` argument uses the current locale + The special ``encoding='locale'`` argument uses the current locale encoding, and has been supported since Python 3.10. To retain the previous behaviour, Python's UTF-8 mode may be disabled with @@ -200,6 +200,10 @@ Other language changes * Several error messages incorrectly using the term "argument" have been corrected. (Contributed by Stan Ulbrych in :gh:`133382`.) +* Unraisable exceptions are now highlighted with color by default. This can be + controlled by :ref:`environment variables `. + (Contributed by Peter Bierma in :gh:`134170`.) + New modules =========== @@ -225,6 +229,14 @@ dbm difflib ------- + .. _whatsnew315-color-difflib: + +* Introduced the optional *color* parameter to :func:`difflib.unified_diff`, + enabling color output similar to :program:`git diff`. + This can be controlled by :ref:`environment variables + `. + (Contributed by Douglas Thor in :gh:`133725`.) + * Improved the styling of HTML diff pages generated by the :class:`difflib.HtmlDiff` class, and migrated the output to the HTML5 standard. (Contributed by Jiahao Li in :gh:`134580`.) @@ -414,6 +426,15 @@ ctypes (Contributed by Bénédikt Tran in :gh:`133866`.) +glob +---- + +* Removed the undocumented :func:`!glob.glob0` and :func:`!glob.glob1` + functions, which have been deprecated since Python 3.13. Use + :func:`glob.glob` and pass a directory to its *root_dir* argument instead. + (Contributed by Barney Gale in :gh:`137466`.) + + http.server ----------- @@ -477,6 +498,15 @@ typing or ``TD = TypedDict("TD", {})`` instead. (Contributed by Bénédikt Tran in :gh:`133823`.) +* Code like ``class ExtraTypeVars(P1[S], Protocol[T, T2]): ...`` now raises + a :exc:`TypeError`, because ``S`` is not listed in ``Protocol`` parameters. + (Contributed by Nikita Sobolev in :gh:`137191`.) + +* Code like ``class B2(A[T2], Protocol[T1, T2]): ...`` now correctly handles + type parameters order: it is ``(T1, T2)``, not ``(T2, T1)`` + as it was incorrectly infered in runtime before. + (Contributed by Nikita Sobolev in :gh:`137191`.) + wave ---- @@ -548,6 +578,18 @@ Deprecated C APIs signed integer type of the same size is now deprecated. (Contributed by Serhiy Storchaka in :gh:`132629`.) +* Deprecate :c:member:`~PyComplexObject.cval` field of the the + :c:type:`PyComplexObject` type. + Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` + to convert a Python complex number to/from the C :c:type:`Py_complex` + representation. + (Contributed by Sergey B Kirpichev in :gh:`128813`.) + +* Functions :c:func:`_Py_c_sum`, :c:func:`_Py_c_diff`, :c:func:`_Py_c_neg`, + :c:func:`_Py_c_prod`, :c:func:`_Py_c_quot`, :c:func:`_Py_c_pow` and + :c:func:`_Py_c_abs` are :term:`soft deprecated`. + (Contributed by Sergey B Kirpichev in :gh:`128813`.) + .. Add C API deprecations above alphabetically, not here at the end. Removed C APIs diff --git a/Include/Python.h b/Include/Python.h index 3f49b78947c9a6..b6ee500aada791 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -16,6 +16,7 @@ // Include standard header files +// When changing these files, remember to update Doc/extending/extending.rst. #include // assert() #include // uintptr_t #include // INT_MAX diff --git a/Include/cpython/complexobject.h b/Include/cpython/complexobject.h index fbdc6a91fe895c..58da80140dc4c9 100644 --- a/Include/cpython/complexobject.h +++ b/Include/cpython/complexobject.h @@ -7,7 +7,8 @@ typedef struct { double imag; } Py_complex; -// Operations on complex numbers. +/* Operations on complex numbers (soft deprecated + since Python 3.15). */ PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex); diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index be582122118e44..e13b2b373c47b1 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -198,6 +198,9 @@ struct _ts { PyObject *current_executor; + /* Internal to the JIT */ + struct _PyExitData *jit_exit; + uint64_t dict_global_version; /* Used to store/retrieve `threading.local` keys/values for this thread */ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index cc2defbdf77821..86eb405da659cc 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -22,8 +22,10 @@ struct _ceval_runtime_state; // Export for '_lsprof' shared extension PyAPI_FUNC(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); +extern int _PyEval_SetProfileAllThreads(PyInterpreterState *interp, Py_tracefunc func, PyObject *arg); extern int _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); +extern int _PyEval_SetTraceAllThreads(PyInterpreterState *interp, Py_tracefunc func, PyObject *arg); extern int _PyEval_SetOpcodeTrace(PyFrameObject *f, bool enable); diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 5e7dda3a3715a1..672537bfedc9af 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -842,6 +842,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call_exception_handler)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call_soon)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cancel)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capath)); @@ -877,6 +878,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(command)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(comment_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compile_mode)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(config)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(consts)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(context)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(contravariant)); @@ -891,7 +893,9 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(d_parameter_type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(data)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(database)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(date)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(day)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(days)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(debug)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decoder)); @@ -933,6 +937,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(errors)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(event)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eventmask)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(excepthook)); @@ -998,11 +1003,13 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hi)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hour)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hours)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignore)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(imag)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(implieslink)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(importlib)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(in_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(include_aliases)); @@ -1052,6 +1059,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw2)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kwargs)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(label)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last)); @@ -1092,8 +1100,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(method)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(microsecond)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(microseconds)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(milliseconds)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minute)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minutes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module)); @@ -1144,6 +1154,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(options)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(order)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(origin)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(other)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(out_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outgoing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outpath)); @@ -1166,6 +1177,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(posix)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(preserve_exc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(print_file_and_line)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress)); @@ -1193,17 +1205,22 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reload)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(replace)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reqrefs)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(require_ready)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reserved)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(resetids)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(restrict)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(return)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reverse)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reversed)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(rounding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(salt)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(script)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(second)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seconds)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(security_attributes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seek)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seekable)); @@ -1221,6 +1238,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(setsigmask)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(setstate)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(shape)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(shared)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(show_cmd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(signed)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(size)); @@ -1267,9 +1285,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(text)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(threading)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(throw)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(time)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timeout)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timer)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(times)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timespec)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timestamp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timetuple)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timeunit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(top)); @@ -1288,6 +1309,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(uid)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(unlink)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(unraisablehook)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(updates)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(uri)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(usedforsecurity)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(value)); @@ -1301,6 +1323,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wbits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(week)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(weekday)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(weeks)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(which)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(who)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(withdata)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 6908cbf78f349e..ed767206ee7322 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -333,6 +333,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(call) STRUCT_FOR_ID(call_exception_handler) STRUCT_FOR_ID(call_soon) + STRUCT_FOR_ID(callable) STRUCT_FOR_ID(callback) STRUCT_FOR_ID(cancel) STRUCT_FOR_ID(capath) @@ -368,6 +369,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(command) STRUCT_FOR_ID(comment_factory) STRUCT_FOR_ID(compile_mode) + STRUCT_FOR_ID(config) STRUCT_FOR_ID(consts) STRUCT_FOR_ID(context) STRUCT_FOR_ID(contravariant) @@ -382,7 +384,9 @@ struct _Py_global_strings { STRUCT_FOR_ID(d_parameter_type) STRUCT_FOR_ID(data) STRUCT_FOR_ID(database) + STRUCT_FOR_ID(date) STRUCT_FOR_ID(day) + STRUCT_FOR_ID(days) STRUCT_FOR_ID(debug) STRUCT_FOR_ID(decode) STRUCT_FOR_ID(decoder) @@ -424,6 +428,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(errors) STRUCT_FOR_ID(event) STRUCT_FOR_ID(eventmask) + STRUCT_FOR_ID(exc) STRUCT_FOR_ID(exc_type) STRUCT_FOR_ID(exc_value) STRUCT_FOR_ID(excepthook) @@ -489,11 +494,13 @@ struct _Py_global_strings { STRUCT_FOR_ID(hi) STRUCT_FOR_ID(hook) STRUCT_FOR_ID(hour) + STRUCT_FOR_ID(hours) STRUCT_FOR_ID(id) STRUCT_FOR_ID(ident) STRUCT_FOR_ID(identity_hint) STRUCT_FOR_ID(ignore) STRUCT_FOR_ID(imag) + STRUCT_FOR_ID(implieslink) STRUCT_FOR_ID(importlib) STRUCT_FOR_ID(in_fd) STRUCT_FOR_ID(include_aliases) @@ -543,6 +550,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(kw) STRUCT_FOR_ID(kw1) STRUCT_FOR_ID(kw2) + STRUCT_FOR_ID(kwargs) STRUCT_FOR_ID(kwdefaults) STRUCT_FOR_ID(label) STRUCT_FOR_ID(last) @@ -583,8 +591,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(metadata) STRUCT_FOR_ID(method) STRUCT_FOR_ID(microsecond) + STRUCT_FOR_ID(microseconds) STRUCT_FOR_ID(milliseconds) STRUCT_FOR_ID(minute) + STRUCT_FOR_ID(minutes) STRUCT_FOR_ID(mod) STRUCT_FOR_ID(mode) STRUCT_FOR_ID(module) @@ -635,6 +645,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(options) STRUCT_FOR_ID(order) STRUCT_FOR_ID(origin) + STRUCT_FOR_ID(other) STRUCT_FOR_ID(out_fd) STRUCT_FOR_ID(outgoing) STRUCT_FOR_ID(outpath) @@ -657,6 +668,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(pos1) STRUCT_FOR_ID(pos2) STRUCT_FOR_ID(posix) + STRUCT_FOR_ID(preserve_exc) STRUCT_FOR_ID(print_file_and_line) STRUCT_FOR_ID(priority) STRUCT_FOR_ID(progress) @@ -684,17 +696,22 @@ struct _Py_global_strings { STRUCT_FOR_ID(reload) STRUCT_FOR_ID(repl) STRUCT_FOR_ID(replace) + STRUCT_FOR_ID(reqrefs) + STRUCT_FOR_ID(require_ready) STRUCT_FOR_ID(reserved) STRUCT_FOR_ID(reset) STRUCT_FOR_ID(resetids) + STRUCT_FOR_ID(restrict) STRUCT_FOR_ID(return) STRUCT_FOR_ID(reverse) STRUCT_FOR_ID(reversed) + STRUCT_FOR_ID(rounding) STRUCT_FOR_ID(salt) STRUCT_FOR_ID(sched_priority) STRUCT_FOR_ID(scheduler) STRUCT_FOR_ID(script) STRUCT_FOR_ID(second) + STRUCT_FOR_ID(seconds) STRUCT_FOR_ID(security_attributes) STRUCT_FOR_ID(seek) STRUCT_FOR_ID(seekable) @@ -712,6 +729,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(setsigmask) STRUCT_FOR_ID(setstate) STRUCT_FOR_ID(shape) + STRUCT_FOR_ID(shared) STRUCT_FOR_ID(show_cmd) STRUCT_FOR_ID(signed) STRUCT_FOR_ID(size) @@ -758,9 +776,12 @@ struct _Py_global_strings { STRUCT_FOR_ID(text) STRUCT_FOR_ID(threading) STRUCT_FOR_ID(throw) + STRUCT_FOR_ID(time) STRUCT_FOR_ID(timeout) STRUCT_FOR_ID(timer) STRUCT_FOR_ID(times) + STRUCT_FOR_ID(timespec) + STRUCT_FOR_ID(timestamp) STRUCT_FOR_ID(timetuple) STRUCT_FOR_ID(timeunit) STRUCT_FOR_ID(top) @@ -779,6 +800,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(uid) STRUCT_FOR_ID(unlink) STRUCT_FOR_ID(unraisablehook) + STRUCT_FOR_ID(updates) STRUCT_FOR_ID(uri) STRUCT_FOR_ID(usedforsecurity) STRUCT_FOR_ID(value) @@ -792,6 +814,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(wbits) STRUCT_FOR_ID(week) STRUCT_FOR_ID(weekday) + STRUCT_FOR_ID(weeks) STRUCT_FOR_ID(which) STRUCT_FOR_ID(who) STRUCT_FOR_ID(withdata) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 542a75617b4d3c..e300732e9e58c3 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -88,6 +88,7 @@ struct _ceval_runtime_state { struct trampoline_api_st trampoline_api; FILE *map_file; Py_ssize_t persist_after_fork; + _PyFrameEvalFunction prev_eval_frame; #else int _not_used; #endif @@ -98,7 +99,6 @@ struct _ceval_runtime_state { // For example, we use a preallocated array // for the list of pending calls. struct _pending_calls pending_mainthread; - PyMutex sys_trace_profile_mutex; }; @@ -943,14 +943,15 @@ struct _is { bool jit; struct _PyExecutorObject *executor_list_head; struct _PyExecutorObject *executor_deletion_list_head; + struct _PyExecutorObject *cold_executor; int executor_deletion_list_remaining_capacity; size_t trace_run_counter; _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; _Py_GlobalMonitors monitors; - bool sys_profile_initialized; - bool sys_trace_initialized; + _PyOnceFlag sys_profile_once_flag; + _PyOnceFlag sys_trace_once_flag; Py_ssize_t sys_profiling_threads; /* Count of threads with c_profilefunc set */ Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */ PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][_PY_MONITORING_EVENTS]; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 347d9762f26bff..81bfd162c7ea80 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -277,10 +277,13 @@ Known values: Python 3.14a7 3622 (Store annotations in different class dict keys) Python 3.14a7 3623 (Add BUILD_INTERPOLATION & BUILD_TEMPLATE opcodes) Python 3.14b1 3624 (Don't optimize LOAD_FAST when local is killed by DELETE_FAST) + Python 3.14b3 3625 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) + Python 3.14rc2 3626 (Fix missing exception handlers in logical expression) Python 3.15a0 3650 (Initial version) Python 3.15a1 3651 (Simplify LOAD_CONST) Python 3.15a1 3652 (Virtual iterators) Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) + Python 3.15a1 3654 (Fix missing exception handlers in logical expression) Python 3.16 will start with 3700 @@ -294,7 +297,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3653 +#define PYC_MAGIC_NUMBER 3654 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index dd1bf2d1d2b51a..7f468bbb932184 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1028,10 +1028,9 @@ enum InstructionFormat { #define HAS_ESCAPES_FLAG (512) #define HAS_EXIT_FLAG (1024) #define HAS_PURE_FLAG (2048) -#define HAS_PASSTHROUGH_FLAG (4096) -#define HAS_OPARG_AND_1_FLAG (8192) -#define HAS_ERROR_NO_POP_FLAG (16384) -#define HAS_NO_SAVE_IP_FLAG (32768) +#define HAS_ERROR_NO_POP_FLAG (4096) +#define HAS_NO_SAVE_IP_FLAG (8192) +#define HAS_PERIODIC_FLAG (16384) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -1044,10 +1043,9 @@ enum InstructionFormat { #define OPCODE_HAS_ESCAPES(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ESCAPES_FLAG)) #define OPCODE_HAS_EXIT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EXIT_FLAG)) #define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) -#define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) -#define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) #define OPCODE_HAS_NO_SAVE_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_SAVE_IP_FLAG)) +#define OPCODE_HAS_PERIODIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PERIODIC_FLAG)) #define OPARG_SIMPLE 0 #define OPARG_CACHE_1 1 @@ -1218,14 +1216,14 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_BORROW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SMALL_INT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1250,7 +1248,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [POP_TOP] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1347,27 +1345,27 @@ _PyOpcode_macro_expansion[256] = { [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 4, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_AND_ALLOCATE_OBJECT, 2, 1 }, { _CREATE_INIT_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 10, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_BOUND_METHOD_GENERAL] = { .nuops = 7, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_O] = { .nuops = 2, .uops = { { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, + [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_O] = { .nuops = 2, .uops = { { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, OPARG_SIMPLE, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, OPARG_SIMPLE, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 3, .uops = { { _GUARD_THIRD_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_ISINSTANCE, OPARG_SIMPLE, 3 }, { _CALL_ISINSTANCE, OPARG_SIMPLE, 3 } } }, [CALL_KW_BOUND_METHOD] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, + [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_KW_PY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_LEN] = { .nuops = 3, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 } } }, [CALL_LIST_APPEND] = { .nuops = 4, .uops = { { _GUARD_CALLABLE_LIST_APPEND, OPARG_SIMPLE, 3 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 3 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 3 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_NON_PY_GENERAL] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_NON_PY_GENERAL] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_PY_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_PY_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_STR_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_STR_1, OPARG_SIMPLE, 3 }, { _CALL_STR_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_TUPLE_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, + [CALL_STR_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_STR_1, OPARG_SIMPLE, 3 }, { _CALL_STR_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_TUPLE_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_TYPE_1] = { .nuops = 3, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TYPE_1, OPARG_SIMPLE, 3 }, { _CALL_TYPE_1, OPARG_SIMPLE, 3 } } }, [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, OPARG_SIMPLE, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 8b7f12bf03d624..1571e19a35032e 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -67,8 +67,9 @@ typedef struct { #endif } _PyUOpInstruction; -typedef struct { +typedef struct _PyExitData { uint32_t target; + uint16_t index; _Py_BackoffCounter temperature; struct _PyExecutorObject *executor; } _PyExitData; @@ -354,6 +355,16 @@ PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **exec_ptr, int chain_depth); +static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit) +{ + _PyExitData *exit0 = exit - exit->index; + return (_PyExecutorObject *)(((char *)exit0) - offsetof(_PyExecutorObject, exits)); +} + +extern _PyExecutorObject *_PyExecutor_GetColdExecutor(void); + +PyAPI_FUNC(void) _PyExecutor_ClearExit(_PyExitData *exit); + static inline int is_terminator(const _PyUOpInstruction *uop) { int opcode = uop->opcode; @@ -363,6 +374,8 @@ static inline int is_terminator(const _PyUOpInstruction *uop) ); } +extern void _PyExecutor_Free(_PyExecutorObject *self); + PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); #ifdef _Py_TIER2 extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 3e41e2fd1569ca..c31c33657002ec 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -111,6 +111,8 @@ extern "C" { _Py_atomic_load_ullong_relaxed(&value) #define FT_ATOMIC_ADD_SSIZE(value, new_value) \ (void)_Py_atomic_add_ssize(&value, new_value) +#define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) +#define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -159,6 +161,8 @@ extern "C" { #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_ADD_SSIZE(value, new_value) (void)(value += new_value) +#define FT_MUTEX_LOCK(lock) do {} while (0) +#define FT_MUTEX_UNLOCK(lock) do {} while (0) #endif diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index da2ed7422c9deb..2f24a138fdbf0c 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -840,6 +840,7 @@ extern "C" { INIT_ID(call), \ INIT_ID(call_exception_handler), \ INIT_ID(call_soon), \ + INIT_ID(callable), \ INIT_ID(callback), \ INIT_ID(cancel), \ INIT_ID(capath), \ @@ -875,6 +876,7 @@ extern "C" { INIT_ID(command), \ INIT_ID(comment_factory), \ INIT_ID(compile_mode), \ + INIT_ID(config), \ INIT_ID(consts), \ INIT_ID(context), \ INIT_ID(contravariant), \ @@ -889,7 +891,9 @@ extern "C" { INIT_ID(d_parameter_type), \ INIT_ID(data), \ INIT_ID(database), \ + INIT_ID(date), \ INIT_ID(day), \ + INIT_ID(days), \ INIT_ID(debug), \ INIT_ID(decode), \ INIT_ID(decoder), \ @@ -931,6 +935,7 @@ extern "C" { INIT_ID(errors), \ INIT_ID(event), \ INIT_ID(eventmask), \ + INIT_ID(exc), \ INIT_ID(exc_type), \ INIT_ID(exc_value), \ INIT_ID(excepthook), \ @@ -996,11 +1001,13 @@ extern "C" { INIT_ID(hi), \ INIT_ID(hook), \ INIT_ID(hour), \ + INIT_ID(hours), \ INIT_ID(id), \ INIT_ID(ident), \ INIT_ID(identity_hint), \ INIT_ID(ignore), \ INIT_ID(imag), \ + INIT_ID(implieslink), \ INIT_ID(importlib), \ INIT_ID(in_fd), \ INIT_ID(include_aliases), \ @@ -1050,6 +1057,7 @@ extern "C" { INIT_ID(kw), \ INIT_ID(kw1), \ INIT_ID(kw2), \ + INIT_ID(kwargs), \ INIT_ID(kwdefaults), \ INIT_ID(label), \ INIT_ID(last), \ @@ -1090,8 +1098,10 @@ extern "C" { INIT_ID(metadata), \ INIT_ID(method), \ INIT_ID(microsecond), \ + INIT_ID(microseconds), \ INIT_ID(milliseconds), \ INIT_ID(minute), \ + INIT_ID(minutes), \ INIT_ID(mod), \ INIT_ID(mode), \ INIT_ID(module), \ @@ -1142,6 +1152,7 @@ extern "C" { INIT_ID(options), \ INIT_ID(order), \ INIT_ID(origin), \ + INIT_ID(other), \ INIT_ID(out_fd), \ INIT_ID(outgoing), \ INIT_ID(outpath), \ @@ -1164,6 +1175,7 @@ extern "C" { INIT_ID(pos1), \ INIT_ID(pos2), \ INIT_ID(posix), \ + INIT_ID(preserve_exc), \ INIT_ID(print_file_and_line), \ INIT_ID(priority), \ INIT_ID(progress), \ @@ -1191,17 +1203,22 @@ extern "C" { INIT_ID(reload), \ INIT_ID(repl), \ INIT_ID(replace), \ + INIT_ID(reqrefs), \ + INIT_ID(require_ready), \ INIT_ID(reserved), \ INIT_ID(reset), \ INIT_ID(resetids), \ + INIT_ID(restrict), \ INIT_ID(return), \ INIT_ID(reverse), \ INIT_ID(reversed), \ + INIT_ID(rounding), \ INIT_ID(salt), \ INIT_ID(sched_priority), \ INIT_ID(scheduler), \ INIT_ID(script), \ INIT_ID(second), \ + INIT_ID(seconds), \ INIT_ID(security_attributes), \ INIT_ID(seek), \ INIT_ID(seekable), \ @@ -1219,6 +1236,7 @@ extern "C" { INIT_ID(setsigmask), \ INIT_ID(setstate), \ INIT_ID(shape), \ + INIT_ID(shared), \ INIT_ID(show_cmd), \ INIT_ID(signed), \ INIT_ID(size), \ @@ -1265,9 +1283,12 @@ extern "C" { INIT_ID(text), \ INIT_ID(threading), \ INIT_ID(throw), \ + INIT_ID(time), \ INIT_ID(timeout), \ INIT_ID(timer), \ INIT_ID(times), \ + INIT_ID(timespec), \ + INIT_ID(timestamp), \ INIT_ID(timetuple), \ INIT_ID(timeunit), \ INIT_ID(top), \ @@ -1286,6 +1307,7 @@ extern "C" { INIT_ID(uid), \ INIT_ID(unlink), \ INIT_ID(unraisablehook), \ + INIT_ID(updates), \ INIT_ID(uri), \ INIT_ID(usedforsecurity), \ INIT_ID(value), \ @@ -1299,6 +1321,7 @@ extern "C" { INIT_ID(wbits), \ INIT_ID(week), \ INIT_ID(weekday), \ + INIT_ID(weeks), \ INIT_ID(which), \ INIT_ID(who), \ INIT_ID(withdata), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index b1f411945e7856..4f8866e729c51e 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1120,6 +1120,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(callable); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(callback); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1260,6 +1264,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(config); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(consts); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1316,10 +1324,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(date); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(day); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(days); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(debug); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1484,6 +1500,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(exc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exc_type); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1744,6 +1764,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(hours); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(id); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1764,6 +1788,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(implieslink); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(importlib); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1960,6 +1988,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(kwargs); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kwdefaults); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2120,6 +2152,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(microseconds); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(milliseconds); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2128,6 +2164,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(minutes); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mod); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2328,6 +2368,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(other); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(out_fd); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2416,6 +2460,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(preserve_exc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(print_file_and_line); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2524,6 +2572,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(reqrefs); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(require_ready); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reserved); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2536,6 +2592,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(restrict); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(return); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2548,6 +2608,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(rounding); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(salt); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2568,6 +2632,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(seconds); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(security_attributes); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2636,6 +2704,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(shared); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(show_cmd); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2820,6 +2892,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(time); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(timeout); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2832,6 +2908,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(timespec); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(timestamp); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(timetuple); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2904,6 +2988,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(updates); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(uri); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2956,6 +3044,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(weeks); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(which); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index a9432401525ebb..749369a40aecdd 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -76,104 +76,107 @@ extern "C" { #define _CHECK_METHOD_VERSION_KW 352 #define _CHECK_PEP_523 353 #define _CHECK_PERIODIC 354 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 355 -#define _CHECK_RECURSION_REMAINING 356 -#define _CHECK_STACK_SPACE 357 -#define _CHECK_STACK_SPACE_OPERAND 358 -#define _CHECK_VALIDITY 359 -#define _COMPARE_OP 360 -#define _COMPARE_OP_FLOAT 361 -#define _COMPARE_OP_INT 362 -#define _COMPARE_OP_STR 363 -#define _CONTAINS_OP 364 -#define _CONTAINS_OP_DICT 365 -#define _CONTAINS_OP_SET 366 +#define _CHECK_PERIODIC_AT_END 355 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 356 +#define _CHECK_RECURSION_REMAINING 357 +#define _CHECK_STACK_SPACE 358 +#define _CHECK_STACK_SPACE_OPERAND 359 +#define _CHECK_VALIDITY 360 +#define _COLD_EXIT 361 +#define _COMPARE_OP 362 +#define _COMPARE_OP_FLOAT 363 +#define _COMPARE_OP_INT 364 +#define _COMPARE_OP_STR 365 +#define _CONTAINS_OP 366 +#define _CONTAINS_OP_DICT 367 +#define _CONTAINS_OP_SET 368 #define _CONVERT_VALUE CONVERT_VALUE -#define _COPY 367 -#define _COPY_1 368 -#define _COPY_2 369 -#define _COPY_3 370 +#define _COPY 369 +#define _COPY_1 370 +#define _COPY_2 371 +#define _COPY_3 372 #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 371 +#define _CREATE_INIT_FRAME 373 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 372 +#define _DEOPT 374 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 373 -#define _DO_CALL_FUNCTION_EX 374 -#define _DO_CALL_KW 375 +#define _DO_CALL 375 +#define _DO_CALL_FUNCTION_EX 376 +#define _DO_CALL_KW 377 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 376 +#define _ERROR_POP_N 378 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 377 -#define _EXPAND_METHOD_KW 378 -#define _FATAL_ERROR 379 +#define _EXPAND_METHOD 379 +#define _EXPAND_METHOD_KW 380 +#define _FATAL_ERROR 381 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 380 -#define _FOR_ITER_GEN_FRAME 381 -#define _FOR_ITER_TIER_TWO 382 +#define _FOR_ITER 382 +#define _FOR_ITER_GEN_FRAME 383 +#define _FOR_ITER_TIER_TWO 384 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 383 -#define _GUARD_CALLABLE_ISINSTANCE 384 -#define _GUARD_CALLABLE_LEN 385 -#define _GUARD_CALLABLE_LIST_APPEND 386 -#define _GUARD_CALLABLE_STR_1 387 -#define _GUARD_CALLABLE_TUPLE_1 388 -#define _GUARD_CALLABLE_TYPE_1 389 -#define _GUARD_DORV_NO_DICT 390 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 391 -#define _GUARD_GLOBALS_VERSION 392 -#define _GUARD_IS_FALSE_POP 393 -#define _GUARD_IS_NONE_POP 394 -#define _GUARD_IS_NOT_NONE_POP 395 -#define _GUARD_IS_TRUE_POP 396 -#define _GUARD_KEYS_VERSION 397 -#define _GUARD_NOS_DICT 398 -#define _GUARD_NOS_FLOAT 399 -#define _GUARD_NOS_INT 400 -#define _GUARD_NOS_LIST 401 -#define _GUARD_NOS_NOT_NULL 402 -#define _GUARD_NOS_NULL 403 -#define _GUARD_NOS_OVERFLOWED 404 -#define _GUARD_NOS_TUPLE 405 -#define _GUARD_NOS_UNICODE 406 -#define _GUARD_NOT_EXHAUSTED_LIST 407 -#define _GUARD_NOT_EXHAUSTED_RANGE 408 -#define _GUARD_NOT_EXHAUSTED_TUPLE 409 -#define _GUARD_THIRD_NULL 410 -#define _GUARD_TOS_ANY_SET 411 -#define _GUARD_TOS_DICT 412 -#define _GUARD_TOS_FLOAT 413 -#define _GUARD_TOS_INT 414 -#define _GUARD_TOS_LIST 415 -#define _GUARD_TOS_OVERFLOWED 416 -#define _GUARD_TOS_SLICE 417 -#define _GUARD_TOS_TUPLE 418 -#define _GUARD_TOS_UNICODE 419 -#define _GUARD_TYPE_VERSION 420 -#define _GUARD_TYPE_VERSION_AND_LOCK 421 +#define _GUARD_BINARY_OP_EXTEND 385 +#define _GUARD_CALLABLE_ISINSTANCE 386 +#define _GUARD_CALLABLE_LEN 387 +#define _GUARD_CALLABLE_LIST_APPEND 388 +#define _GUARD_CALLABLE_STR_1 389 +#define _GUARD_CALLABLE_TUPLE_1 390 +#define _GUARD_CALLABLE_TYPE_1 391 +#define _GUARD_DORV_NO_DICT 392 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 393 +#define _GUARD_GLOBALS_VERSION 394 +#define _GUARD_IS_FALSE_POP 395 +#define _GUARD_IS_NONE_POP 396 +#define _GUARD_IS_NOT_NONE_POP 397 +#define _GUARD_IS_TRUE_POP 398 +#define _GUARD_KEYS_VERSION 399 +#define _GUARD_NOS_DICT 400 +#define _GUARD_NOS_FLOAT 401 +#define _GUARD_NOS_INT 402 +#define _GUARD_NOS_LIST 403 +#define _GUARD_NOS_NOT_NULL 404 +#define _GUARD_NOS_NULL 405 +#define _GUARD_NOS_OVERFLOWED 406 +#define _GUARD_NOS_TUPLE 407 +#define _GUARD_NOS_UNICODE 408 +#define _GUARD_NOT_EXHAUSTED_LIST 409 +#define _GUARD_NOT_EXHAUSTED_RANGE 410 +#define _GUARD_NOT_EXHAUSTED_TUPLE 411 +#define _GUARD_THIRD_NULL 412 +#define _GUARD_TOS_ANY_SET 413 +#define _GUARD_TOS_DICT 414 +#define _GUARD_TOS_FLOAT 415 +#define _GUARD_TOS_INT 416 +#define _GUARD_TOS_LIST 417 +#define _GUARD_TOS_OVERFLOWED 418 +#define _GUARD_TOS_SLICE 419 +#define _GUARD_TOS_TUPLE 420 +#define _GUARD_TOS_UNICODE 421 +#define _GUARD_TYPE_VERSION 422 +#define _GUARD_TYPE_VERSION_AND_LOCK 423 +#define _HANDLE_PENDING_AND_DEOPT 424 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 422 -#define _INIT_CALL_PY_EXACT_ARGS 423 -#define _INIT_CALL_PY_EXACT_ARGS_0 424 -#define _INIT_CALL_PY_EXACT_ARGS_1 425 -#define _INIT_CALL_PY_EXACT_ARGS_2 426 -#define _INIT_CALL_PY_EXACT_ARGS_3 427 -#define _INIT_CALL_PY_EXACT_ARGS_4 428 -#define _INSERT_NULL 429 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 425 +#define _INIT_CALL_PY_EXACT_ARGS 426 +#define _INIT_CALL_PY_EXACT_ARGS_0 427 +#define _INIT_CALL_PY_EXACT_ARGS_1 428 +#define _INIT_CALL_PY_EXACT_ARGS_2 429 +#define _INIT_CALL_PY_EXACT_ARGS_3 430 +#define _INIT_CALL_PY_EXACT_ARGS_4 431 +#define _INSERT_NULL 432 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -183,177 +186,177 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 430 +#define _IS_NONE 433 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 431 -#define _ITER_CHECK_RANGE 432 -#define _ITER_CHECK_TUPLE 433 -#define _ITER_JUMP_LIST 434 -#define _ITER_JUMP_RANGE 435 -#define _ITER_JUMP_TUPLE 436 -#define _ITER_NEXT_LIST 437 -#define _ITER_NEXT_LIST_TIER_TWO 438 -#define _ITER_NEXT_RANGE 439 -#define _ITER_NEXT_TUPLE 440 -#define _JUMP_TO_TOP 441 +#define _ITER_CHECK_LIST 434 +#define _ITER_CHECK_RANGE 435 +#define _ITER_CHECK_TUPLE 436 +#define _ITER_JUMP_LIST 437 +#define _ITER_JUMP_RANGE 438 +#define _ITER_JUMP_TUPLE 439 +#define _ITER_NEXT_LIST 440 +#define _ITER_NEXT_LIST_TIER_TWO 441 +#define _ITER_NEXT_RANGE 442 +#define _ITER_NEXT_TUPLE 443 +#define _JUMP_TO_TOP 444 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 442 -#define _LOAD_ATTR_CLASS 443 +#define _LOAD_ATTR 445 +#define _LOAD_ATTR_CLASS 446 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 444 -#define _LOAD_ATTR_METHOD_LAZY_DICT 445 -#define _LOAD_ATTR_METHOD_NO_DICT 446 -#define _LOAD_ATTR_METHOD_WITH_VALUES 447 -#define _LOAD_ATTR_MODULE 448 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 449 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 450 -#define _LOAD_ATTR_PROPERTY_FRAME 451 -#define _LOAD_ATTR_SLOT 452 -#define _LOAD_ATTR_WITH_HINT 453 +#define _LOAD_ATTR_INSTANCE_VALUE 447 +#define _LOAD_ATTR_METHOD_LAZY_DICT 448 +#define _LOAD_ATTR_METHOD_NO_DICT 449 +#define _LOAD_ATTR_METHOD_WITH_VALUES 450 +#define _LOAD_ATTR_MODULE 451 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 452 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 453 +#define _LOAD_ATTR_PROPERTY_FRAME 454 +#define _LOAD_ATTR_SLOT 455 +#define _LOAD_ATTR_WITH_HINT 456 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 454 +#define _LOAD_BYTECODE 457 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 455 -#define _LOAD_CONST_INLINE_BORROW 456 -#define _LOAD_CONST_UNDER_INLINE 457 -#define _LOAD_CONST_UNDER_INLINE_BORROW 458 +#define _LOAD_CONST_INLINE 458 +#define _LOAD_CONST_INLINE_BORROW 459 +#define _LOAD_CONST_UNDER_INLINE 460 +#define _LOAD_CONST_UNDER_INLINE_BORROW 461 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 459 -#define _LOAD_FAST_0 460 -#define _LOAD_FAST_1 461 -#define _LOAD_FAST_2 462 -#define _LOAD_FAST_3 463 -#define _LOAD_FAST_4 464 -#define _LOAD_FAST_5 465 -#define _LOAD_FAST_6 466 -#define _LOAD_FAST_7 467 +#define _LOAD_FAST 462 +#define _LOAD_FAST_0 463 +#define _LOAD_FAST_1 464 +#define _LOAD_FAST_2 465 +#define _LOAD_FAST_3 466 +#define _LOAD_FAST_4 467 +#define _LOAD_FAST_5 468 +#define _LOAD_FAST_6 469 +#define _LOAD_FAST_7 470 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 468 -#define _LOAD_FAST_BORROW_0 469 -#define _LOAD_FAST_BORROW_1 470 -#define _LOAD_FAST_BORROW_2 471 -#define _LOAD_FAST_BORROW_3 472 -#define _LOAD_FAST_BORROW_4 473 -#define _LOAD_FAST_BORROW_5 474 -#define _LOAD_FAST_BORROW_6 475 -#define _LOAD_FAST_BORROW_7 476 +#define _LOAD_FAST_BORROW 471 +#define _LOAD_FAST_BORROW_0 472 +#define _LOAD_FAST_BORROW_1 473 +#define _LOAD_FAST_BORROW_2 474 +#define _LOAD_FAST_BORROW_3 475 +#define _LOAD_FAST_BORROW_4 476 +#define _LOAD_FAST_BORROW_5 477 +#define _LOAD_FAST_BORROW_6 478 +#define _LOAD_FAST_BORROW_7 479 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 477 -#define _LOAD_GLOBAL_BUILTINS 478 -#define _LOAD_GLOBAL_MODULE 479 +#define _LOAD_GLOBAL 480 +#define _LOAD_GLOBAL_BUILTINS 481 +#define _LOAD_GLOBAL_MODULE 482 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 480 -#define _LOAD_SMALL_INT_0 481 -#define _LOAD_SMALL_INT_1 482 -#define _LOAD_SMALL_INT_2 483 -#define _LOAD_SMALL_INT_3 484 -#define _LOAD_SPECIAL 485 +#define _LOAD_SMALL_INT 483 +#define _LOAD_SMALL_INT_0 484 +#define _LOAD_SMALL_INT_1 485 +#define _LOAD_SMALL_INT_2 486 +#define _LOAD_SMALL_INT_3 487 +#define _LOAD_SPECIAL 488 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 486 +#define _MAKE_CALLARGS_A_TUPLE 489 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 487 +#define _MAKE_WARM 490 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 488 -#define _MAYBE_EXPAND_METHOD_KW 489 -#define _MONITOR_CALL 490 -#define _MONITOR_CALL_KW 491 -#define _MONITOR_JUMP_BACKWARD 492 -#define _MONITOR_RESUME 493 +#define _MAYBE_EXPAND_METHOD 491 +#define _MAYBE_EXPAND_METHOD_KW 492 +#define _MONITOR_CALL 493 +#define _MONITOR_CALL_KW 494 +#define _MONITOR_JUMP_BACKWARD 495 +#define _MONITOR_RESUME 496 #define _NOP NOP -#define _POP_CALL 494 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 495 -#define _POP_CALL_ONE 496 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 497 -#define _POP_CALL_TWO 498 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 499 +#define _POP_CALL 497 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 498 +#define _POP_CALL_ONE 499 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 500 +#define _POP_CALL_TWO 501 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 502 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 500 -#define _POP_JUMP_IF_TRUE 501 +#define _POP_JUMP_IF_FALSE 503 +#define _POP_JUMP_IF_TRUE 504 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 502 -#define _POP_TOP_INT 503 -#define _POP_TOP_LOAD_CONST_INLINE 504 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 505 -#define _POP_TOP_NOP 506 -#define _POP_TOP_UNICODE 507 -#define _POP_TWO 508 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 509 +#define _POP_TOP_FLOAT 505 +#define _POP_TOP_INT 506 +#define _POP_TOP_LOAD_CONST_INLINE 507 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 508 +#define _POP_TOP_NOP 509 +#define _POP_TOP_UNICODE 510 +#define _POP_TWO 511 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 512 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 510 +#define _PUSH_FRAME 513 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 511 -#define _PY_FRAME_GENERAL 512 -#define _PY_FRAME_KW 513 -#define _QUICKEN_RESUME 514 -#define _REPLACE_WITH_TRUE 515 +#define _PUSH_NULL_CONDITIONAL 514 +#define _PY_FRAME_GENERAL 515 +#define _PY_FRAME_KW 516 +#define _QUICKEN_RESUME 517 +#define _REPLACE_WITH_TRUE 518 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 516 -#define _SEND 517 -#define _SEND_GEN_FRAME 518 +#define _SAVE_RETURN_OFFSET 519 +#define _SEND 520 +#define _SEND_GEN_FRAME 521 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 519 -#define _STORE_ATTR 520 -#define _STORE_ATTR_INSTANCE_VALUE 521 -#define _STORE_ATTR_SLOT 522 -#define _STORE_ATTR_WITH_HINT 523 +#define _START_EXECUTOR 522 +#define _STORE_ATTR 523 +#define _STORE_ATTR_INSTANCE_VALUE 524 +#define _STORE_ATTR_SLOT 525 +#define _STORE_ATTR_WITH_HINT 526 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 524 -#define _STORE_FAST_0 525 -#define _STORE_FAST_1 526 -#define _STORE_FAST_2 527 -#define _STORE_FAST_3 528 -#define _STORE_FAST_4 529 -#define _STORE_FAST_5 530 -#define _STORE_FAST_6 531 -#define _STORE_FAST_7 532 +#define _STORE_FAST 527 +#define _STORE_FAST_0 528 +#define _STORE_FAST_1 529 +#define _STORE_FAST_2 530 +#define _STORE_FAST_3 531 +#define _STORE_FAST_4 532 +#define _STORE_FAST_5 533 +#define _STORE_FAST_6 534 +#define _STORE_FAST_7 535 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 533 -#define _STORE_SUBSCR 534 -#define _STORE_SUBSCR_DICT 535 -#define _STORE_SUBSCR_LIST_INT 536 -#define _SWAP 537 -#define _SWAP_2 538 -#define _SWAP_3 539 -#define _TIER2_RESUME_CHECK 540 -#define _TO_BOOL 541 +#define _STORE_SLICE 536 +#define _STORE_SUBSCR 537 +#define _STORE_SUBSCR_DICT 538 +#define _STORE_SUBSCR_LIST_INT 539 +#define _SWAP 540 +#define _SWAP_2 541 +#define _SWAP_3 542 +#define _TIER2_RESUME_CHECK 543 +#define _TO_BOOL 544 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 542 +#define _TO_BOOL_LIST 545 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 543 +#define _TO_BOOL_STR 546 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 544 -#define _UNPACK_SEQUENCE_LIST 545 -#define _UNPACK_SEQUENCE_TUPLE 546 -#define _UNPACK_SEQUENCE_TWO_TUPLE 547 +#define _UNPACK_SEQUENCE 547 +#define _UNPACK_SEQUENCE_LIST 548 +#define _UNPACK_SEQUENCE_TUPLE 549 +#define _UNPACK_SEQUENCE_TWO_TUPLE 550 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 547 +#define MAX_UOP_ID 550 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index ff7e800aa9bb1a..bf361233560c55 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -24,7 +24,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, - [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, @@ -148,7 +148,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_LOCALS] = HAS_ERROR_FLAG, [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_NULL_CONDITIONAL] = HAS_ARG_FLAG, @@ -308,7 +308,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, - [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, [_JUMP_TO_TOP] = 0, [_SET_IP] = 0, @@ -330,12 +330,14 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_CONST_UNDER_INLINE] = 0, [_LOAD_CONST_UNDER_INLINE_BORROW] = 0, [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, - [_START_EXECUTOR] = 0, + [_START_EXECUTOR] = HAS_DEOPT_FLAG, [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, [_DEOPT] = 0, + [_HANDLE_PENDING_AND_DEOPT] = HAS_ESCAPES_FLAG, [_ERROR_POP_N] = HAS_ARG_FLAG, - [_TIER2_RESUME_CHECK] = HAS_DEOPT_FLAG, + [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, + [_COLD_EXIT] = HAS_ESCAPES_FLAG, }; const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { @@ -419,6 +421,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_COLD_EXIT] = "_COLD_EXIT", [_COMPARE_OP] = "_COMPARE_OP", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", [_COMPARE_OP_INT] = "_COMPARE_OP_INT", @@ -499,6 +502,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_TOS_UNICODE] = "_GUARD_TOS_UNICODE", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_GUARD_TYPE_VERSION_AND_LOCK] = "_GUARD_TYPE_VERSION_AND_LOCK", + [_HANDLE_PENDING_AND_DEOPT] = "_HANDLE_PENDING_AND_DEOPT", [_IMPORT_FROM] = "_IMPORT_FROM", [_IMPORT_NAME] = "_IMPORT_NAME", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", @@ -1297,10 +1301,14 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _DEOPT: return 0; + case _HANDLE_PENDING_AND_DEOPT: + return 0; case _ERROR_POP_N: return 0; case _TIER2_RESUME_CHECK: return 0; + case _COLD_EXIT: + return 0; default: return -1; } diff --git a/InternalDocs/README.md b/InternalDocs/README.md index 6b1d919826402c..a6e2df5ae4a9c3 100644 --- a/InternalDocs/README.md +++ b/InternalDocs/README.md @@ -44,6 +44,8 @@ Program Execution - [Quiescent-State Based Reclamation (QSBR)](qsbr.md) +- [Stack protection](stack_protection.md) + Modules --- diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index 9c35684c945b3e..a7d872f3ec4392 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -329,15 +329,16 @@ Once the GC knows the list of unreachable objects, a very delicate process start with the objective of completely destroying these objects. Roughly, the process follows these steps in order: -1. Handle and clear weak references (if any). Weak references to unreachable objects - are set to `None`. If the weak reference has an associated callback, the callback - is enqueued to be called once the clearing of weak references is finished. We only - invoke callbacks for weak references that are themselves reachable. If both the weak - reference and the pointed-to object are unreachable we do not execute the callback. - This is partly for historical reasons: the callback could resurrect an unreachable - object and support for weak references predates support for object resurrection. - Ignoring the weak reference's callback is fine because both the object and the weakref - are going away, so it's legitimate to say the weak reference is going away first. +1. Handle weak references with callbacks (if any). If the weak reference has + an associated callback, the callback is enqueued to be called after the weak + reference is cleared. We only invoke callbacks for weak references that + are themselves reachable. If both the weak reference and the pointed-to + object are unreachable we do not execute the callback. This is partly for + historical reasons: the callback could resurrect an unreachable object + and support for weak references predates support for object resurrection. + Ignoring the weak reference's callback is fine because both the object and + the weakref are going away, so it's legitimate to say the weak reference is + going away first. 2. If an object has legacy finalizers (`tp_del` slot) move it to the `gc.garbage` list. 3. Call the finalizers (`tp_finalize` slot) and mark the objects as already @@ -346,7 +347,12 @@ follows these steps in order: 4. Deal with resurrected objects. If some objects have been resurrected, the GC finds the new subset of objects that are still unreachable by running the cycle detection algorithm again and continues with them. -5. Call the `tp_clear` slot of every object so all internal links are broken and +5. Clear any weak references that still refer to unreachable objects. The + `wr_object` attribute for these weakrefs are set to `None`. Note that some + of these weak references maybe have been newly created during the running of + finalizers in step 3. Also, clear any weak references that are part of the + unreachable set. +6. Call the `tp_clear` slot of every object so all internal links are broken and the reference counts fall to 0, triggering the destruction of all unreachable objects. diff --git a/InternalDocs/stack_protection.md b/InternalDocs/stack_protection.md new file mode 100644 index 00000000000000..5e35b2d8195bd1 --- /dev/null +++ b/InternalDocs/stack_protection.md @@ -0,0 +1,61 @@ +# Stack Protection + +CPython protects against stack overflow in the form of runaway, or just very deep, recursion by raising a `RecursionError` instead of just crashing. +Protection against pure Python stack recursion has existed since very early, but in 3.12 we added protection against stack overflow +in C code. This was initially implemented using a counter and later improved in 3.14 to use the actual stack depth. +For those platforms that support it (Windows, Mac, and most Linuxes) we query the operating system to find the stack bounds. +For other platforms we use conserative estimates. + + +The C stack looks like this: + +``` + +-------+ <--- Top of machine stack + | | + | | + + ~~ + + | | + | | + +-------+ <--- Soft limit + | | + | | _PyOS_STACK_MARGIN_BYTES + | | + +-------+ <--- Hard limit + | | + | | _PyOS_STACK_MARGIN_BYTES + | | + +-------+ <--- Bottom of machine stack +``` + + +We get the current stack pointer using compiler intrinsics where available, or by taking the address of a C local variable. See `_Py_get_machine_stack_pointer()`. + +The soft and hard limits pointers are set by calling `_Py_InitializeRecursionLimits()` during thread initialization. + +Recursion checks are performed by `_Py_EnterRecursiveCall()` or `_Py_EnterRecursiveCallTstate()` which compare the stack pointer to the soft limit. If the stack pointer is lower than the soft limit, then `_Py_CheckRecursiveCall()` is called which checks against both the hard and soft limits: + +```python +kb_used = (stack_top - stack_pointer)>>10 +if stack_pointer < hard_limit: + FatalError(f"Unrecoverable stack overflow (used {kb_used} kB)") +elif stack_pointer < soft_limit: + raise RecursionError(f"Stack overflow (used {kb_used} kB)") +``` + +### Diagnosing and fixing stack overflows + +For stack protection to work correctly the amount of stack consumed between calls to `_Py_EnterRecursiveCall()` must be less than `_PyOS_STACK_MARGIN_BYTES`. + +If you see a traceback ending in: `RecursionError: Stack overflow (used ... kB)` then the stack protection is working as intended. If you don't expect to see the error, then check the amount of stack used. If it seems low then CPython may not be configured properly. + +However, if you see a fatal error or crash, then something is not right. +Either a recursive call is not checking `_Py_EnterRecursiveCall()`, or the amount of C stack consumed by a single call exceeds `_PyOS_STACK_MARGIN_BYTES`. If a hard crash occurs, it probably means that the amount of C stack consumed is more than double `_PyOS_STACK_MARGIN_BYTES`. + +Likely causes: +* Recursive code is not calling `_Py_EnterRecursiveCall()` +* `-O0` compilation flags, especially for Clang. With no optimization, C calls can consume a lot of stack space +* Giant, complex functions in third-party C extensions. This is unlikely as the function in question would need to be more complicated than the bytecode interpreter. +* `_PyOS_STACK_MARGIN_BYTES` is just too low. +* `_Py_InitializeRecursionLimits()` is not setting the soft and hard limits correctly for that platform. diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 4a310a402358b6..325efed274aed7 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -172,7 +172,18 @@ class Argparse(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) +class Difflib(ThemeSection): + """A 'git diff'-like theme for `difflib.unified_diff`.""" + added: str = ANSIColors.GREEN + context: str = ANSIColors.RESET # context lines + header: str = ANSIColors.BOLD # eg "---" and "+++" lines + hunk: str = ANSIColors.CYAN # the "@@" lines + removed: str = ANSIColors.RED + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) class Syntax(ThemeSection): prompt: str = ANSIColors.BOLD_MAGENTA keyword: str = ANSIColors.BOLD_BLUE @@ -186,7 +197,7 @@ class Syntax(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Traceback(ThemeSection): type: str = ANSIColors.BOLD_MAGENTA message: str = ANSIColors.MAGENTA @@ -198,7 +209,7 @@ class Traceback(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Unittest(ThemeSection): passed: str = ANSIColors.GREEN warn: str = ANSIColors.YELLOW @@ -207,7 +218,7 @@ class Unittest(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Theme: """A suite of themes for all sections of Python. @@ -215,6 +226,7 @@ class Theme: below. """ argparse: Argparse = field(default_factory=Argparse) + difflib: Difflib = field(default_factory=Difflib) syntax: Syntax = field(default_factory=Syntax) traceback: Traceback = field(default_factory=Traceback) unittest: Unittest = field(default_factory=Unittest) @@ -223,6 +235,7 @@ def copy_with( self, *, argparse: Argparse | None = None, + difflib: Difflib | None = None, syntax: Syntax | None = None, traceback: Traceback | None = None, unittest: Unittest | None = None, @@ -234,6 +247,7 @@ def copy_with( """ return type(self)( argparse=argparse or self.argparse, + difflib=difflib or self.difflib, syntax=syntax or self.syntax, traceback=traceback or self.traceback, unittest=unittest or self.unittest, @@ -249,6 +263,7 @@ def no_colors(cls) -> Self: """ return cls( argparse=Argparse.no_colors(), + difflib=Difflib.no_colors(), syntax=Syntax.no_colors(), traceback=Traceback.no_colors(), unittest=Unittest.no_colors(), diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index bc35823f70144e..0955005df5ccee 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -1083,7 +1083,7 @@ def fromisocalendar(cls, year, week, day): @classmethod def strptime(cls, date_string, format): - """Parse a date string according to the given format (like time.strptime()).""" + """Parse string according to the given date format (like time.strptime()).""" import _strptime return _strptime._strptime_datetime_date(cls, date_string, format) @@ -1310,7 +1310,7 @@ def __reduce__(self): class tzinfo: - """Abstract base class for time zone info classes. + """Abstract base class for time zone info objects. Subclasses must override the tzname(), utcoffset() and dst() methods. """ @@ -1468,7 +1468,7 @@ def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold @classmethod def strptime(cls, date_string, format): - """string, format -> new time parsed from a string (like time.strptime()).""" + """Parse string according to the given time format (like time.strptime()).""" import _strptime return _strptime._strptime_datetime_time(cls, date_string, format) @@ -1776,7 +1776,7 @@ def __reduce__(self): class datetime(date): - """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]]) + """A combination of a date and a time. The year, month and day arguments are required. tzinfo may be None, or an instance of a tzinfo subclass. The remaining arguments may be ints. @@ -2209,7 +2209,7 @@ def __str__(self): @classmethod def strptime(cls, date_string, format): - 'string, format -> new datetime parsed from a string (like time.strptime()).' + """Parse string according to the given date and time format (like time.strptime()).""" import _strptime return _strptime._strptime_datetime_datetime(cls, date_string, format) @@ -2435,6 +2435,8 @@ def _isoweek1monday(year): class timezone(tzinfo): + """Fixed offset from UTC implementation of tzinfo.""" + __slots__ = '_offset', '_name' # Sentinel value to disallow None diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index c83a1573ccd3d1..bee019cd51591e 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -158,21 +158,13 @@ def evaluate( # as a way of emulating annotation scopes when calling `eval()` type_params = getattr(owner, "__type_params__", None) - # type parameters require some special handling, - # as they exist in their own scope - # but `eval()` does not have a dedicated parameter for that scope. - # For classes, names in type parameter scopes should override - # names in the global scope (which here are called `localns`!), - # but should in turn be overridden by names in the class scope - # (which here are called `globalns`!) + # Type parameters exist in their own scope, which is logically + # between the locals and the globals. We simulate this by adding + # them to the globals. if type_params is not None: globals = dict(globals) - locals = dict(locals) for param in type_params: - param_name = param.__name__ - if not self.__forward_is_class__ or param_name not in globals: - globals[param_name] = param - locals.pop(param_name, None) + globals[param.__name__] = param if self.__extra_names__: locals = {**locals, **self.__extra_names__} diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index ff3a69d1e17297..5ef2e1f9efc9ed 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -1,7 +1,6 @@ import argparse import ast import asyncio -import asyncio.tools import concurrent.futures import contextvars import inspect @@ -11,6 +10,9 @@ import threading import types import warnings +from asyncio.tools import (TaskTableOutputFormat, + display_awaited_by_tasks_table, + display_awaited_by_tasks_tree) from _colorize import get_theme from _pyrepl.console import InteractiveColoredConsole @@ -153,6 +155,11 @@ def interrupt(self) -> None: "ps", help="Display a table of all pending tasks in a process" ) ps.add_argument("pid", type=int, help="Process ID to inspect") + formats = [fmt.value for fmt in TaskTableOutputFormat] + formats_to_show = [fmt for fmt in formats + if fmt != TaskTableOutputFormat.bsv.value] + ps.add_argument("--format", choices=formats, default="table", + metavar=f"{{{','.join(formats_to_show)}}}") pstree = subparsers.add_parser( "pstree", help="Display a tree of all pending tasks in a process" ) @@ -160,10 +167,10 @@ def interrupt(self) -> None: args = parser.parse_args() match args.command: case "ps": - asyncio.tools.display_awaited_by_tasks_table(args.pid) + display_awaited_by_tasks_table(args.pid, format=args.format) sys.exit(0) case "pstree": - asyncio.tools.display_awaited_by_tasks_tree(args.pid) + display_awaited_by_tasks_tree(args.pid) sys.exit(0) case None: pass # continue to the interactive shell diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 520d4b398545bf..8cbb71f708537f 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -636,7 +636,7 @@ def _check_running(self): def _run_forever_setup(self): """Prepare the run loop to process events. - This method exists so that custom custom event loop subclasses (e.g., event loops + This method exists so that custom event loop subclasses (e.g., event loops that integrate a GUI event loop with Python's event loop) have access to all the loop setup logic. """ @@ -656,7 +656,7 @@ def _run_forever_setup(self): def _run_forever_cleanup(self): """Clean up after an event loop finishes the looping over events. - This method exists so that custom custom event loop subclasses (e.g., event loops + This method exists so that custom event loop subclasses (e.g., event loops that integrate a GUI event loop with Python's event loop) have access to all the loop cleanup logic. """ diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 2ad65d8648e6c5..845aed4c6a3b35 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -62,7 +62,6 @@ async def staggered_race(coro_fns, delay, *, loop=None): coroutine's entry is ``None``. """ - # TODO: when we have aiter() and anext(), allow async iterables in coro_fns. loop = loop or events.get_running_loop() parent_task = tasks.current_task(loop) enum_coro_fns = enumerate(coro_fns) diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 64aac4cc50d15a..c8c01f36474183 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -271,7 +271,6 @@ def connection_lost(self, exc): self._closed.set_exception(exc) super().connection_lost(exc) self._stream_reader_wr = None - self._stream_writer = None self._task = None self._transport = None diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 2683f34cc7113b..3887fb8ab5bf52 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -1,8 +1,9 @@ """Tools to analyze tasks running in asyncio programs.""" -from collections import defaultdict, namedtuple +from collections import defaultdict +import csv from itertools import count -from enum import Enum +from enum import Enum, StrEnum, auto import sys from _remote_debugging import RemoteUnwinder, FrameInfo @@ -232,18 +233,56 @@ def _get_awaited_by_tasks(pid: int) -> list: sys.exit(1) -def display_awaited_by_tasks_table(pid: int) -> None: +class TaskTableOutputFormat(StrEnum): + table = auto() + csv = auto() + bsv = auto() + # 🍌SV is not just a format. It's a lifestyle. A philosophy. + # https://www.youtube.com/watch?v=RrsVi1P6n0w + + +def display_awaited_by_tasks_table(pid, *, format=TaskTableOutputFormat.table): """Build and print a table of all pending tasks under `pid`.""" tasks = _get_awaited_by_tasks(pid) table = build_task_table(tasks) - # Print the table in a simple tabular format - print( - f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine stack':<50} {'awaiter chain':<50} {'awaiter name':<15} {'awaiter id':<15}" - ) - print("-" * 180) + format = TaskTableOutputFormat(format) + if format == TaskTableOutputFormat.table: + _display_awaited_by_tasks_table(table) + else: + _display_awaited_by_tasks_csv(table, format=format) + + +_row_header = ('tid', 'task id', 'task name', 'coroutine stack', + 'awaiter chain', 'awaiter name', 'awaiter id') + + +def _display_awaited_by_tasks_table(table): + """Print the table in a simple tabular format.""" + print(_fmt_table_row(*_row_header)) + print('-' * 180) for row in table: - print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<50} {row[5]:<15} {row[6]:<15}") + print(_fmt_table_row(*row)) + + +def _fmt_table_row(tid, task_id, task_name, coro_stack, + awaiter_chain, awaiter_name, awaiter_id): + # Format a single row for the table format + return (f'{tid:<10} {task_id:<20} {task_name:<20} {coro_stack:<50} ' + f'{awaiter_chain:<50} {awaiter_name:<15} {awaiter_id:<15}') + + +def _display_awaited_by_tasks_csv(table, *, format): + """Print the table in CSV format""" + if format == TaskTableOutputFormat.csv: + delimiter = ',' + elif format == TaskTableOutputFormat.bsv: + delimiter = '\N{BANANA}' + else: + raise ValueError(f"Unknown output format: {format}") + csv_writer = csv.writer(sys.stdout, delimiter=delimiter) + csv_writer.writerow(_row_header) + csv_writer.writerows(table) def display_awaited_by_tasks_tree(pid: int) -> None: diff --git a/Lib/calendar.py b/Lib/calendar.py index 45bb265a65602c..fd6c561a9d39b8 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -149,6 +149,14 @@ def __len__(self): except ValueError: standalone_month_name = month_name standalone_month_abbr = month_abbr +else: + # Some systems that do not support '%OB' will keep it as-is (i.e., + # we get [..., '%OB', '%OB', '%OB']), so for non-distinct names, + # we fall back to month_name/month_abbr. + if len(set(standalone_month_name)) != len(set(month_name)): + standalone_month_name = month_name + if len(set(standalone_month_abbr)) != len(set(month_abbr)): + standalone_month_abbr = month_abbr def isleap(year): diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 83ea623dce6281..d29f1615f276d2 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1283,6 +1283,10 @@ def _add_slots(cls, is_frozen, weakref_slot, defined_fields): if '__slots__' in cls.__dict__: raise TypeError(f'{cls.__name__} already specifies __slots__') + # gh-102069: Remove existing __weakref__ descriptor. + # gh-135228: Make sure the original class can be garbage collected. + sys._clear_type_descriptors(cls) + # Create a new dict for our new class. cls_dict = dict(cls.__dict__) field_names = tuple(f.name for f in fields(cls)) @@ -1300,12 +1304,6 @@ def _add_slots(cls, is_frozen, weakref_slot, defined_fields): # available in _MARKER. cls_dict.pop(field_name, None) - # Remove __dict__ itself. - cls_dict.pop('__dict__', None) - - # Clear existing `__weakref__` descriptor, it belongs to a previous type: - cls_dict.pop('__weakref__', None) # gh-102069 - # And finally create the class. qualname = getattr(cls, '__qualname__', None) newcls = type(cls)(cls.__name__, cls.__bases__, cls_dict) diff --git a/Lib/difflib.py b/Lib/difflib.py index 487936dbf47cdc..fedc85009aa03b 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -30,6 +30,7 @@ 'Differ','IS_CHARACTER_JUNK', 'IS_LINE_JUNK', 'context_diff', 'unified_diff', 'diff_bytes', 'HtmlDiff', 'Match'] +from _colorize import can_colorize, get_theme from heapq import nlargest as _nlargest from collections import namedtuple as _namedtuple from types import GenericAlias @@ -1094,7 +1095,7 @@ def _format_range_unified(start, stop): return '{},{}'.format(beginning, length) def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', - tofiledate='', n=3, lineterm='\n'): + tofiledate='', n=3, lineterm='\n', *, color=False): r""" Compare two sequences of lines; generate the delta as a unified diff. @@ -1111,6 +1112,10 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', For inputs that do not have trailing newlines, set the lineterm argument to "" so that the output will be uniformly newline free. + Set 'color' to True to enable output in color, similar to + 'git diff --color'. Even if enabled, it can be + controlled using environment variables such as 'NO_COLOR'. + The unidiff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. @@ -1134,6 +1139,11 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', four """ + if color and can_colorize(): + t = get_theme(force_color=True).difflib + else: + t = get_theme(force_no_color=True).difflib + _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm) started = False for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): @@ -1141,25 +1151,25 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', started = True fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' todate = '\t{}'.format(tofiledate) if tofiledate else '' - yield '--- {}{}{}'.format(fromfile, fromdate, lineterm) - yield '+++ {}{}{}'.format(tofile, todate, lineterm) + yield f'{t.header}--- {fromfile}{fromdate}{lineterm}{t.reset}' + yield f'{t.header}+++ {tofile}{todate}{lineterm}{t.reset}' first, last = group[0], group[-1] file1_range = _format_range_unified(first[1], last[2]) file2_range = _format_range_unified(first[3], last[4]) - yield '@@ -{} +{} @@{}'.format(file1_range, file2_range, lineterm) + yield f'{t.hunk}@@ -{file1_range} +{file2_range} @@{lineterm}{t.reset}' for tag, i1, i2, j1, j2 in group: if tag == 'equal': for line in a[i1:i2]: - yield ' ' + line + yield f'{t.context} {line}{t.reset}' continue if tag in {'replace', 'delete'}: for line in a[i1:i2]: - yield '-' + line + yield f'{t.removed}-{line}{t.reset}' if tag in {'replace', 'insert'}: for line in b[j1:j2]: - yield '+' + line + yield f'{t.added}+{line}{t.reset}' ######################################################################## diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index aa641e94a8b336..4bd85990e8614a 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] -_PIP_VERSION = "25.1.1" +_PIP_VERSION = "25.2" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora diff --git a/Lib/ensurepip/_bundled/pip-25.1.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl similarity index 58% rename from Lib/ensurepip/_bundled/pip-25.1.1-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl index 2fdcfbf9ff8139..e14bb3f37c0ff4 100644 Binary files a/Lib/ensurepip/_bundled/pip-25.1.1-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl differ diff --git a/Lib/glob.py b/Lib/glob.py index 1e48fe43167200..5d42077003a240 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -122,21 +122,6 @@ def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): return [basename] return [] -_deprecated_function_message = ( - "{name} is deprecated and will be removed in Python {remove}. Use " - "glob.glob and pass a directory to its root_dir argument instead." -) - -def glob0(dirname, pattern): - import warnings - warnings._deprecated("glob.glob0", _deprecated_function_message, remove=(3, 15)) - return _glob0(dirname, pattern, None, False) - -def glob1(dirname, pattern): - import warnings - warnings._deprecated("glob.glob1", _deprecated_function_message, remove=(3, 15)) - return _glob1(dirname, pattern, None, False) - # This helper function recursively yields relative pathnames inside a literal # directory. diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 694b1b09a0567c..74349bb63d66e2 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -426,7 +426,7 @@ def OutputString(self, attrs=None): ( # Optional group: there may not be a value. \s*=\s* # Equal Sign (?P # Start of group 'val' - "(?:[^\\"]|\\.)*" # Any double-quoted string + "(?:\\"|.)*?" # Any double-quoted string | # or # Special case for "expires" attr (\w{3,6}day|\w{3}),\s # Day of the week or abbreviated day diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 29f01f77eff4a0..1e47495f65fa02 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -64,20 +64,14 @@ def invalidate_caches(self): class ResourceLoader(Loader): """Abstract base class for loaders which can return data from their - back-end storage. + back-end storage to facilitate reading data to perform an import. This ABC represents one of the optional protocols specified by PEP 302. - """ - - def __init__(self): - import warnings - warnings.warn('importlib.abc.ResourceLoader is deprecated in ' - 'favour of supporting resource loading through ' - 'importlib.resources.abc.TraversableResources.', - DeprecationWarning, stacklevel=2) - super().__init__() + For directly loading resources, use TraversableResources instead. This class + primarily exists for backwards compatibility with other ABCs in this module. + """ @abc.abstractmethod def get_data(self, path): diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 8b60b9d5c9cd51..7470dc6c52d3a8 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1479,6 +1479,10 @@ def with_hostmask(self): return '%s/%s' % (self._string_from_ip_int(self._ip), self.hostmask) + @property + def is_unspecified(self): + return self._ip == 0 and self.network.is_unspecified + class IPv4Network(_BaseV4, _BaseNetwork): diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index c4d0ca81e7034a..c53092f6e34b32 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -20,6 +20,7 @@ import sys import threading import warnings +from collections import deque from . import spawn from . import util @@ -62,6 +63,7 @@ def __init__(self): self._fd = None self._pid = None self._exitcode = None + self._reentrant_messages = deque() def _reentrant_call_error(self): # gh-109629: this happens if an explicit call to the ResourceTracker @@ -98,7 +100,7 @@ def _stop_locked( # This shouldn't happen (it might when called by a finalizer) # so we check for it anyway. if self._lock._recursion_count() > 1: - return self._reentrant_call_error() + raise self._reentrant_call_error() if self._fd is None: # not running return @@ -128,69 +130,99 @@ def ensure_running(self): This can be run from any process. Usually a child process will use the resource created by its parent.''' + return self._ensure_running_and_write() + + def _teardown_dead_process(self): + os.close(self._fd) + + # Clean-up to avoid dangling processes. + try: + # _pid can be None if this process is a child from another + # python process, which has started the resource_tracker. + if self._pid is not None: + os.waitpid(self._pid, 0) + except ChildProcessError: + # The resource_tracker has already been terminated. + pass + self._fd = None + self._pid = None + self._exitcode = None + + warnings.warn('resource_tracker: process died unexpectedly, ' + 'relaunching. Some resources might leak.') + + def _launch(self): + fds_to_pass = [] + try: + fds_to_pass.append(sys.stderr.fileno()) + except Exception: + pass + r, w = os.pipe() + try: + fds_to_pass.append(r) + # process will out live us, so no need to wait on pid + exe = spawn.get_executable() + args = [ + exe, + *util._args_from_interpreter_flags(), + '-c', + f'from multiprocessing.resource_tracker import main;main({r})', + ] + # bpo-33613: Register a signal mask that will block the signals. + # This signal mask will be inherited by the child that is going + # to be spawned and will protect the child from a race condition + # that can make the child die before it registers signal handlers + # for SIGINT and SIGTERM. The mask is unregistered after spawning + # the child. + prev_sigmask = None + try: + if _HAVE_SIGMASK: + prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) + pid = util.spawnv_passfds(exe, args, fds_to_pass) + finally: + if prev_sigmask is not None: + signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask) + except: + os.close(w) + raise + else: + self._fd = w + self._pid = pid + finally: + os.close(r) + + def _ensure_running_and_write(self, msg=None): with self._lock: if self._lock._recursion_count() > 1: # The code below is certainly not reentrant-safe, so bail out - return self._reentrant_call_error() + if msg is None: + raise self._reentrant_call_error() + return self._reentrant_messages.append(msg) + if self._fd is not None: # resource tracker was launched before, is it still running? - if self._check_alive(): - # => still alive - return - # => dead, launch it again - os.close(self._fd) - - # Clean-up to avoid dangling processes. + if msg is None: + to_send = b'PROBE:0:noop\n' + else: + to_send = msg try: - # _pid can be None if this process is a child from another - # python process, which has started the resource_tracker. - if self._pid is not None: - os.waitpid(self._pid, 0) - except ChildProcessError: - # The resource_tracker has already been terminated. - pass - self._fd = None - self._pid = None - self._exitcode = None + self._write(to_send) + except OSError: + self._teardown_dead_process() + self._launch() - warnings.warn('resource_tracker: process died unexpectedly, ' - 'relaunching. Some resources might leak.') + msg = None # message was sent in probe + else: + self._launch() - fds_to_pass = [] + while True: try: - fds_to_pass.append(sys.stderr.fileno()) - except Exception: - pass - cmd = 'from multiprocessing.resource_tracker import main;main(%d)' - r, w = os.pipe() - try: - fds_to_pass.append(r) - # process will out live us, so no need to wait on pid - exe = spawn.get_executable() - args = [exe] + util._args_from_interpreter_flags() - args += ['-c', cmd % r] - # bpo-33613: Register a signal mask that will block the signals. - # This signal mask will be inherited by the child that is going - # to be spawned and will protect the child from a race condition - # that can make the child die before it registers signal handlers - # for SIGINT and SIGTERM. The mask is unregistered after spawning - # the child. - prev_sigmask = None - try: - if _HAVE_SIGMASK: - prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) - pid = util.spawnv_passfds(exe, args, fds_to_pass) - finally: - if prev_sigmask is not None: - signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask) - except: - os.close(w) - raise - else: - self._fd = w - self._pid = pid - finally: - os.close(r) + reentrant_msg = self._reentrant_messages.popleft() + except IndexError: + break + self._write(reentrant_msg) + if msg is not None: + self._write(msg) def _check_alive(self): '''Check that the pipe has not been closed by sending a probe.''' @@ -211,27 +243,18 @@ def unregister(self, name, rtype): '''Unregister name of resource with resource tracker.''' self._send('UNREGISTER', name, rtype) + def _write(self, msg): + nbytes = os.write(self._fd, msg) + assert nbytes == len(msg), f"{nbytes=} != {len(msg)=}" + def _send(self, cmd, name, rtype): - try: - self.ensure_running() - except ReentrantCallError: - # The code below might or might not work, depending on whether - # the resource tracker was already running and still alive. - # Better warn the user. - # (XXX is warnings.warn itself reentrant-safe? :-) - warnings.warn( - f"ResourceTracker called reentrantly for resource cleanup, " - f"which is unsupported. " - f"The {rtype} object {name!r} might leak.") - msg = '{0}:{1}:{2}\n'.format(cmd, name, rtype).encode('ascii') + msg = f"{cmd}:{name}:{rtype}\n".encode("ascii") if len(msg) > 512: # posix guarantees that writes to a pipe of less than PIPE_BUF # bytes are atomic, and that PIPE_BUF >= 512 raise ValueError('msg too long') - nbytes = os.write(self._fd, msg) - assert nbytes == len(msg), "nbytes {0:n} but len(msg) {1:n}".format( - nbytes, len(msg)) + self._ensure_running_and_write(msg) _resource_tracker = ResourceTracker() ensure_running = _resource_tracker.ensure_running diff --git a/Lib/numbers.py b/Lib/numbers.py index a2913e32cfada7..37fddb8917727b 100644 --- a/Lib/numbers.py +++ b/Lib/numbers.py @@ -290,18 +290,27 @@ def conjugate(self): class Rational(Real): - """.numerator and .denominator should be in lowest terms.""" + """To Real, Rational adds numerator and denominator properties. + + The numerator and denominator values should be in lowest terms, + with a positive denominator. + """ __slots__ = () @property @abstractmethod def numerator(self): + """The numerator of a rational number in lowest terms.""" raise NotImplementedError @property @abstractmethod def denominator(self): + """The denominator of a rational number in lowest terms. + + This denominator should be positive. + """ raise NotImplementedError # Concrete implementation of Real's conversion to float. diff --git a/Lib/profile/_sync_coordinator.py b/Lib/profile/_sync_coordinator.py new file mode 100644 index 00000000000000..79e8858ca17529 --- /dev/null +++ b/Lib/profile/_sync_coordinator.py @@ -0,0 +1,243 @@ +""" +Internal synchronization coordinator for the sample profiler. + +This module is used internally by the sample profiler to coordinate +the startup of target processes. It should not be called directly by users. +""" + +import os +import sys +import socket +import runpy +import time +from typing import List, NoReturn + + +class CoordinatorError(Exception): + """Base exception for coordinator errors.""" + pass + + +class ArgumentError(CoordinatorError): + """Raised when invalid arguments are provided.""" + pass + + +class SyncError(CoordinatorError): + """Raised when synchronization with profiler fails.""" + pass + + +class TargetError(CoordinatorError): + """Raised when target execution fails.""" + pass + + +def _validate_arguments(args: List[str]) -> tuple[int, str, List[str]]: + """ + Validate and parse command line arguments. + + Args: + args: Command line arguments including script name + + Returns: + Tuple of (sync_port, working_directory, target_args) + + Raises: + ArgumentError: If arguments are invalid + """ + if len(args) < 4: + raise ArgumentError( + "Insufficient arguments. Expected: [args...]" + ) + + try: + sync_port = int(args[1]) + if not (1 <= sync_port <= 65535): + raise ValueError("Port out of range") + except ValueError as e: + raise ArgumentError(f"Invalid sync port '{args[1]}': {e}") from e + + cwd = args[2] + if not os.path.isdir(cwd): + raise ArgumentError(f"Working directory does not exist: {cwd}") + + target_args = args[3:] + if not target_args: + raise ArgumentError("No target specified") + + return sync_port, cwd, target_args + + +# Constants for socket communication +_MAX_RETRIES = 3 +_INITIAL_RETRY_DELAY = 0.1 +_SOCKET_TIMEOUT = 2.0 +_READY_MESSAGE = b"ready" + + +def _signal_readiness(sync_port: int) -> None: + """ + Signal readiness to the profiler via TCP socket. + + Args: + sync_port: Port number where profiler is listening + + Raises: + SyncError: If unable to signal readiness + """ + last_error = None + + for attempt in range(_MAX_RETRIES): + try: + # Use context manager for automatic cleanup + with socket.create_connection(("127.0.0.1", sync_port), timeout=_SOCKET_TIMEOUT) as sock: + sock.send(_READY_MESSAGE) + return + except (socket.error, OSError) as e: + last_error = e + if attempt < _MAX_RETRIES - 1: + # Exponential backoff before retry + time.sleep(_INITIAL_RETRY_DELAY * (2 ** attempt)) + + # If we get here, all retries failed + raise SyncError(f"Failed to signal readiness after {_MAX_RETRIES} attempts: {last_error}") from last_error + + +def _setup_environment(cwd: str) -> None: + """ + Set up the execution environment. + + Args: + cwd: Working directory to change to + + Raises: + TargetError: If unable to set up environment + """ + try: + os.chdir(cwd) + except OSError as e: + raise TargetError(f"Failed to change to directory {cwd}: {e}") from e + + # Add current directory to sys.path if not present (for module imports) + if cwd not in sys.path: + sys.path.insert(0, cwd) + + +def _execute_module(module_name: str, module_args: List[str]) -> None: + """ + Execute a Python module. + + Args: + module_name: Name of the module to execute + module_args: Arguments to pass to the module + + Raises: + TargetError: If module execution fails + """ + # Replace sys.argv to match how Python normally runs modules + # When running 'python -m module args', sys.argv is ["__main__.py", "args"] + sys.argv = [f"__main__.py"] + module_args + + try: + runpy.run_module(module_name, run_name="__main__", alter_sys=True) + except ImportError as e: + raise TargetError(f"Module '{module_name}' not found: {e}") from e + except SystemExit: + # SystemExit is normal for modules + pass + except Exception as e: + raise TargetError(f"Error executing module '{module_name}': {e}") from e + + +def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: + """ + Execute a Python script. + + Args: + script_path: Path to the script to execute + script_args: Arguments to pass to the script + cwd: Current working directory for path resolution + + Raises: + TargetError: If script execution fails + """ + # Make script path absolute if it isn't already + if not os.path.isabs(script_path): + script_path = os.path.join(cwd, script_path) + + if not os.path.isfile(script_path): + raise TargetError(f"Script not found: {script_path}") + + # Replace sys.argv to match original script call + sys.argv = [script_path] + script_args + + try: + with open(script_path, 'rb') as f: + source_code = f.read() + + # Compile and execute the script + code = compile(source_code, script_path, 'exec') + exec(code, {'__name__': '__main__', '__file__': script_path}) + except FileNotFoundError as e: + raise TargetError(f"Script file not found: {script_path}") from e + except PermissionError as e: + raise TargetError(f"Permission denied reading script: {script_path}") from e + except SyntaxError as e: + raise TargetError(f"Syntax error in script {script_path}: {e}") from e + except SystemExit: + # SystemExit is normal for scripts + pass + except Exception as e: + raise TargetError(f"Error executing script '{script_path}': {e}") from e + + +def main() -> NoReturn: + """ + Main coordinator function. + + This function coordinates the startup of a target Python process + with the sample profiler by signaling when the process is ready + to be profiled. + """ + try: + # Parse and validate arguments + sync_port, cwd, target_args = _validate_arguments(sys.argv) + + # Set up execution environment + _setup_environment(cwd) + + # Signal readiness to profiler + _signal_readiness(sync_port) + + # Execute the target + if target_args[0] == "-m": + # Module execution + if len(target_args) < 2: + raise ArgumentError("Module name required after -m") + + module_name = target_args[1] + module_args = target_args[2:] + _execute_module(module_name, module_args) + else: + # Script execution + script_path = target_args[0] + script_args = target_args[1:] + _execute_script(script_path, script_args, cwd) + + except CoordinatorError as e: + print(f"Profiler coordinator error: {e}", file=sys.stderr) + sys.exit(1) + except KeyboardInterrupt: + print("Interrupted", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Unexpected error in profiler coordinator: {e}", file=sys.stderr) + sys.exit(1) + + # Normal exit + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/Lib/profile/sample.py b/Lib/profile/sample.py index 97d23611e67ad7..5f3c86d4839a66 100644 --- a/Lib/profile/sample.py +++ b/Lib/profile/sample.py @@ -2,6 +2,8 @@ import _remote_debugging import os import pstats +import socket +import subprocess import statistics import sys import sysconfig @@ -12,14 +14,111 @@ from .pstats_collector import PstatsCollector from .stack_collector import CollapsedStackCollector -FREE_THREADED_BUILD = sysconfig.get_config_var("Py_GIL_DISABLED") is not None +_FREE_THREADED_BUILD = sysconfig.get_config_var("Py_GIL_DISABLED") is not None +_MAX_STARTUP_ATTEMPTS = 5 +_STARTUP_RETRY_DELAY_SECONDS = 0.1 +_HELP_DESCRIPTION = """Sample a process's stack frames and generate profiling data. +Supports the following target modes: + - -p PID: Profile an existing process by PID + - -m MODULE [ARGS...]: Profile a module as python -m module ... + - filename [ARGS...]: Profile the specified script by running it in a subprocess + +Examples: + # Profile process 1234 for 10 seconds with default settings + python -m profile.sample -p 1234 + + # Profile a script by running it in a subprocess + python -m profile.sample myscript.py arg1 arg2 + + # Profile a module by running it as python -m module in a subprocess + python -m profile.sample -m mymodule arg1 arg2 + + # Profile with custom interval and duration, save to file + python -m profile.sample -i 50 -d 30 -o profile.stats -p 1234 + + # Generate collapsed stacks for flamegraph + python -m profile.sample --collapsed -p 1234 + + # Profile all threads, sort by total time + python -m profile.sample -a --sort-tottime -p 1234 + + # Profile for 1 minute with 1ms sampling interval + python -m profile.sample -i 1000 -d 60 -p 1234 + + # Show only top 20 functions sorted by direct samples + python -m profile.sample --sort-nsamples -l 20 -p 1234 + + # Profile all threads and save collapsed stacks + python -m profile.sample -a --collapsed -o stacks.txt -p 1234 + + # Profile with real-time sampling statistics + python -m profile.sample --realtime-stats -p 1234 + + # Sort by sample percentage to find most sampled functions + python -m profile.sample --sort-sample-pct -p 1234 + + # Sort by cumulative samples to find functions most on call stack + python -m profile.sample --sort-nsamples-cumul -p 1234""" + + +# Constants for socket synchronization +_SYNC_TIMEOUT = 5.0 +_PROCESS_KILL_TIMEOUT = 2.0 +_READY_MESSAGE = b"ready" +_RECV_BUFFER_SIZE = 1024 + + +def _run_with_sync(original_cmd): + """Run a command with socket-based synchronization and return the process.""" + # Create a TCP socket for synchronization with better socket options + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sync_sock: + # Set SO_REUSEADDR to avoid "Address already in use" errors + sync_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sync_sock.bind(("127.0.0.1", 0)) # Let OS choose a free port + sync_port = sync_sock.getsockname()[1] + sync_sock.listen(1) + sync_sock.settimeout(_SYNC_TIMEOUT) + + # Get current working directory to preserve it + cwd = os.getcwd() + + # Build command using the sync coordinator + target_args = original_cmd[1:] # Remove python executable + cmd = (sys.executable, "-m", "profile._sync_coordinator", str(sync_port), cwd) + tuple(target_args) + + # Start the process with coordinator + process = subprocess.Popen(cmd) + + try: + # Wait for ready signal with timeout + with sync_sock.accept()[0] as conn: + ready_signal = conn.recv(_RECV_BUFFER_SIZE) + + if ready_signal != _READY_MESSAGE: + raise RuntimeError(f"Invalid ready signal received: {ready_signal!r}") + + except socket.timeout: + # If we timeout, kill the process and raise an error + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=_PROCESS_KILL_TIMEOUT) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + raise RuntimeError("Process failed to signal readiness within timeout") + + return process + + + class SampleProfiler: def __init__(self, pid, sample_interval_usec, all_threads): self.pid = pid self.sample_interval_usec = sample_interval_usec self.all_threads = all_threads - if FREE_THREADED_BUILD: + if _FREE_THREADED_BUILD: self.unwinder = _remote_debugging.RemoteUnwinder( self.pid, all_threads=self.all_threads ) @@ -50,6 +149,7 @@ def sample(self, collector, duration_sec=10): stack_frames = self.unwinder.get_stack_trace() collector.collect(stack_frames) except ProcessLookupError: + duration_sec = current_time - start_time break except (RuntimeError, UnicodeDecodeError, MemoryError, OSError): errors += 1 @@ -532,56 +632,54 @@ def _validate_collapsed_format_args(args, parser): f"The following options are only valid with --pstats format: {', '.join(invalid_opts)}" ) - # Set default output filename for collapsed format - if not args.outfile: + # Set default output filename for collapsed format only if we have a PID + # For module/script execution, this will be set later with the subprocess PID + if not args.outfile and args.pid is not None: args.outfile = f"collapsed.{args.pid}.txt" +def wait_for_process_and_sample(pid, sort_value, args): + """Sample the process immediately since it has already signaled readiness.""" + # Set default collapsed filename with subprocess PID if not already set + filename = args.outfile + if not filename and args.format == "collapsed": + filename = f"collapsed.{pid}.txt" + + sample( + pid, + sort=sort_value, + sample_interval_usec=args.interval, + duration_sec=args.duration, + filename=filename, + all_threads=args.all_threads, + limit=args.limit, + show_summary=not args.no_summary, + output_format=args.format, + realtime_stats=args.realtime_stats, + ) + + def main(): # Create the main parser parser = argparse.ArgumentParser( - description=( - "Sample a process's stack frames and generate profiling data.\n" - "Supports two output formats:\n" - " - pstats: Detailed profiling statistics with sorting options\n" - " - collapsed: Stack traces for generating flamegraphs\n" - "\n" - "Examples:\n" - " # Profile process 1234 for 10 seconds with default settings\n" - " python -m profile.sample 1234\n" - "\n" - " # Profile with custom interval and duration, save to file\n" - " python -m profile.sample -i 50 -d 30 -o profile.stats 1234\n" - "\n" - " # Generate collapsed stacks for flamegraph\n" - " python -m profile.sample --collapsed 1234\n" - "\n" - " # Profile all threads, sort by total time\n" - " python -m profile.sample -a --sort-tottime 1234\n" - "\n" - " # Profile for 1 minute with 1ms sampling interval\n" - " python -m profile.sample -i 1000 -d 60 1234\n" - "\n" - " # Show only top 20 functions sorted by direct samples\n" - " python -m profile.sample --sort-nsamples -l 20 1234\n" - "\n" - " # Profile all threads and save collapsed stacks\n" - " python -m profile.sample -a --collapsed -o stacks.txt 1234\n" - "\n" - " # Profile with real-time sampling statistics\n" - " python -m profile.sample --realtime-stats 1234\n" - "\n" - " # Sort by sample percentage to find most sampled functions\n" - " python -m profile.sample --sort-sample-pct 1234\n" - "\n" - " # Sort by cumulative samples to find functions most on call stack\n" - " python -m profile.sample --sort-nsamples-cumul 1234" - ), + description=_HELP_DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter, ) - # Required arguments - parser.add_argument("pid", type=int, help="Process ID to sample") + # Target selection + target_group = parser.add_mutually_exclusive_group(required=False) + target_group.add_argument( + "-p", "--pid", type=int, help="Process ID to sample" + ) + target_group.add_argument( + "-m", "--module", + help="Run and profile a module as python -m module [ARGS...]" + ) + parser.add_argument( + "args", + nargs=argparse.REMAINDER, + help="Script to run and profile, with optional arguments" + ) # Sampling options sampling_group = parser.add_argument_group("Sampling configuration") @@ -712,19 +810,55 @@ def main(): sort_value = args.sort if args.sort is not None else 2 - sample( - args.pid, - sample_interval_usec=args.interval, - duration_sec=args.duration, - filename=args.outfile, - all_threads=args.all_threads, - limit=args.limit, - sort=sort_value, - show_summary=not args.no_summary, - output_format=args.format, - realtime_stats=args.realtime_stats, - ) + if args.module is not None and not args.module: + parser.error("argument -m/--module: expected one argument") + + # Validate that we have exactly one target type + # Note: args can be present with -m (module arguments) but not as standalone script + has_pid = args.pid is not None + has_module = args.module is not None + has_script = bool(args.args) and args.module is None + + target_count = sum([has_pid, has_module, has_script]) + + if target_count == 0: + parser.error("one of the arguments -p/--pid -m/--module or script name is required") + elif target_count > 1: + parser.error("only one target type can be specified: -p/--pid, -m/--module, or script") + + if args.pid: + sample( + args.pid, + sample_interval_usec=args.interval, + duration_sec=args.duration, + filename=args.outfile, + all_threads=args.all_threads, + limit=args.limit, + sort=sort_value, + show_summary=not args.no_summary, + output_format=args.format, + realtime_stats=args.realtime_stats, + ) + elif args.module or args.args: + if args.module: + cmd = (sys.executable, "-m", args.module, *args.args) + else: + cmd = (sys.executable, *args.args) + + # Use synchronized process startup + process = _run_with_sync(cmd) + # Process has already signaled readiness, start sampling immediately + try: + wait_for_process_and_sample(process.pid, sort_value, args) + finally: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + process.kill() + process.wait() if __name__ == "__main__": main() diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 49e0986517ce97..c583b74ce54fc6 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -412,7 +412,13 @@ def _init_non_posix(vars): vars['EXE'] = '.exe' vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) - vars['TZPATH'] = '' + # No standard path exists on Windows for this, but we'll check + # whether someone is imitating a POSIX-like layout + check_tzpath = os.path.join(vars['prefix'], 'share', 'zoneinfo') + if os.path.exists(check_tzpath): + vars['TZPATH'] = check_tzpath + else: + vars['TZPATH'] = '' # # public APIs diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 45f58eb8ac93cf..c603ba019ab481 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -353,7 +353,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, fileobj = _StreamProxy(fileobj) comptype = fileobj.getcomptype() - self.name = name or "" + self.name = os.fspath(name) if name is not None else "" self.mode = mode self.comptype = comptype self.fileobj = fileobj @@ -2723,6 +2723,9 @@ def makelink_with_filter(self, tarinfo, targetpath, return else: if os.path.exists(tarinfo._link_target): + if os.path.lexists(targetpath): + # Avoid FileExistsError on following os.link. + os.unlink(targetpath) os.link(tarinfo._link_target, targetpath) return except symlink_exception: diff --git a/Lib/test/.ruff.toml b/Lib/test/.ruff.toml index f1a967203ce4ba..a1b749798fa263 100644 --- a/Lib/test/.ruff.toml +++ b/Lib/test/.ruff.toml @@ -1,5 +1,7 @@ extend = "../../.ruff.toml" # Inherit the project-wide settings +target-version = "py312" + extend-exclude = [ # Excluded (run with the other AC files in its own separate ruff job in pre-commit) "test_clinic.py", @@ -8,15 +10,20 @@ extend-exclude = [ # Non UTF-8 files "encoded_modules/module_iso_8859_1.py", "encoded_modules/module_koi8_r.py", - # SyntaxError because of t-strings - "test_annotationlib.py", - "test_string/test_templatelib.py", - "test_tstring.py", # New grammar constructions may not yet be recognized by Ruff, # and tests re-use the same names as only the grammar is being checked. "test_grammar.py", ] +[per-file-target-version] +# Type parameter defaults +"test_type_params.py" = "py313" + +# Template string literals +"test_annotationlib.py" = "py314" +"test_string/test_templatelib.py" = "py314" +"test_tstring.py" = "py314" + [lint] select = [ "F401", # Unused import diff --git a/Lib/test/_test_eintr.py b/Lib/test/_test_eintr.py index 0ce42276bfe3d6..4a050792df73c4 100644 --- a/Lib/test/_test_eintr.py +++ b/Lib/test/_test_eintr.py @@ -380,6 +380,8 @@ def os_open(self, path): @unittest.skipIf(sys.platform == "darwin", "hangs under macOS; see bpo-25234, bpo-35363") + @unittest.skipIf(sys.platform.startswith('netbsd'), + "hangs on NetBSD; see gh-137397") def test_os_open(self): self._test_open("fd = os.open(path, os.O_RDONLY)\nos.close(fd)", self.os_open) diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index dc5b4b27a07f99..b0f7e402469ffc 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5084,14 +5084,18 @@ Test_an_metho_arg_named_arg_impl(TestObj *self, int arg) Test.__init__ *args: tuple -Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE. +Varargs init method. + +For example, nargs is translated to PyTuple_GET_SIZE. [clinic start generated code]*/ PyDoc_STRVAR(Test___init____doc__, "Test(*args)\n" "--\n" "\n" -"Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE."); +"Varargs init method.\n" +"\n" +"For example, nargs is translated to PyTuple_GET_SIZE."); static int Test___init___impl(TestObj *self, PyObject *args); @@ -5120,21 +5124,25 @@ Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) static int Test___init___impl(TestObj *self, PyObject *args) -/*[clinic end generated code: output=f172425cec373cd6 input=4b8388c4e6baab6f]*/ +/*[clinic end generated code: output=0e5836c40dbc2397 input=a615a4485c0fc3e2]*/ /*[clinic input] @classmethod Test.__new__ *args: tuple -Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE. +Varargs new method. + +For example, nargs is translated to PyTuple_GET_SIZE. [clinic start generated code]*/ PyDoc_STRVAR(Test__doc__, "Test(*args)\n" "--\n" "\n" -"Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE."); +"Varargs new method.\n" +"\n" +"For example, nargs is translated to PyTuple_GET_SIZE."); static PyObject * Test_impl(PyTypeObject *type, PyObject *args); @@ -5162,7 +5170,7 @@ Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) static PyObject * Test_impl(PyTypeObject *type, PyObject *args) -/*[clinic end generated code: output=ee1e8892a67abd4a input=a8259521129cad20]*/ +/*[clinic end generated code: output=e6fba0c8951882fd input=8ce30adb836aeacb]*/ /*[clinic input] diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 3bd3a866570042..4e42ae52765d0f 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -2147,14 +2147,20 @@ def test_fromisocalendar_value_errors(self): (10000, 1, 1), (0, 1, 1), (9999999, 1, 1), + ] + for isocal in isocals: + with self.subTest(isocal=isocal): + with self.assertRaises(ValueError): + self.theclass.fromisocalendar(*isocal) + + isocals = [ (2<<32, 1, 1), (2019, 2<<32, 1), (2019, 1, 2<<32), ] - for isocal in isocals: with self.subTest(isocal=isocal): - with self.assertRaises(ValueError): + with self.assertRaises((ValueError, OverflowError)): self.theclass.fromisocalendar(*isocal) def test_fromisocalendar_type_errors(self): @@ -2301,7 +2307,7 @@ def test_isoformat_timezone(self): dt = dt_base.replace(tzinfo=tzi) exp = exp_base + exp_tz with self.subTest(tzi=tzi): - assert dt.isoformat() == exp + self.assertEqual(dt.isoformat(), exp) def test_format(self): dt = self.theclass(2007, 9, 10, 4, 5, 1, 123) @@ -3349,7 +3355,7 @@ def test_fromisoformat_timezone(self): with self.subTest(tstr=dtstr): dt_rt = self.theclass.fromisoformat(dtstr) - assert dt == dt_rt, dt_rt + self.assertEqual(dt_rt, dt) def test_fromisoformat_separators(self): separators = [ @@ -3865,7 +3871,7 @@ def test_isoformat_timezone(self): t = t_base.replace(tzinfo=tzi) exp = exp_base + exp_tz with self.subTest(tzi=tzi): - assert t.isoformat() == exp + self.assertEqual(t.isoformat(), exp) def test_1653736(self): # verify it doesn't accept extra keyword arguments @@ -4350,7 +4356,7 @@ def utcoffset(self, t): elif x is d2: expected = -1 else: - assert y is d2 + self.assertIs(y, d2) expected = 1 self.assertEqual(got, expected) @@ -4678,7 +4684,7 @@ def test_fromisoformat_timezone(self): with self.subTest(tstr=tstr): t_rt = self.theclass.fromisoformat(tstr) - assert t == t_rt + self.assertEqual(t_rt, t) def test_fromisoformat_timespecs(self): time_bases = [ @@ -5515,7 +5521,7 @@ def utcoffset(self, t): elif x is d2: expected = timedelta(minutes=(11-59)-0) else: - assert y is d2 + self.assertIs(y, d2) expected = timedelta(minutes=0-(11-59)) self.assertEqual(got, expected) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 72b8ea89e62ee0..d94fb84a743828 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -536,7 +536,7 @@ def normalize_test_name(test_full_name: str, *, if is_error and short_name in _TEST_LIFECYCLE_HOOKS: if test_full_name.startswith(('setUpModule (', 'tearDownModule (')): # if setUpModule() or tearDownModule() failed, don't filter - # tests with the test file name, don't use use filters. + # tests with the test file name, don't use filters. return None # This means that we have a failure in a life-cycle hook, diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index cea2f09aae5d51..4bfd01ed14a0a1 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -541,7 +541,6 @@ def has_no_debug_ranges(): except ImportError: raise unittest.SkipTest("_testinternalcapi required") return not _testcapi.config_get('code_debug_ranges') - return not bool(config['code_debug_ranges']) def requires_debug_ranges(reason='requires co_positions / debug_ranges'): try: @@ -1686,7 +1685,7 @@ def check__all__(test_case, module, name_of_module=None, extra=(), 'module'. The 'name_of_module' argument can specify (as a string or tuple thereof) - what module(s) an API could be defined in in order to be detected as a + what module(s) an API could be defined in order to be detected as a public API. One case for this is when 'module' imports part of its public API from other modules, possibly a C backend (like 'csv' and its '_csv'). diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index a32d5d81d2bf2d..ec5a26bc1c05ad 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -474,19 +474,19 @@ def setUp(self): def test_signatures(self): # See https://github.com/python/cpython/issues/126654 - msg = r'_interpreters.exec\(\) argument 3 must be dict, not int' + msg = r"exec\(\) argument 'shared' must be dict, not int" with self.assertRaisesRegex(TypeError, msg): _interpreters.exec(self.id, 'a', 1) with self.assertRaisesRegex(TypeError, msg): _interpreters.exec(self.id, 'a', shared=1) - msg = r'_interpreters.run_string\(\) argument 3 must be dict, not int' + msg = r"run_string\(\) argument 'shared' must be dict, not int" with self.assertRaisesRegex(TypeError, msg): _interpreters.run_string(self.id, 'a', shared=1) - msg = r'_interpreters.run_func\(\) argument 3 must be dict, not int' + msg = r"run_func\(\) argument 'shared' must be dict, not int" with self.assertRaisesRegex(TypeError, msg): _interpreters.run_func(self.id, lambda: None, shared=1) # See https://github.com/python/cpython/issues/135855 - msg = r'_interpreters.set___main___attrs\(\) argument 2 must be dict, not int' + msg = r"set___main___attrs\(\) argument 'updates' must be dict, not int" with self.assertRaisesRegex(TypeError, msg): _interpreters.set___main___attrs(self.id, 1) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index ae0e73f08c5bd0..88e0d611647f28 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1365,6 +1365,11 @@ def test_annotations_to_string(self): class A: pass +TypeParamsAlias1 = int + +class TypeParamsSample[TypeParamsAlias1, TypeParamsAlias2]: + TypeParamsAlias2 = str + class TestForwardRefClass(unittest.TestCase): def test_forwardref_instance_type_error(self): @@ -1597,6 +1602,21 @@ class Gen[T]: ForwardRef("alias").evaluate(owner=Gen, locals={"alias": str}), str ) + def test_evaluate_with_type_params_and_scope_conflict(self): + for is_class in (False, True): + with self.subTest(is_class=is_class): + fwdref1 = ForwardRef("TypeParamsAlias1", owner=TypeParamsSample, is_class=is_class) + fwdref2 = ForwardRef("TypeParamsAlias2", owner=TypeParamsSample, is_class=is_class) + + self.assertIs( + fwdref1.evaluate(), + TypeParamsSample.__type_params__[0], + ) + self.assertIs( + fwdref2.evaluate(), + TypeParamsSample.TypeParamsAlias2, + ) + def test_fwdref_with_module(self): self.assertIs(ForwardRef("Format", module="annotationlib").evaluate(), Format) self.assertIs( diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 13dcb5238945b6..1e6f60074308e2 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -220,6 +220,131 @@ def test_negative_locations_for_compile(self): # This also must not crash: ast.parse(tree, optimize=2) + def test_docstring_optimization_single_node(self): + # https://github.com/python/cpython/issues/137308 + class_example1 = textwrap.dedent(''' + class A: + """Docstring""" + ''') + class_example2 = textwrap.dedent(''' + class A: + """ + Docstring""" + ''') + def_example1 = textwrap.dedent(''' + def some(): + """Docstring""" + ''') + def_example2 = textwrap.dedent(''' + def some(): + """Docstring + """ + ''') + async_def_example1 = textwrap.dedent(''' + async def some(): + """Docstring""" + ''') + async_def_example2 = textwrap.dedent(''' + async def some(): + """ + Docstring + """ + ''') + for code in [ + class_example1, + class_example2, + def_example1, + def_example2, + async_def_example1, + async_def_example2, + ]: + for opt_level in [0, 1, 2]: + with self.subTest(code=code, opt_level=opt_level): + mod = ast.parse(code, optimize=opt_level) + self.assertEqual(len(mod.body[0].body), 1) + if opt_level == 2: + pass_stmt = mod.body[0].body[0] + self.assertIsInstance(pass_stmt, ast.Pass) + self.assertEqual( + vars(pass_stmt), + { + 'lineno': 3, + 'col_offset': 4, + 'end_lineno': 3, + 'end_col_offset': 8, + }, + ) + else: + self.assertIsInstance(mod.body[0].body[0], ast.Expr) + self.assertIsInstance( + mod.body[0].body[0].value, + ast.Constant, + ) + + compile(code, "a", "exec") + compile(code, "a", "exec", optimize=opt_level) + compile(mod, "a", "exec") + compile(mod, "a", "exec", optimize=opt_level) + + def test_docstring_optimization_multiple_nodes(self): + # https://github.com/python/cpython/issues/137308 + class_example = textwrap.dedent( + """ + class A: + ''' + Docstring + ''' + x = 1 + """ + ) + + def_example = textwrap.dedent( + """ + def some(): + ''' + Docstring + + ''' + x = 1 + """ + ) + + async_def_example = textwrap.dedent( + """ + async def some(): + + '''Docstring + + ''' + x = 1 + """ + ) + + for code in [ + class_example, + def_example, + async_def_example, + ]: + for opt_level in [0, 1, 2]: + with self.subTest(code=code, opt_level=opt_level): + mod = ast.parse(code, optimize=opt_level) + if opt_level == 2: + self.assertNotIsInstance( + mod.body[0].body[0], + (ast.Pass, ast.Expr), + ) + else: + self.assertIsInstance(mod.body[0].body[0], ast.Expr) + self.assertIsInstance( + mod.body[0].body[0].value, + ast.Constant, + ) + + compile(code, "a", "exec") + compile(code, "a", "exec", optimize=opt_level) + compile(mod, "a", "exec") + compile(mod, "a", "exec", optimize=opt_level) + def test_slice(self): slc = ast.parse("x[::]").body[0].value.slice self.assertIsNone(slc.upper) diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index ade55338e63b69..4967f02b007e06 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -6,7 +6,7 @@ import textwrap from test import support -from test.support import import_helper +from test.support import import_helper, force_not_colorized from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE from test.support.script_helper import assert_python_failure, assert_python_ok from test.support.testcase import ExceptionIsLikeMixin @@ -337,6 +337,10 @@ def test_err_writeunraisable(self): self.assertIsNone(cm.unraisable.err_msg) self.assertIsNone(cm.unraisable.object) + @force_not_colorized + def test_err_writeunraisable_lines(self): + writeunraisable = _testcapi.err_writeunraisable + with (support.swap_attr(sys, 'unraisablehook', None), support.captured_stderr() as stderr): writeunraisable(CustomError('oops!'), hex) @@ -387,6 +391,10 @@ def test_err_formatunraisable(self): self.assertIsNone(cm.unraisable.err_msg) self.assertIsNone(cm.unraisable.object) + @force_not_colorized + def test_err_formatunraisable_lines(self): + formatunraisable = _testcapi.err_formatunraisable + with (support.swap_attr(sys, 'unraisablehook', None), support.captured_stderr() as stderr): formatunraisable(CustomError('oops!'), b'Error in %R', []) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index f30a1874ab96d4..3ed7a360d64e3c 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -489,6 +489,7 @@ def test_unmached_quote(self): self.assertRegex(err.decode('ascii', 'ignore'), 'SyntaxError') self.assertEqual(b'', out) + @force_not_colorized def test_stdout_flush_at_shutdown(self): # Issue #5319: if stdout.flush() fails at shutdown, an error should # be printed out. diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index d8666f7290e72e..fd7769e8c275d3 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3293,7 +3293,7 @@ def test_code_page_name(self): codecs.code_page_encode, 932, '\xff') self.assertRaisesRegex(UnicodeDecodeError, 'cp932', codecs.code_page_decode, 932, b'\x81\x00', 'strict', True) - self.assertRaisesRegex(UnicodeDecodeError, 'CP_UTF8', + self.assertRaisesRegex(UnicodeDecodeError, 'cp65001', codecs.code_page_decode, self.CP_UTF8, b'\xff', 'strict', True) def check_decode(self, cp, tests): diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index d9d61e5c2053e3..f33e4b3256a9b9 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -736,7 +736,7 @@ def validate_abstract_methods(self, abc, *names): stubs = methodstubs.copy() del stubs[name] C = type('C', (abc,), stubs) - self.assertRaises(TypeError, C, name) + self.assertRaises(TypeError, C) def validate_isinstance(self, abc, name): stub = lambda s, *args: 0 @@ -963,7 +963,7 @@ class AnextOnly: async def __anext__(self): raise StopAsyncIteration self.assertNotIsInstance(AnextOnly(), AsyncIterator) - self.validate_abstract_methods(AsyncIterator, '__anext__', '__aiter__') + self.validate_abstract_methods(AsyncIterator, '__anext__') def test_Iterable(self): # Check some non-iterables @@ -1159,7 +1159,7 @@ def test_Iterator(self): for x in samples: self.assertIsInstance(x, Iterator) self.assertIsSubclass(type(x), Iterator) - self.validate_abstract_methods(Iterator, '__next__', '__iter__') + self.validate_abstract_methods(Iterator, '__next__') # Issue 10565 class NextOnly: @@ -1843,8 +1843,7 @@ def test_Mapping(self): for sample in [dict]: self.assertIsInstance(sample(), Mapping) self.assertIsSubclass(sample, Mapping) - self.validate_abstract_methods(Mapping, '__contains__', '__iter__', '__len__', - '__getitem__') + self.validate_abstract_methods(Mapping, '__iter__', '__len__', '__getitem__') class MyMapping(Mapping): def __len__(self): return 0 @@ -1859,7 +1858,7 @@ def test_MutableMapping(self): for sample in [dict]: self.assertIsInstance(sample(), MutableMapping) self.assertIsSubclass(sample, MutableMapping) - self.validate_abstract_methods(MutableMapping, '__contains__', '__iter__', '__len__', + self.validate_abstract_methods(MutableMapping, '__iter__', '__len__', '__getitem__', '__setitem__', '__delitem__') def test_MutableMapping_subclass(self): @@ -1898,8 +1897,7 @@ def test_Sequence(self): self.assertIsInstance(memoryview(b""), Sequence) self.assertIsSubclass(memoryview, Sequence) self.assertIsSubclass(str, Sequence) - self.validate_abstract_methods(Sequence, '__contains__', '__iter__', '__len__', - '__getitem__') + self.validate_abstract_methods(Sequence, '__len__', '__getitem__') def test_Sequence_mixins(self): class SequenceSubclass(Sequence): @@ -1954,8 +1952,8 @@ def test_MutableSequence(self): self.assertIsSubclass(sample, MutableSequence) self.assertIsSubclass(array.array, MutableSequence) self.assertNotIsSubclass(str, MutableSequence) - self.validate_abstract_methods(MutableSequence, '__contains__', '__iter__', - '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert') + self.validate_abstract_methods(MutableSequence, '__len__', '__getitem__', + '__setitem__', '__delitem__', 'insert') def test_MutableSequence_mixins(self): # Test the mixins of MutableSequence by creating a minimal concrete diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 57e5f29b015637..8a66be9b331262 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1723,6 +1723,21 @@ def test_compound(self): self.assertIs(res, v[3]) self.assertEqual([e.called for e in v], [1, 1, 0, 1, 0]) + def test_exception(self): + # See gh-137288 + class Foo: + def __bool__(self): + raise NotImplementedError() + + a = Foo() + b = Foo() + + with self.assertRaises(NotImplementedError): + bool(a) + + with self.assertRaises(NotImplementedError): + c = a or b + @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): # Ensure that compiled code snippets have correct line and column numbers diff --git a/Lib/test/test_concurrent_futures/test_shutdown.py b/Lib/test/test_concurrent_futures/test_shutdown.py index 99b315b47e2530..43812248104c91 100644 --- a/Lib/test/test_concurrent_futures/test_shutdown.py +++ b/Lib/test/test_concurrent_futures/test_shutdown.py @@ -49,6 +49,7 @@ def test_interpreter_shutdown(self): self.assertFalse(err) self.assertEqual(out.strip(), b"apple") + @support.force_not_colorized def test_submit_after_interpreter_shutdown(self): # Test the atexit hook for shutdown of worker threads and processes rc, out, err = assert_python_ok('-c', """if 1: diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index e98a8f284cec9f..6bf5e5b3e5554b 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -3804,6 +3804,41 @@ class WithCorrectSuper(CorrectSuper): # that we create internally. self.assertEqual(CorrectSuper.args, ["default", "default"]) + def test_original_class_is_gced(self): + # gh-135228: Make sure when we replace the class with slots=True, the original class + # gets garbage collected. + def make_simple(): + @dataclass(slots=True) + class SlotsTest: + pass + + return SlotsTest + + def make_with_annotations(): + @dataclass(slots=True) + class SlotsTest: + x: int + + return SlotsTest + + def make_with_annotations_and_method(): + @dataclass(slots=True) + class SlotsTest: + x: int + + def method(self) -> int: + return self.x + + return SlotsTest + + for make in (make_simple, make_with_annotations, make_with_annotations_and_method): + with self.subTest(make=make): + C = make() + support.gc_collect() + candidates = [cls for cls in object.__subclasses__() if cls.__name__ == 'SlotsTest' + and cls.__firstlineno__ == make.__code__.co_firstlineno + 1] + self.assertEqual(candidates, [C]) + class TestDescriptors(unittest.TestCase): def test_set_name(self): diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 6ac584a08d1e86..0eab3f523dc5fe 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -1,5 +1,5 @@ import difflib -from test.support import findfile +from test.support import findfile, force_colorized import unittest import doctest import sys @@ -355,6 +355,22 @@ def test_range_format_context(self): self.assertEqual(fmt(3,6), '4,6') self.assertEqual(fmt(0,0), '0') + @force_colorized + def test_unified_diff_colored_output(self): + args = [['one', 'three'], ['two', 'three'], 'Original', 'Current', + '2005-01-26 23:30:50', '2010-04-02 10:20:52'] + actual = list(difflib.unified_diff(*args, lineterm='', color=True)) + + expect = [ + "\033[1m--- Original\t2005-01-26 23:30:50\033[0m", + "\033[1m+++ Current\t2010-04-02 10:20:52\033[0m", + "\033[36m@@ -1,2 +1,2 @@\033[0m", + "\033[31m-one\033[0m", + "\033[32m+two\033[0m", + "\033[0m three\033[0m", + ] + self.assertEqual(expect, actual) + class TestBytes(unittest.TestCase): # don't really care about the content of the output, just the fact diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 355990ed58ee09..fb47345857f1f8 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -453,52 +453,53 @@ def foo(a: int, b: str) -> str: """ dis_traceback = """\ -%4d RESUME 0 +%4d RESUME 0 -%4d NOP +%4d NOP -%4d L1: LOAD_SMALL_INT 1 - LOAD_SMALL_INT 0 - --> BINARY_OP 11 (/) - POP_TOP +%4d L1: LOAD_SMALL_INT 1 + LOAD_SMALL_INT 0 + --> BINARY_OP 11 (/) + POP_TOP -%4d L2: LOAD_FAST_CHECK 1 (tb) - RETURN_VALUE +%4d L2: LOAD_FAST_CHECK 1 (tb) + RETURN_VALUE - -- L3: PUSH_EXC_INFO + -- L3: PUSH_EXC_INFO -%4d LOAD_GLOBAL 0 (Exception) - CHECK_EXC_MATCH - POP_JUMP_IF_FALSE 24 (to L7) - NOT_TAKEN - STORE_FAST 0 (e) +%4d LOAD_GLOBAL 0 (Exception) + CHECK_EXC_MATCH + POP_JUMP_IF_FALSE 24 (to L9) + L4: NOT_TAKEN + L5: STORE_FAST 0 (e) -%4d L4: LOAD_FAST 0 (e) - LOAD_ATTR 2 (__traceback__) - STORE_FAST 1 (tb) - L5: POP_EXCEPT - LOAD_CONST 1 (None) - STORE_FAST 0 (e) - DELETE_FAST 0 (e) +%4d L6: LOAD_FAST 0 (e) + LOAD_ATTR 2 (__traceback__) + STORE_FAST 1 (tb) + L7: POP_EXCEPT + LOAD_CONST 1 (None) + STORE_FAST 0 (e) + DELETE_FAST 0 (e) -%4d LOAD_FAST 1 (tb) - RETURN_VALUE +%4d LOAD_FAST 1 (tb) + RETURN_VALUE - -- L6: LOAD_CONST 1 (None) - STORE_FAST 0 (e) - DELETE_FAST 0 (e) - RERAISE 1 + -- L8: LOAD_CONST 1 (None) + STORE_FAST 0 (e) + DELETE_FAST 0 (e) + RERAISE 1 -%4d L7: RERAISE 0 +%4d L9: RERAISE 0 - -- L8: COPY 3 - POP_EXCEPT - RERAISE 1 + -- L10: COPY 3 + POP_EXCEPT + RERAISE 1 ExceptionTable: L1 to L2 -> L3 [0] - L3 to L4 -> L8 [1] lasti - L4 to L5 -> L6 [1] lasti - L6 to L8 -> L8 [1] lasti + L3 to L4 -> L10 [1] lasti + L5 to L6 -> L10 [1] lasti + L6 to L7 -> L8 [1] lasti + L8 to L10 -> L10 [1] lasti """ % (TRACEBACK_CODE.co_firstlineno, TRACEBACK_CODE.co_firstlineno + 1, TRACEBACK_CODE.co_firstlineno + 2, @@ -567,11 +568,11 @@ def _with(c): %4d L3: PUSH_EXC_INFO WITH_EXCEPT_START TO_BOOL - POP_JUMP_IF_TRUE 2 (to L4) - NOT_TAKEN - RERAISE 2 - L4: POP_TOP - L5: POP_EXCEPT + POP_JUMP_IF_TRUE 2 (to L6) + L4: NOT_TAKEN + L5: RERAISE 2 + L6: POP_TOP + L7: POP_EXCEPT POP_TOP POP_TOP POP_TOP @@ -581,12 +582,13 @@ def _with(c): LOAD_CONST 1 (None) RETURN_VALUE - -- L6: COPY 3 + -- L8: COPY 3 POP_EXCEPT RERAISE 1 ExceptionTable: L1 to L2 -> L3 [2] lasti - L3 to L5 -> L6 [4] lasti + L3 to L4 -> L8 [4] lasti + L5 to L7 -> L8 [4] lasti """ % (_with.__code__.co_firstlineno, _with.__code__.co_firstlineno + 1, _with.__code__.co_firstlineno + 2, diff --git a/Lib/test/test_finalization.py b/Lib/test/test_finalization.py index 42871f8a09b16b..9dd68cf8d57413 100644 --- a/Lib/test/test_finalization.py +++ b/Lib/test/test_finalization.py @@ -174,7 +174,7 @@ def test_simple(self): gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) @@ -188,12 +188,12 @@ def test_simple_resurrect(self): gc.collect() self.assert_del_calls(ids) self.assert_survivors(ids) - self.assertIsNot(wr(), None) + self.assertIsNotNone(wr()) self.clear_survivors() gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) @support.cpython_only def test_non_gc(self): @@ -265,7 +265,7 @@ def test_simple(self): gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) @@ -276,19 +276,24 @@ def test_simple_resurrect(self): s = SelfCycleResurrector() ids = [id(s)] wr = weakref.ref(s) + wrc = weakref.ref(s, lambda x: None) del s gc.collect() self.assert_del_calls(ids) self.assert_survivors(ids) - # XXX is this desirable? - self.assertIs(wr(), None) + # This used to be None because weakrefs were cleared before + # calling finalizers. Now they are cleared after. + self.assertIsNotNone(wr()) + # A weakref with a callback is still cleared before calling + # finalizers. + self.assertIsNone(wrc()) # When trying to destroy the object a second time, __del__ # isn't called anymore (and the object isn't resurrected). self.clear_survivors() gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) def test_simple_suicide(self): # Test the GC is able to deal with an object that kills its last @@ -301,11 +306,11 @@ def test_simple_suicide(self): gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) class ChainedBase: @@ -378,18 +383,27 @@ def check_non_resurrecting_chain(self, classes): def check_resurrecting_chain(self, classes): N = len(classes) + def dummy_callback(ref): + pass with SimpleBase.test(): nodes = self.build_chain(classes) N = len(nodes) ids = [id(s) for s in nodes] survivor_ids = [id(s) for s in nodes if isinstance(s, SimpleResurrector)] wrs = [weakref.ref(s) for s in nodes] + wrcs = [weakref.ref(s, dummy_callback) for s in nodes] del nodes gc.collect() self.assert_del_calls(ids) self.assert_survivors(survivor_ids) - # XXX desirable? - self.assertEqual([wr() for wr in wrs], [None] * N) + for wr in wrs: + # These values used to be None because weakrefs were cleared + # before calling finalizers. Now they are cleared after. + self.assertIsNotNone(wr()) + for wr in wrcs: + # Weakrefs with callbacks are still cleared before calling + # finalizers. + self.assertIsNone(wr()) self.clear_survivors() gc.collect() self.assert_del_calls(ids) @@ -491,7 +505,7 @@ def test_legacy(self): self.assert_del_calls(ids) self.assert_tp_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) gc.collect() self.assert_del_calls(ids) self.assert_tp_del_calls(ids) @@ -507,13 +521,13 @@ def test_legacy_resurrect(self): self.assert_tp_del_calls(ids) self.assert_survivors(ids) # weakrefs are cleared before tp_del is called. - self.assertIs(wr(), None) + self.assertIsNone(wr()) self.clear_survivors() gc.collect() self.assert_del_calls(ids) self.assert_tp_del_calls(ids * 2) self.assert_survivors(ids) - self.assertIs(wr(), None) + self.assertIsNone(wr()) def test_legacy_self_cycle(self): # Self-cycles with legacy finalizers end up in gc.garbage. @@ -527,11 +541,11 @@ def test_legacy_self_cycle(self): self.assert_tp_del_calls([]) self.assert_survivors([]) self.assert_garbage(ids) - self.assertIsNot(wr(), None) + self.assertIsNotNone(wr()) # Break the cycle to allow collection gc.garbage[0].ref = None self.assert_garbage([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) if __name__ == "__main__": diff --git a/Lib/test/test_free_threading/test_monitoring.py b/Lib/test/test_free_threading/test_monitoring.py index c3d0a2bcea5c17..407bf7cbdee917 100644 --- a/Lib/test/test_free_threading/test_monitoring.py +++ b/Lib/test/test_free_threading/test_monitoring.py @@ -194,6 +194,65 @@ def during_threads(self): self.set = not self.set +@threading_helper.requires_working_threading() +class SetProfileAllThreadsMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): + """Uses threading.setprofile_all_threads and repeatedly toggles instrumentation on and off""" + + def setUp(self): + self.set = False + self.called = False + + def after_test(self): + self.assertTrue(self.called) + + def tearDown(self): + threading.setprofile_all_threads(None) + + def trace_func(self, frame, event, arg): + self.called = True + return self.trace_func + + def during_threads(self): + if self.set: + threading.setprofile_all_threads(self.trace_func) + else: + threading.setprofile_all_threads(None) + self.set = not self.set + + +class SetProfileAllMultiThreaded(TestCase): + def test_profile_all_threads(self): + done = threading.Event() + + def func(): + pass + + def bg_thread(): + while not done.is_set(): + func() + func() + func() + func() + func() + + def my_profile(frame, event, arg): + return None + + bg_threads = [] + for i in range(10): + t = threading.Thread(target=bg_thread) + t.start() + bg_threads.append(t) + + for i in range(100): + threading.setprofile_all_threads(my_profile) + threading.setprofile_all_threads(None) + + done.set() + for t in bg_threads: + t.join() + + class TraceBuf: def __init__(self): self.traces = [] @@ -387,6 +446,38 @@ def noop(): self.observe_threads(noop, buf) + def test_trace_concurrent(self): + # Test calling a function concurrently from a tracing and a non-tracing + # thread + b = threading.Barrier(2) + + def func(): + for _ in range(100): + pass + + def noop(): + pass + + def bg_thread(): + b.wait() + func() # this may instrument `func` + + def tracefunc(frame, event, arg): + # These calls run under tracing can race with the background thread + for _ in range(10): + func() + return tracefunc + + t = Thread(target=bg_thread) + t.start() + try: + sys.settrace(tracefunc) + b.wait() + noop() + finally: + sys.settrace(None) + t.join() + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index ae996e7db3c7fd..2d995751005d71 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -127,6 +127,20 @@ class ClassB(Base): obj.__class__ = ClassB + def test_name_change(self): + class Foo: + pass + + def writer(): + for _ in range(1000): + Foo.__name__ = 'Bar' + + def reader(): + for _ in range(1000): + Foo.__name__ + + self.run_one(writer, reader) + def run_one(self, writer_func, reader_func): barrier = threading.Barrier(NTHREADS) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index b41e02c3a16379..41cefe0e286d50 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1831,6 +1831,34 @@ def test_newlines_in_format_specifiers(self): for case in valid_cases: compile(case, "", "exec") + def test_raw_fstring_format_spec(self): + # Test raw f-string format spec behavior (Issue #137314). + # + # Raw f-strings should preserve literal backslashes in format specifications, + # not interpret them as escape sequences. + class UnchangedFormat: + """Test helper that returns the format spec unchanged.""" + def __format__(self, format): + return format + + # Test basic escape sequences + self.assertEqual(f"{UnchangedFormat():\xFF}", 'ÿ') + self.assertEqual(rf"{UnchangedFormat():\xFF}", '\\xFF') + + # Test nested expressions with raw/non-raw combinations + self.assertEqual(rf"{UnchangedFormat():{'\xFF'}}", 'ÿ') + self.assertEqual(f"{UnchangedFormat():{r'\xFF'}}", '\\xFF') + self.assertEqual(rf"{UnchangedFormat():{r'\xFF'}}", '\\xFF') + + # Test continuation character in format specs + self.assertEqual(f"""{UnchangedFormat():{'a'\ + 'b'}}""", 'ab') + self.assertEqual(rf"""{UnchangedFormat():{'a'\ + 'b'}}""", 'ab') + + # Test multiple format specs in same raw f-string + self.assertEqual(rf"{UnchangedFormat():\xFF} {UnchangedFormat():\n}", '\\xFF \\n') + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 96ebb23f73de9d..7c9adf3049a131 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -309,6 +309,26 @@ def func(): self.assertRegex(stdout, rb"""\A\s*func=None""") self.assertFalse(stderr) + def test_datetime_weakref_cycle(self): + # https://github.com/python/cpython/issues/132413 + # If the weakref used by the datetime extension gets cleared by the GC (due to being + # in an unreachable cycle) then datetime functions would crash (get_module_state() + # was returning a NULL pointer). This bug is fixed by clearing weakrefs without + # callbacks *after* running finalizers. + code = """if 1: + import _datetime + class C: + def __del__(self): + print('__del__ called') + _datetime.timedelta(days=1) # crash? + + l = [C()] + l.append(l) + """ + rc, stdout, stderr = assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertEqual(stdout.strip(), b'__del__ called') + @refcount_test def test_frame(self): def f(): @@ -658,9 +678,8 @@ def callback(ignored): gc.collect() self.assertEqual(len(ouch), 2) # else the callbacks didn't run for x in ouch: - # If the callback resurrected one of these guys, the instance - # would be damaged, with an empty __dict__. - self.assertEqual(x, None) + # The weakref should be cleared before executing the callback. + self.assertIsNone(x) def test_bug21435(self): # This is a poor test - its only virtue is that it happened to @@ -1136,6 +1155,37 @@ def test_something(self): """) assert_python_ok("-c", source) + def test_do_not_cleanup_type_subclasses_before_finalization(self): + # See https://github.com/python/cpython/issues/135552 + # If we cleanup weakrefs for tp_subclasses before calling + # the finalizer (__del__) then the line `fail = BaseNode.next.next` + # should fail because we are trying to access a subclass + # attribute. But subclass type cache was not properly invalidated. + code = """ + class BaseNode: + def __del__(self): + BaseNode.next = BaseNode.next.next + fail = BaseNode.next.next + + class Node(BaseNode): + pass + + BaseNode.next = Node() + BaseNode.next.next = Node() + """ + # this test checks garbage collection while interp + # finalization + assert_python_ok("-c", textwrap.dedent(code)) + + code_inside_function = textwrap.dedent(F""" + def test(): + {textwrap.indent(code, ' ')} + + test() + """) + # this test checks regular garbage collection + assert_python_ok("-c", code_inside_function) + class IncrementalGCTests(unittest.TestCase): @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") @@ -1335,6 +1385,7 @@ def setUp(self): def tearDown(self): gc.disable() + @unittest.skipIf(Py_GIL_DISABLED, "requires GC generations or increments") def test_bug1055820c(self): # Corresponds to temp2c.py in the bug report. This is pretty # elaborate. @@ -1410,6 +1461,7 @@ def callback(ignored): self.assertEqual(x, None) @gc_threshold(1000, 0, 0) + @unittest.skipIf(Py_GIL_DISABLED, "requires GC generations or increments") def test_bug1055820d(self): # Corresponds to temp2d.py in the bug report. This is very much like # test_bug1055820c, but uses a __del__ method instead of a weakref diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index d0ed5129253cdf..9e4e82b2542c15 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -4,7 +4,6 @@ import shutil import sys import unittest -import warnings from test.support import is_wasi, Py_DEBUG from test.support.os_helper import (TESTFN, skip_unless_symlink, @@ -393,36 +392,6 @@ def test_glob_many_open_files(self): for it in iters: self.assertEqual(next(it), p) - def test_glob0(self): - with self.assertWarns(DeprecationWarning): - glob.glob0(self.tempdir, 'a') - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - eq = self.assertSequencesEqual_noorder - eq(glob.glob0(self.tempdir, 'a'), ['a']) - eq(glob.glob0(self.tempdir, '.bb'), ['.bb']) - eq(glob.glob0(self.tempdir, '.b*'), []) - eq(glob.glob0(self.tempdir, 'b'), []) - eq(glob.glob0(self.tempdir, '?'), []) - eq(glob.glob0(self.tempdir, '*a'), []) - eq(glob.glob0(self.tempdir, 'a*'), []) - - def test_glob1(self): - with self.assertWarns(DeprecationWarning): - glob.glob1(self.tempdir, 'a') - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - eq = self.assertSequencesEqual_noorder - eq(glob.glob1(self.tempdir, 'a'), ['a']) - eq(glob.glob1(self.tempdir, '.bb'), ['.bb']) - eq(glob.glob1(self.tempdir, '.b*'), ['.bb']) - eq(glob.glob1(self.tempdir, 'b'), []) - eq(glob.glob1(self.tempdir, '?'), ['a']) - eq(glob.glob1(self.tempdir, '*a'), ['a', 'aaa']) - eq(glob.glob1(self.tempdir, 'a*'), ['a', 'aaa', 'aab']) - def test_translate_matching(self): match = re.compile(glob.translate('*')).match self.assertIsNotNone(match('foo')) diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index b2b64a76a9f0f6..33845d8a9e2651 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -27,24 +27,17 @@ from http.client import HTTPException -default_builtin_hashes = {'md5', 'sha1', 'sha256', 'sha512', 'sha3', 'blake2'} +default_builtin_hashes = {'md5', 'sha1', 'sha2', 'sha3', 'blake2'} # --with-builtin-hashlib-hashes override builtin_hashes = sysconfig.get_config_var("PY_BUILTIN_HASHLIB_HASHES") if builtin_hashes is None: builtin_hashes = default_builtin_hashes else: - builtin_hashes = { - m.strip() for m in builtin_hashes.strip('"').lower().split(",") - } + builtin_hash_names = builtin_hashes.strip('"').lower().split(",") + builtin_hashes = set(map(str.strip, builtin_hash_names)) -# hashlib with and without OpenSSL backend for PBKDF2 -# only import builtin_hashlib when all builtin hashes are available. -# Otherwise import prints noise on stderr +# Public 'hashlib' module with OpenSSL backend for PBKDF2. openssl_hashlib = import_fresh_module('hashlib', fresh=['_hashlib']) -if builtin_hashes == default_builtin_hashes: - builtin_hashlib = import_fresh_module('hashlib', blocked=['_hashlib']) -else: - builtin_hashlib = None try: from _hashlib import HASH, HASHXOF, openssl_md_meth_names, get_fips_mode diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index 2fbc142de2fd34..c2ed30831b2e0e 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -48,6 +48,29 @@ def test_basic(self): 'Set-Cookie: d=r', 'Set-Cookie: f=h' )) + }, + + # gh-92936: allow double quote in cookie values + { + 'data': 'cookie="{"key": "value"}"', + 'dict': {'cookie': '{"key": "value"}'}, + 'repr': "", + 'output': 'Set-Cookie: cookie="{"key": "value"}"', + }, + { + 'data': 'key="some value; surrounded by quotes"', + 'dict': {'key': 'some value; surrounded by quotes'}, + 'repr': "", + 'output': 'Set-Cookie: key="some value; surrounded by quotes"', + }, + { + 'data': 'session="user123"; preferences="{"theme": "dark"}"', + 'dict': {'session': 'user123', 'preferences': '{"theme": "dark"}'}, + 'repr': "", + 'output': '\n'.join(( + 'Set-Cookie: preferences="{"theme": "dark"}"', + 'Set-Cookie: session="user123"', + )) } ] diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index 070920d0da7e19..dd943210ffca3c 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -224,15 +224,7 @@ class ResourceLoaderDefaultsTests(ABCTestHarness): SPLIT = make_abc_subclasses(ResourceLoader) def test_get_data(self): - with ( - self.assertRaises(IOError), - self.assertWarnsRegex( - DeprecationWarning, - r"importlib\.abc\.ResourceLoader is deprecated in favour of " - r"supporting resource loading through importlib\.resources" - r"\.abc\.TraversableResources.", - ), - ): + with self.assertRaises(IOError): self.ins.get_data('/some/path') @@ -936,13 +928,8 @@ def get_filename(self, fullname): def path_stats(self, path): return {'mtime': 1} - with self.assertWarnsRegex( - DeprecationWarning, - r"importlib\.abc\.ResourceLoader is deprecated in favour of " - r"supporting resource loading through importlib\.resources" - r"\.abc\.TraversableResources.", - ): - loader = DummySourceLoader() + + loader = DummySourceLoader() with self.assertWarnsRegex( DeprecationWarning, @@ -952,17 +939,5 @@ def path_stats(self, path): loader.path_mtime('foo.py') -class ResourceLoaderDeprecationWarningsTests(unittest.TestCase): - """Tests ResourceLoader deprecation warnings.""" - - def test_deprecated_resource_loader(self): - from importlib.abc import ResourceLoader - class DummyLoader(ResourceLoader): - def get_data(self, path): - return b'' - - with self.assertWarns(DeprecationWarning): - DummyLoader() - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 30e01b8cd87a75..c8c1a5226114b9 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -5831,6 +5831,21 @@ def test_collections_abc_module_has_signatures(self): import collections.abc self._test_module_has_signatures(collections.abc) + def test_datetime_module_has_signatures(self): + # Only test if the C implementation is available. + import_helper.import_module('_datetime') + import datetime + no_signature = {'tzinfo'} + unsupported_signature = {'timezone'} + methods_unsupported_signature = { + 'date': {'replace'}, + 'time': {'replace'}, + 'datetime': {'replace', 'combine'}, + } + self._test_module_has_signatures(datetime, + no_signature, unsupported_signature, + methods_unsupported_signature=methods_unsupported_signature) + def test_errno_module_has_signatures(self): import errno self._test_module_has_signatures(errno) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index b487bcabf01ca4..92be2763e5ed1e 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -808,7 +808,12 @@ def test_closefd_attr(self): def test_garbage_collection(self): # FileIO objects are collected, and collecting them flushes # all data to disk. - with warnings_helper.check_warnings(('', ResourceWarning)): + # + # Note that using warnings_helper.check_warnings() will keep the + # file alive due to the `source` argument to warn(). So, use + # catch_warnings() instead. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) f = self.FileIO(os_helper.TESTFN, "wb") f.write(b"abcxxx") f.f = f @@ -1809,7 +1814,11 @@ def test_garbage_collection(self): # C BufferedReader objects are collected. # The Python version has __del__, so it ends into gc.garbage instead self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with warnings_helper.check_warnings(('', ResourceWarning)): + # Note that using warnings_helper.check_warnings() will keep the + # file alive due to the `source` argument to warn(). So, use + # catch_warnings() instead. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) rawio = self.FileIO(os_helper.TESTFN, "w+b") f = self.tp(rawio) f.f = f @@ -2158,7 +2167,11 @@ def test_garbage_collection(self): # all data to disk. # The Python version has __del__, so it ends into gc.garbage instead self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with warnings_helper.check_warnings(('', ResourceWarning)): + # Note that using warnings_helper.check_warnings() will keep the + # file alive due to the `source` argument to warn(). So, use + # catch_warnings() instead. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) rawio = self.FileIO(os_helper.TESTFN, "w+b") f = self.tp(rawio) f.write(b"123xxx") @@ -4080,7 +4093,8 @@ def test_garbage_collection(self): # C TextIOWrapper objects are collected, and collecting them flushes # all data to disk. # The Python version has __del__, so it ends in gc.garbage instead. - with warnings_helper.check_warnings(('', ResourceWarning)): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) rawio = self.FileIO(os_helper.TESTFN, "wb") b = self.BufferedWriter(rawio) t = self.TextIOWrapper(b, encoding="ascii") diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index db1c38243e2268..62929f9dd68b5a 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -2269,6 +2269,10 @@ def testReservedIpv4(self): self.assertEqual(False, ipaddress.ip_network('240.0.0.0').is_multicast) self.assertEqual(True, ipaddress.ip_network('240.0.0.0').is_reserved) + self.assertTrue(ipaddress.ip_interface('0.0.0.0/32').is_unspecified) + self.assertFalse(ipaddress.ip_interface('0.0.0.0/31').is_unspecified) + self.assertFalse(ipaddress.ip_interface('1.2.3.4/32').is_unspecified) + self.assertEqual(True, ipaddress.ip_interface( '192.168.1.1/17').is_private) self.assertEqual(False, ipaddress.ip_network('192.169.0.0').is_private) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 55b502e52ca454..698e137e3e8abd 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -5,6 +5,7 @@ from unittest import mock import unittest import locale +import os import sys import codecs @@ -486,6 +487,54 @@ def test_japanese(self): self.check('jp_jp', 'ja_JP.eucJP') +class TestRealLocales(unittest.TestCase): + def setUp(self): + oldlocale = locale.setlocale(locale.LC_CTYPE) + self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) + + def test_getsetlocale_issue1813(self): + # Issue #1813: setting and getting the locale under a Turkish locale + try: + locale.setlocale(locale.LC_CTYPE, 'tr_TR') + except locale.Error: + # Unsupported locale on this system + self.skipTest('test needs Turkish locale') + loc = locale.getlocale(locale.LC_CTYPE) + if verbose: + print('testing with %a' % (loc,), end=' ', flush=True) + try: + locale.setlocale(locale.LC_CTYPE, loc) + except locale.Error as exc: + # bpo-37945: setlocale(LC_CTYPE) fails with getlocale(LC_CTYPE) + # and the tr_TR locale on Windows. getlocale() builds a locale + # which is not recognize by setlocale(). + self.skipTest(f"setlocale(LC_CTYPE, {loc!r}) failed: {exc!r}") + self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE)) + + @unittest.skipUnless(os.name == 'nt', 'requires Windows') + def test_setlocale_long_encoding(self): + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_CTYPE, 'English.%016d' % 1252) + locale.setlocale(locale.LC_CTYPE, 'English.%015d' % 1252) + loc = locale.setlocale(locale.LC_ALL) + self.assertIn('.1252', loc) + loc2 = loc.replace('.1252', '.%016d' % 1252, 1) + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_ALL, loc2) + loc2 = loc.replace('.1252', '.%015d' % 1252, 1) + locale.setlocale(locale.LC_ALL, loc2) + + # gh-137273: Debug assertion failure on Windows for long encoding. + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_CTYPE, 'en_US.' + 'x'*16) + locale.setlocale(locale.LC_CTYPE, 'en_US.UTF-8') + loc = locale.setlocale(locale.LC_ALL) + self.assertIn('.UTF-8', loc) + loc2 = loc.replace('.UTF-8', '.' + 'x'*16, 1) + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_ALL, loc2) + + class TestMiscellaneous(unittest.TestCase): def test_defaults_UTF8(self): # Issue #18378: on (at least) macOS setting LC_CTYPE to "UTF-8" is @@ -552,27 +601,6 @@ def test_setlocale_category(self): # crasher from bug #7419 self.assertRaises(locale.Error, locale.setlocale, 12345) - def test_getsetlocale_issue1813(self): - # Issue #1813: setting and getting the locale under a Turkish locale - oldlocale = locale.setlocale(locale.LC_CTYPE) - self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) - try: - locale.setlocale(locale.LC_CTYPE, 'tr_TR') - except locale.Error: - # Unsupported locale on this system - self.skipTest('test needs Turkish locale') - loc = locale.getlocale(locale.LC_CTYPE) - if verbose: - print('testing with %a' % (loc,), end=' ', flush=True) - try: - locale.setlocale(locale.LC_CTYPE, loc) - except locale.Error as exc: - # bpo-37945: setlocale(LC_CTYPE) fails with getlocale(LC_CTYPE) - # and the tr_TR locale on Windows. getlocale() builds a locale - # which is not recognize by setlocale(). - self.skipTest(f"setlocale(LC_CTYPE, {loc!r}) failed: {exc!r}") - self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE)) - def test_invalid_locale_format_in_localetuple(self): with self.assertRaises(TypeError): locale.setlocale(locale.LC_ALL, b'fi_FI') diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 0207843cc0e8f7..13424991639215 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -162,48 +162,55 @@ def baz(): @unittest.skipIf(support.check_bolt_optimized(), "fails on BOLT instrumented binaries") def test_sys_api(self): - code = """if 1: - import sys - def foo(): - pass - - def spam(): - pass + for define_eval_hook in (False, True): + code = """if 1: + import sys + def foo(): + pass - def bar(): - sys.deactivate_stack_trampoline() - foo() - sys.activate_stack_trampoline("perf") - spam() + def spam(): + pass - def baz(): - bar() + def bar(): + sys.deactivate_stack_trampoline() + foo() + sys.activate_stack_trampoline("perf") + spam() - sys.activate_stack_trampoline("perf") - baz() - """ - with temp_dir() as script_dir: - script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} - with subprocess.Popen( - [sys.executable, script], - text=True, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - env=env, - ) as process: - stdout, stderr = process.communicate() + def baz(): + bar() - self.assertEqual(stderr, "") - self.assertEqual(stdout, "") + sys.activate_stack_trampoline("perf") + baz() + """ + if define_eval_hook: + set_eval_hook = """if 1: + import _testinternalcapi + _testinternalcapi.set_eval_frame_record([]) +""" + code = set_eval_hook + code + with temp_dir() as script_dir: + script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} + with subprocess.Popen( + [sys.executable, script], + text=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + env=env, + ) as process: + stdout, stderr = process.communicate() - perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") - self.assertTrue(perf_file.exists()) - perf_file_contents = perf_file.read_text() - self.assertNotIn(f"py::foo:{script}", perf_file_contents) - self.assertIn(f"py::spam:{script}", perf_file_contents) - self.assertIn(f"py::bar:{script}", perf_file_contents) - self.assertIn(f"py::baz:{script}", perf_file_contents) + self.assertEqual(stderr, "") + self.assertEqual(stdout, "") + + perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") + self.assertTrue(perf_file.exists()) + perf_file_contents = perf_file.read_text() + self.assertNotIn(f"py::foo:{script}", perf_file_contents) + self.assertIn(f"py::spam:{script}", perf_file_contents) + self.assertIn(f"py::bar:{script}", perf_file_contents) + self.assertIn(f"py::baz:{script}", perf_file_contents) def test_sys_api_with_existing_trampoline(self): code = """if 1: diff --git a/Lib/test/test_resource.py b/Lib/test/test_resource.py index d23d3623235f38..fe05224828bd27 100644 --- a/Lib/test/test_resource.py +++ b/Lib/test/test_resource.py @@ -14,89 +14,154 @@ class ResourceTest(unittest.TestCase): def test_args(self): self.assertRaises(TypeError, resource.getrlimit) - self.assertRaises(TypeError, resource.getrlimit, 42, 42) + self.assertRaises(TypeError, resource.getrlimit, 0, 42) + self.assertRaises(OverflowError, resource.getrlimit, 2**1000) + self.assertRaises(OverflowError, resource.getrlimit, -2**1000) + self.assertRaises(TypeError, resource.getrlimit, '0') self.assertRaises(TypeError, resource.setrlimit) - self.assertRaises(TypeError, resource.setrlimit, 42, 42, 42) + self.assertRaises(TypeError, resource.setrlimit, 0) + self.assertRaises(TypeError, resource.setrlimit, 0, 42) + self.assertRaises(TypeError, resource.setrlimit, 0, 42, 42) + self.assertRaises(OverflowError, resource.setrlimit, 2**1000, (42, 42)) + self.assertRaises(OverflowError, resource.setrlimit, -2**1000, (42, 42)) + self.assertRaises(ValueError, resource.setrlimit, 0, (42,)) + self.assertRaises(ValueError, resource.setrlimit, 0, (42, 42, 42)) + self.assertRaises(TypeError, resource.setrlimit, '0', (42, 42)) + self.assertRaises(TypeError, resource.setrlimit, 0, ('42', 42)) + self.assertRaises(TypeError, resource.setrlimit, 0, (42, '42')) @unittest.skipIf(sys.platform == "vxworks", "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') def test_fsize_ismax(self): - try: - (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) - except AttributeError: - pass - else: - # RLIMIT_FSIZE should be RLIM_INFINITY, which will be a really big - # number on a platform with large file support. On these platforms, - # we need to test that the get/setrlimit functions properly convert - # the number to a C long long and that the conversion doesn't raise - # an error. - self.assertEqual(resource.RLIM_INFINITY, max) - resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + # RLIMIT_FSIZE should be RLIM_INFINITY, which will be a really big + # number on a platform with large file support. On these platforms, + # we need to test that the get/setrlimit functions properly convert + # the number to a C long long and that the conversion doesn't raise + # an error. + self.assertEqual(resource.RLIM_INFINITY, max) + resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) + @unittest.skipIf(sys.platform == "vxworks", + "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') def test_fsize_enforced(self): + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + # Check to see what happens when the RLIMIT_FSIZE is small. Some + # versions of Python were terminated by an uncaught SIGXFSZ, but + # pythonrun.c has been fixed to ignore that exception. If so, the + # write() should return EFBIG when the limit is exceeded. + + # At least one platform has an unlimited RLIMIT_FSIZE and attempts + # to change it raise ValueError instead. try: - (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) - except AttributeError: - pass - else: - # Check to see what happens when the RLIMIT_FSIZE is small. Some - # versions of Python were terminated by an uncaught SIGXFSZ, but - # pythonrun.c has been fixed to ignore that exception. If so, the - # write() should return EFBIG when the limit is exceeded. - - # At least one platform has an unlimited RLIMIT_FSIZE and attempts - # to change it raise ValueError instead. try: + resource.setrlimit(resource.RLIMIT_FSIZE, (1024, max)) + limit_set = True + except ValueError: + limit_set = False + f = open(os_helper.TESTFN, "wb") + try: + f.write(b"X" * 1024) try: - resource.setrlimit(resource.RLIMIT_FSIZE, (1024, max)) - limit_set = True - except ValueError: - limit_set = False - f = open(os_helper.TESTFN, "wb") - try: - f.write(b"X" * 1024) - try: - f.write(b"Y") + f.write(b"Y") + f.flush() + # On some systems (e.g., Ubuntu on hppa) the flush() + # doesn't always cause the exception, but the close() + # does eventually. Try flushing several times in + # an attempt to ensure the file is really synced and + # the exception raised. + for i in range(5): + time.sleep(.1) f.flush() - # On some systems (e.g., Ubuntu on hppa) the flush() - # doesn't always cause the exception, but the close() - # does eventually. Try flushing several times in - # an attempt to ensure the file is really synced and - # the exception raised. - for i in range(5): - time.sleep(.1) - f.flush() - except OSError: - if not limit_set: - raise - if limit_set: - # Close will attempt to flush the byte we wrote - # Restore limit first to avoid getting a spurious error - resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) - finally: - f.close() - finally: + except OSError: + if not limit_set: + raise if limit_set: + # Close will attempt to flush the byte we wrote + # Restore limit first to avoid getting a spurious error resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) - os_helper.unlink(os_helper.TESTFN) + finally: + f.close() + finally: + if limit_set: + resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) + os_helper.unlink(os_helper.TESTFN) - def test_fsize_toobig(self): + @unittest.skipIf(sys.platform == "vxworks", + "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') + def test_fsize_too_big(self): # Be sure that setrlimit is checking for really large values too_big = 10**50 + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + try: + resource.setrlimit(resource.RLIMIT_FSIZE, (too_big, max)) + except (OverflowError, ValueError): + pass try: - (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) - except AttributeError: + resource.setrlimit(resource.RLIMIT_FSIZE, (max, too_big)) + except (OverflowError, ValueError): pass + + @unittest.skipIf(sys.platform == "vxworks", + "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') + def test_fsize_not_too_big(self): + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + self.addCleanup(resource.setrlimit, resource.RLIMIT_FSIZE, (cur, max)) + + def expected(cur): + if resource.RLIM_INFINITY < 0: + return [(cur, max), (resource.RLIM_INFINITY, max)] + elif resource.RLIM_INFINITY < cur: + return [(resource.RLIM_INFINITY, max)] + else: + return [(cur, max)] + + resource.setrlimit(resource.RLIMIT_FSIZE, (2**31-5, max)) + self.assertEqual(resource.getrlimit(resource.RLIMIT_FSIZE), (2**31-5, max)) + + try: + resource.setrlimit(resource.RLIMIT_FSIZE, (2**32, max)) + except OverflowError: + resource.setrlimit(resource.RLIMIT_FSIZE, (2**31, max)) + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**31)) + resource.setrlimit(resource.RLIMIT_FSIZE, (2**32-5, max)) + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**32-5)) else: + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**32)) + resource.setrlimit(resource.RLIMIT_FSIZE, (2**31, max)) + self.assertEqual(resource.getrlimit(resource.RLIMIT_FSIZE), (2**31, max)) + resource.setrlimit(resource.RLIMIT_FSIZE, (2**32-5, max)) + self.assertEqual(resource.getrlimit(resource.RLIMIT_FSIZE), (2**32-5, max)) + + resource.setrlimit(resource.RLIMIT_FSIZE, (2**63-5, max)) + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**63-5)) try: - resource.setrlimit(resource.RLIMIT_FSIZE, (too_big, max)) - except (OverflowError, ValueError): - pass - try: - resource.setrlimit(resource.RLIMIT_FSIZE, (max, too_big)) - except (OverflowError, ValueError): + resource.setrlimit(resource.RLIMIT_FSIZE, (2**63, max)) + except ValueError: + # There is a hard limit on macOS. pass + else: + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**63)) + resource.setrlimit(resource.RLIMIT_FSIZE, (2**64-5, max)) + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**64-5)) + + @unittest.skipIf(sys.platform == "vxworks", + "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') + def test_fsize_negative(self): + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + for value in -5, -2**31, -2**32-5, -2**63, -2**64-5, -2**1000: + with self.subTest(value=value): + # This test assumes that the values don't map to RLIM_INFINITY, + # though Posix doesn't guarantee it. + self.assertNotEqual(value, resource.RLIM_INFINITY) + + self.assertRaises(ValueError, resource.setrlimit, resource.RLIMIT_FSIZE, (value, max)) + self.assertRaises(ValueError, resource.setrlimit, resource.RLIMIT_FSIZE, (cur, value)) @unittest.skipUnless(hasattr(resource, "getrusage"), "needs getrusage") def test_getrusage(self): @@ -117,21 +182,18 @@ def test_getrusage(self): # Issue 6083: Reference counting bug @unittest.skipIf(sys.platform == "vxworks", "setting RLIMIT_CPU is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_CPU'), 'requires resource.RLIMIT_CPU') def test_setrusage_refcount(self): - try: - limits = resource.getrlimit(resource.RLIMIT_CPU) - except AttributeError: - pass - else: - class BadSequence: - def __len__(self): - return 2 - def __getitem__(self, key): - if key in (0, 1): - return len(tuple(range(1000000))) - raise IndexError + limits = resource.getrlimit(resource.RLIMIT_CPU) + class BadSequence: + def __len__(self): + return 2 + def __getitem__(self, key): + if key in (0, 1): + return len(tuple(range(1000000))) + raise IndexError - resource.setrlimit(resource.RLIMIT_CPU, BadSequence()) + resource.setrlimit(resource.RLIMIT_CPU, BadSequence()) def test_pagesize(self): pagesize = resource.getpagesize() @@ -168,7 +230,8 @@ class BadSeq: def __len__(self): return 2 def __getitem__(self, key): - return limits[key] - 1 # new reference + lim = limits[key] + return lim - 1 if lim > 0 else lim + sys.maxsize*2 # new reference limits = resource.getrlimit(resource.RLIMIT_AS) self.assertEqual(resource.prlimit(0, resource.RLIMIT_AS, BadSeq()), diff --git a/Lib/test/test_sample_profiler.py b/Lib/test/test_sample_profiler.py index 2c7fa1cba712c9..f234af60c8abcf 100644 --- a/Lib/test/test_sample_profiler.py +++ b/Lib/test/test_sample_profiler.py @@ -4,6 +4,7 @@ import io import marshal import os +import shutil import socket import subprocess import sys @@ -1588,6 +1589,68 @@ def test_sampling_all_threads(self): # Just verify that sampling completed without error # We're not testing output format here + def test_sample_target_script(self): + script_file = tempfile.NamedTemporaryFile(delete=False) + script_file.write(self.test_script.encode("utf-8")) + script_file.flush() + self.addCleanup(close_and_unlink, script_file) + + test_args = ["profile.sample", "-d", "1", script_file.name] + + with ( + mock.patch("sys.argv", test_args), + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profile.sample.main() + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Basic checks on output + self.assertIn("Captured", output) + self.assertIn("samples", output) + self.assertIn("Profile Stats", output) + + # Should see some of our test functions + self.assertIn("slow_fibonacci", output) + + + def test_sample_target_module(self): + tempdir = tempfile.TemporaryDirectory(delete=False) + self.addCleanup(lambda x: shutil.rmtree(x), tempdir.name) + + module_path = os.path.join(tempdir.name, "test_module.py") + + with open(module_path, "w") as f: + f.write(self.test_script) + + test_args = ["profile.sample", "-d", "1", "-m", "test_module"] + + with ( + mock.patch("sys.argv", test_args), + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + # Change to temp directory so subprocess can find the module + contextlib.chdir(tempdir.name), + ): + try: + profile.sample.main() + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Basic checks on output + self.assertIn("Captured", output) + self.assertIn("samples", output) + self.assertIn("Profile Stats", output) + + # Should see some of our test functions + self.assertIn("slow_fibonacci", output) + @skip_if_not_supported @unittest.skipIf( @@ -1660,12 +1723,11 @@ def test_is_process_running(self): self.assertIsNotNone(profiler.unwinder.get_stack_trace()) proc.kill() proc.wait() - # ValueError on MacOS (yeah I know), ProcessLookupError on Linux and Windows - self.assertRaises((ValueError, ProcessLookupError), profiler.unwinder.get_stack_trace) + self.assertRaises(ProcessLookupError, profiler.unwinder.get_stack_trace) # Exit the context manager to ensure the process is terminated self.assertFalse(profiler._is_process_running()) - self.assertRaises((ValueError, ProcessLookupError), profiler.unwinder.get_stack_trace) + self.assertRaises(ProcessLookupError, profiler.unwinder.get_stack_trace) @unittest.skipUnless(sys.platform == "linux", "Only valid on Linux") def test_esrch_signal_handling(self): @@ -1690,16 +1752,315 @@ def test_esrch_signal_handling(self): class TestSampleProfilerCLI(unittest.TestCase): + def _setup_sync_mocks(self, mock_socket, mock_popen): + """Helper to set up socket and process mocks for coordinator tests.""" + # Mock the sync socket with context manager support + mock_sock_instance = mock.MagicMock() + mock_sock_instance.getsockname.return_value = ("127.0.0.1", 12345) + + # Mock the connection with context manager support + mock_conn = mock.MagicMock() + mock_conn.recv.return_value = b"ready" + mock_conn.__enter__.return_value = mock_conn + mock_conn.__exit__.return_value = None + + # Mock accept() to return (connection, address) and support indexing + mock_accept_result = mock.MagicMock() + mock_accept_result.__getitem__.return_value = mock_conn # [0] returns the connection + mock_sock_instance.accept.return_value = mock_accept_result + + # Mock socket with context manager support + mock_sock_instance.__enter__.return_value = mock_sock_instance + mock_sock_instance.__exit__.return_value = None + mock_socket.return_value = mock_sock_instance + + # Mock the subprocess + mock_process = mock.MagicMock() + mock_process.pid = 12345 + mock_process.poll.return_value = None + mock_popen.return_value = mock_process + return mock_process + + def _verify_coordinator_command(self, mock_popen, expected_target_args): + """Helper to verify the coordinator command was called correctly.""" + args, kwargs = mock_popen.call_args + coordinator_cmd = args[0] + self.assertEqual(coordinator_cmd[0], sys.executable) + self.assertEqual(coordinator_cmd[1], "-m") + self.assertEqual(coordinator_cmd[2], "profile._sync_coordinator") + self.assertEqual(coordinator_cmd[3], "12345") # port + # cwd is coordinator_cmd[4] + self.assertEqual(coordinator_cmd[5:], expected_target_args) + + def test_cli_module_argument_parsing(self): + test_args = ["profile.sample", "-m", "mymodule"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profile.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profile.sample.main() + + self._verify_coordinator_command(mock_popen, ("-m", "mymodule")) + mock_sample.assert_called_once_with( + 12345, + sort=2, # default sort (sort_value from args.sort) + sample_interval_usec=100, + duration_sec=10, + filename=None, + all_threads=False, + limit=15, + show_summary=True, + output_format="pstats", + realtime_stats=False, + ) + + def test_cli_module_with_arguments(self): + test_args = ["profile.sample", "-m", "mymodule", "arg1", "arg2", "--flag"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profile.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profile.sample.main() + + self._verify_coordinator_command(mock_popen, ("-m", "mymodule", "arg1", "arg2", "--flag")) + mock_sample.assert_called_once_with( + 12345, + sort=2, + sample_interval_usec=100, + duration_sec=10, + filename=None, + all_threads=False, + limit=15, + show_summary=True, + output_format="pstats", + realtime_stats=False, + ) + + def test_cli_script_argument_parsing(self): + test_args = ["profile.sample", "myscript.py"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profile.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profile.sample.main() + + self._verify_coordinator_command(mock_popen, ("myscript.py",)) + mock_sample.assert_called_once_with( + 12345, + sort=2, + sample_interval_usec=100, + duration_sec=10, + filename=None, + all_threads=False, + limit=15, + show_summary=True, + output_format="pstats", + realtime_stats=False, + ) + + def test_cli_script_with_arguments(self): + test_args = ["profile.sample", "myscript.py", "arg1", "arg2", "--flag"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profile.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + # Use the helper to set up mocks consistently + mock_process = self._setup_sync_mocks(mock_socket, mock_popen) + # Override specific behavior for this test + mock_process.wait.side_effect = [subprocess.TimeoutExpired(test_args, 0.1), None] + + profile.sample.main() + + # Verify the coordinator command was called + args, kwargs = mock_popen.call_args + coordinator_cmd = args[0] + self.assertEqual(coordinator_cmd[0], sys.executable) + self.assertEqual(coordinator_cmd[1], "-m") + self.assertEqual(coordinator_cmd[2], "profile._sync_coordinator") + self.assertEqual(coordinator_cmd[3], "12345") # port + # cwd is coordinator_cmd[4] + self.assertEqual(coordinator_cmd[5:], ("myscript.py", "arg1", "arg2", "--flag")) + + def test_cli_mutually_exclusive_pid_module(self): + test_args = ["profile.sample", "-p", "12345", "-m", "mymodule"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profile.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("not allowed with argument", error_msg) + + def test_cli_mutually_exclusive_pid_script(self): + test_args = ["profile.sample", "-p", "12345", "myscript.py"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profile.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("only one target type can be specified", error_msg) + + def test_cli_no_target_specified(self): + test_args = ["profile.sample", "-d", "5"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profile.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("one of the arguments", error_msg) + + def test_cli_module_with_profiler_options(self): + test_args = [ + "profile.sample", "-i", "1000", "-d", "30", "-a", + "--sort-tottime", "-l", "20", "-m", "mymodule", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profile.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profile.sample.main() + + self._verify_coordinator_command(mock_popen, ("-m", "mymodule")) + mock_sample.assert_called_once_with( + 12345, + sort=1, # sort-tottime + sample_interval_usec=1000, + duration_sec=30, + filename=None, + all_threads=True, + limit=20, + show_summary=True, + output_format="pstats", + realtime_stats=False, + ) + + def test_cli_script_with_profiler_options(self): + """Test script with various profiler options.""" + test_args = [ + "profile.sample", "-i", "2000", "-d", "60", + "--collapsed", "-o", "output.txt", + "myscript.py", "scriptarg", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profile.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profile.sample.main() + + self._verify_coordinator_command(mock_popen, ("myscript.py", "scriptarg")) + # Verify profiler options were passed correctly + mock_sample.assert_called_once_with( + 12345, + sort=2, # default sort + sample_interval_usec=2000, + duration_sec=60, + filename="output.txt", + all_threads=False, + limit=15, + show_summary=True, + output_format="collapsed", + realtime_stats=False, + ) + + def test_cli_empty_module_name(self): + test_args = ["profile.sample", "-m"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profile.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("argument -m/--module: expected one argument", error_msg) + + def test_cli_long_module_option(self): + test_args = ["profile.sample", "--module", "mymodule", "arg1"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profile.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profile.sample.main() + + self._verify_coordinator_command(mock_popen, ("-m", "mymodule", "arg1")) + + def test_cli_complex_script_arguments(self): + test_args = [ + "profile.sample", "script.py", + "--input", "file.txt", "-v", "--output=/tmp/out", "positional" + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profile.sample.sample") as mock_sample, + mock.patch("profile.sample._run_with_sync") as mock_run_with_sync, + ): + mock_process = mock.MagicMock() + mock_process.pid = 12345 + mock_process.wait.side_effect = [subprocess.TimeoutExpired(test_args, 0.1), None] + mock_process.poll.return_value = None + mock_run_with_sync.return_value = mock_process + + profile.sample.main() + + mock_run_with_sync.assert_called_once_with(( + sys.executable, "script.py", + "--input", "file.txt", "-v", "--output=/tmp/out", "positional", + )) + def test_cli_collapsed_format_validation(self): """Test that CLI properly validates incompatible options with collapsed format.""" test_cases = [ # Test sort options are invalid with collapsed ( - ["profile.sample", "--collapsed", "--sort-nsamples", "12345"], + ["profile.sample", "--collapsed", "--sort-nsamples", "-p", "12345"], "sort", ), ( - ["profile.sample", "--collapsed", "--sort-tottime", "12345"], + ["profile.sample", "--collapsed", "--sort-tottime", "-p", "12345"], "sort", ), ( @@ -1707,6 +2068,7 @@ def test_cli_collapsed_format_validation(self): "profile.sample", "--collapsed", "--sort-cumtime", + "-p", "12345", ], "sort", @@ -1716,6 +2078,7 @@ def test_cli_collapsed_format_validation(self): "profile.sample", "--collapsed", "--sort-sample-pct", + "-p", "12345", ], "sort", @@ -1725,23 +2088,24 @@ def test_cli_collapsed_format_validation(self): "profile.sample", "--collapsed", "--sort-cumul-pct", + "-p", "12345", ], "sort", ), ( - ["profile.sample", "--collapsed", "--sort-name", "12345"], + ["profile.sample", "--collapsed", "--sort-name", "-p", "12345"], "sort", ), # Test limit option is invalid with collapsed - (["profile.sample", "--collapsed", "-l", "20", "12345"], "limit"), + (["profile.sample", "--collapsed", "-l", "20", "-p", "12345"], "limit"), ( - ["profile.sample", "--collapsed", "--limit", "20", "12345"], + ["profile.sample", "--collapsed", "--limit", "20", "-p", "12345"], "limit", ), # Test no-summary option is invalid with collapsed ( - ["profile.sample", "--collapsed", "--no-summary", "12345"], + ["profile.sample", "--collapsed", "--no-summary", "-p", "12345"], "summary", ), ] @@ -1761,7 +2125,7 @@ def test_cli_collapsed_format_validation(self): def test_cli_default_collapsed_filename(self): """Test that collapsed format gets a default filename when not specified.""" - test_args = ["profile.sample", "--collapsed", "12345"] + test_args = ["profile.sample", "--collapsed", "-p", "12345"] with ( mock.patch("sys.argv", test_args), @@ -1779,12 +2143,12 @@ def test_cli_custom_output_filenames(self): """Test custom output filenames for both formats.""" test_cases = [ ( - ["profile.sample", "--pstats", "-o", "custom.pstats", "12345"], + ["profile.sample", "--pstats", "-o", "custom.pstats", "-p", "12345"], "custom.pstats", "pstats", ), ( - ["profile.sample", "--collapsed", "-o", "custom.txt", "12345"], + ["profile.sample", "--collapsed", "-o", "custom.txt", "-p", "12345"], "custom.txt", "collapsed", ), @@ -1816,7 +2180,7 @@ def test_cli_mutually_exclusive_format_options(self): with ( mock.patch( "sys.argv", - ["profile.sample", "--pstats", "--collapsed", "12345"], + ["profile.sample", "--pstats", "--collapsed", "-p", "12345"], ), mock.patch("sys.stderr", io.StringIO()), ): @@ -1824,7 +2188,7 @@ def test_cli_mutually_exclusive_format_options(self): profile.sample.main() def test_argument_parsing_basic(self): - test_args = ["profile.sample", "12345"] + test_args = ["profile.sample", "-p", "12345"] with ( mock.patch("sys.argv", test_args), @@ -1856,7 +2220,7 @@ def test_sort_options(self): ] for option, expected_sort_value in sort_options: - test_args = ["profile.sample", option, "12345"] + test_args = ["profile.sample", option, "-p", "12345"] with ( mock.patch("sys.argv", test_args), diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 6d62d6119255a8..d6cc22558ec4fa 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -14,7 +14,7 @@ import unittest from test import support from test.support import ( - is_apple, is_apple_mobile, os_helper, threading_helper + force_not_colorized, is_apple, is_apple_mobile, os_helper, threading_helper ) from test.support.script_helper import assert_python_ok, spawn_python try: @@ -353,6 +353,7 @@ def check_signum(signals): @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + @force_not_colorized def test_wakeup_write_error(self): # Issue #16105: write() errors in the C signal handler should not # pass silently. diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 3dd67b2a2aba97..76fd33c7dc8767 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1538,6 +1538,40 @@ def testSetSockOpt(self): reuse = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) self.assertFalse(reuse == 0, "failed to set reuse mode") + def test_setsockopt_errors(self): + # See issue #107546. + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(sock.close) + + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # No error expected. + + with self.assertRaises(OverflowError): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 2 ** 100) + + with self.assertRaises(OverflowError): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, - 2 ** 100) + + with self.assertRaises(OverflowError): + sock.setsockopt(socket.SOL_SOCKET, 2 ** 100, 1) + + with self.assertRaises(OverflowError): + sock.setsockopt(2 ** 100, socket.SO_REUSEADDR, 1) + + with self.assertRaisesRegex(TypeError, "socket option should be int, bytes-like object or None"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, dict()) + + with self.assertRaisesRegex(TypeError, "requires 4 arguments when the third argument is None"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, None) + + with self.assertRaisesRegex(TypeError, "only takes 4 arguments when the third argument is None"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1, 2) + + with self.assertRaisesRegex(TypeError, "takes at least 3 arguments"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) + + with self.assertRaisesRegex(TypeError, "takes at most 4 arguments"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1, 2, 3) + def testSendAfterClose(self): # testing send() after close() with timeout with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 3602726437d8cf..74a511ba7c88c2 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -630,6 +630,14 @@ def test_deserialize_corrupt_database(self): class OpenTests(unittest.TestCase): _sql = "create table test(id integer)" + def test_open_with_bytes_path(self): + path = os.fsencode(TESTFN) + self.addCleanup(unlink, path) + self.assertFalse(os.path.exists(path)) + with contextlib.closing(sqlite.connect(path)) as cx: + self.assertTrue(os.path.exists(path)) + cx.execute(self._sql) + def test_open_with_path_like_object(self): """ Checks that we can successfully connect to a database using an object that is PathLike, i.e. has __fspath__(). """ diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index c8939383c75d6d..7d1eb564930367 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -49,6 +49,7 @@ HOST = socket_helper.HOST IS_OPENSSL_3_0_0 = ssl.OPENSSL_VERSION_INFO >= (3, 0, 0) CAN_GET_SELECTED_OPENSSL_GROUP = ssl.OPENSSL_VERSION_INFO >= (3, 2) +CAN_IGNORE_UNKNOWN_OPENSSL_GROUPS = ssl.OPENSSL_VERSION_INFO >= (3, 3) CAN_GET_AVAILABLE_OPENSSL_GROUPS = ssl.OPENSSL_VERSION_INFO >= (3, 5) PY_SSL_DEFAULT_CIPHERS = sysconfig.get_config_var('PY_SSL_DEFAULT_CIPHERS') @@ -964,8 +965,14 @@ def test_get_ciphers(self): def test_set_groups(self): ctx = ssl.create_default_context() - self.assertIsNone(ctx.set_groups('P-256:X25519')) - self.assertRaises(ssl.SSLError, ctx.set_groups, 'P-256:xxx') + # We use P-256 and P-384 (FIPS 186-4) that are alloed by OpenSSL + # even if FIPS module is enabled. Ignoring unknown groups is only + # supported since OpenSSL 3.3. + self.assertIsNone(ctx.set_groups('P-256:P-384')) + + self.assertRaises(ssl.SSLError, ctx.set_groups, 'P-256:foo') + if CAN_IGNORE_UNKNOWN_OPENSSL_GROUPS: + self.assertIsNone(ctx.set_groups('P-256:?foo')) @unittest.skipUnless(CAN_GET_AVAILABLE_OPENSSL_GROUPS, "OpenSSL version doesn't support getting groups") @@ -4620,6 +4627,42 @@ def server_callback(identity): with client_context.wrap_socket(socket.socket()) as s: s.connect((HOST, server.port)) + def test_thread_recv_while_main_thread_sends(self): + # GH-137583: Locking was added to calls to send() and recv() on SSL + # socket objects. This seemed fine at the surface level because those + # calls weren't re-entrant, but recv() calls would implicitly mimick + # holding a lock by blocking until it received data. This means that + # if a thread started to infinitely block until data was received, calls + # to send() would deadlock, because it would wait forever on the lock + # that the recv() call held. + data = b"1" * 1024 + event = threading.Event() + def background(sock): + event.set() + received = sock.recv(len(data)) + self.assertEqual(received, data) + + client_context, server_context, hostname = testing_context() + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as sock: + sock.connect((HOST, server.port)) + sock.settimeout(1) + sock.setblocking(1) + # Ensure that the server is ready to accept requests + sock.sendall(b"123") + self.assertEqual(sock.recv(3), b"123") + with threading_helper.catch_threading_exception() as cm: + thread = threading.Thread(target=background, + args=(sock,), daemon=True) + thread.start() + event.wait() + sock.sendall(data) + thread.join() + if cm.exc_value is not None: + raise cm.exc_value + @unittest.skipUnless(has_tls_version('TLSv1_3') and ssl.HAS_PHA, "Test needs TLS 1.3 PHA") diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 486bf10a0b5647..f89237931b7185 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1340,6 +1340,7 @@ def test_disable_gil_abi(self): @test.support.cpython_only +@force_not_colorized class UnraisableHookTest(unittest.TestCase): def test_original_unraisablehook(self): _testcapi = import_helper.import_module('_testcapi') diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 28914df6b010d0..860413b88eb6b5 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -841,6 +841,57 @@ def test_next_on_empty_tarfile(self): with tarfile.open(fileobj=fd, mode="r") as tf: self.assertEqual(tf.next(), None) + def _setup_symlink_to_target(self, temp_dirpath): + target_filepath = os.path.join(temp_dirpath, "target") + ustar_dirpath = os.path.join(temp_dirpath, "ustar") + hardlink_filepath = os.path.join(ustar_dirpath, "lnktype") + with open(target_filepath, "wb") as f: + f.write(b"target") + os.makedirs(ustar_dirpath) + os.symlink(target_filepath, hardlink_filepath) + return target_filepath, hardlink_filepath + + def _assert_on_file_content(self, filepath, digest): + with open(filepath, "rb") as f: + data = f.read() + self.assertEqual(sha256sum(data), digest) + + @unittest.skipUnless( + hasattr(os, "link"), "Missing hardlink implementation" + ) + @os_helper.skip_unless_symlink + def test_extract_hardlink_on_symlink(self): + """ + This test verifies that extracting a hardlink will not follow an + existing symlink after a FileExistsError on os.link. + """ + with os_helper.temp_dir() as DIR: + target_filepath, hardlink_filepath = self._setup_symlink_to_target(DIR) + with tarfile.open(tarname, encoding="iso8859-1") as tar: + tar.extract("ustar/regtype", DIR, filter="data") + tar.extract("ustar/lnktype", DIR, filter="data") + self._assert_on_file_content(target_filepath, sha256sum(b"target")) + self._assert_on_file_content(hardlink_filepath, sha256_regtype) + + @unittest.skipUnless( + hasattr(os, "link"), "Missing hardlink implementation" + ) + @os_helper.skip_unless_symlink + def test_extractall_hardlink_on_symlink(self): + """ + This test verifies that extracting a hardlink will not follow an + existing symlink after a FileExistsError on os.link. + """ + with os_helper.temp_dir() as DIR: + target_filepath, hardlink_filepath = self._setup_symlink_to_target(DIR) + with tarfile.open(tarname, encoding="iso8859-1") as tar: + tar.extractall( + DIR, members=["ustar/regtype", "ustar/lnktype"], filter="data", + ) + self._assert_on_file_content(target_filepath, sha256sum(b"target")) + self._assert_on_file_content(hardlink_filepath, sha256_regtype) + + class MiscReadTest(MiscReadTestBase, unittest.TestCase): test_fail_comp = None @@ -1737,6 +1788,16 @@ def test_file_mode(self): finally: os.umask(original_umask) + def test_pathlike_name(self): + expected_name = os.path.abspath(tmpname) + tarpath = os_helper.FakePath(tmpname) + + for func in (tarfile.open, tarfile.TarFile.open): + with self.subTest(): + with func(tarpath, self.mode) as tar: + self.assertEqual(tar.name, expected_name) + os_helper.unlink(tmpname) + class GzipStreamWriteTest(GzipTest, StreamWriteTest): def test_source_directory_not_leaked(self): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 002a1feeb85c94..0ba78b9a1807d2 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -2494,6 +2494,7 @@ def test_atexit_called_once(self): self.assertFalse(err) + @force_not_colorized def test_atexit_after_shutdown(self): # The only way to do this is by registering an atexit within # an atexit, which is intended to raise an exception. diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index b1615bbff383c2..6317d4657619f0 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3958,6 +3958,7 @@ class C: pass def test_defining_generic_protocols(self): T = TypeVar('T') + T2 = TypeVar('T2') S = TypeVar('S') @runtime_checkable @@ -3967,17 +3968,26 @@ def meth(self): pass class P(PR[int, T], Protocol[T]): y = 1 + self.assertEqual(P.__parameters__, (T,)) + with self.assertRaises(TypeError): PR[int] with self.assertRaises(TypeError): P[int, str] + with self.assertRaisesRegex( + TypeError, + re.escape('Some type variables (~S) are not listed in Protocol[~T, ~T2]'), + ): + class ExtraTypeVars(P[S], Protocol[T, T2]): ... class C(PR[int, T]): pass + self.assertEqual(C.__parameters__, (T,)) self.assertIsInstance(C[str](), C) def test_defining_generic_protocols_old_style(self): T = TypeVar('T') + T2 = TypeVar('T2') S = TypeVar('S') @runtime_checkable @@ -3996,9 +4006,19 @@ class P(PR[int, str], Protocol): class P1(Protocol, Generic[T]): def bar(self, x: T) -> str: ... + self.assertEqual(P1.__parameters__, (T,)) + class P2(Generic[T], Protocol): def bar(self, x: T) -> str: ... + self.assertEqual(P2.__parameters__, (T,)) + + msg = re.escape('Some type variables (~S) are not listed in Protocol[~T, ~T2]') + with self.assertRaisesRegex(TypeError, msg): + class ExtraTypeVars(P1[S], Protocol[T, T2]): ... + with self.assertRaisesRegex(TypeError, msg): + class ExtraTypeVars(P2[S], Protocol[T, T2]): ... + @runtime_checkable class PSub(P1[str], Protocol): x = 1 @@ -4011,6 +4031,28 @@ def bar(self, x: str) -> str: self.assertIsInstance(Test(), PSub) + def test_protocol_parameter_order(self): + # https://github.com/python/cpython/issues/137191 + T1 = TypeVar("T1") + T2 = TypeVar("T2", default=object) + + class A(Protocol[T1]): ... + + class B0(A[T2], Generic[T1, T2]): ... + self.assertEqual(B0.__parameters__, (T1, T2)) + + class B1(A[T2], Protocol, Generic[T1, T2]): ... + self.assertEqual(B1.__parameters__, (T1, T2)) + + class B2(A[T2], Protocol[T1, T2]): ... + self.assertEqual(B2.__parameters__, (T1, T2)) + + class B3[T1, T2](A[T2], Protocol): + @staticmethod + def get_typeparams(): + return (T1, T2) + self.assertEqual(B3.__parameters__, B3.get_typeparams()) + def test_pep695_generic_protocol_callable_members(self): @runtime_checkable class Foo[T](Protocol): diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 4c7c900eb56ae1..47f6b46061ac30 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -1044,6 +1044,32 @@ def callback(obj): stderr = res.err.decode("ascii", "backslashreplace") self.assertNotRegex(stderr, "_Py_Dealloc: Deallocator of type 'TestObj'") + def test_clearing_weakrefs_in_gc(self): + # This test checks that when finalizers are called: + # 1. weakrefs with callbacks have been cleared + # 2. weakrefs without callbacks have not been cleared + errors = [] + def test(): + class Class: + def __init__(self): + self._self = self + self.wr1 = weakref.ref(Class, lambda x: None) + self.wr2 = weakref.ref(Class) + + def __del__(self): + # we can't use assert* here, because gc will swallow + # exceptions + if self.wr1() is not None: + errors.append("weakref with callback as cleared") + if self.wr2() is not Class: + errors.append("weakref without callback was cleared") + + Class() + + test() + gc.collect() + self.assertEqual(errors, []) + class SubclassableWeakrefTestCase(TestBase): diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index a693b04870b995..9526d8b949fa3b 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -7,25 +7,25 @@ LabelFrame and PanedWindow. Properties of the widgets are specified with keyword arguments. -Keyword arguments have the same name as the corresponding resource +Keyword arguments have the same name as the corresponding options under Tk. Widgets are positioned with one of the geometry managers Place, Pack or Grid. These managers can be called with methods place, pack, grid available in every Widget. -Actions are bound to events by resources (e.g. keyword argument -command) or with the method bind. +Actions are bound to events by options (e.g. the command +keyword argument) or with the bind() method. Example (Hello, World): import tkinter from tkinter.constants import * tk = tkinter.Tk() frame = tkinter.Frame(tk, relief=RIDGE, borderwidth=2) -frame.pack(fill=BOTH,expand=1) +frame.pack(fill=BOTH, expand=1) label = tkinter.Label(frame, text="Hello, World") label.pack(fill=X, expand=1) -button = tkinter.Button(frame,text="Exit",command=tk.destroy) +button = tkinter.Button(frame, text="Exit", command=tk.destroy) button.pack(side=BOTTOM) tk.mainloop() """ @@ -847,7 +847,7 @@ def tk_focusNext(self): The focus order first goes to the next child, then to the children of the child recursively and then to the next sibling which is higher in the stacking order. A - widget is omitted if it has the takefocus resource set + widget is omitted if it has the takefocus option set to 0.""" name = self.tk.call('tk_focusNext', self._w) if not name: return None @@ -1827,18 +1827,24 @@ def _configure(self, cmd, cnf, kw): # These used to be defined in Widget: def configure(self, cnf=None, **kw): - """Configure resources of a widget. + """Query or modify the configuration options of the widget. - The values for resources are specified as keyword - arguments. To get an overview about - the allowed keyword arguments call the method keys. + If no arguments are specified, return a dictionary describing + all of the available options for the widget. + + If an option name is specified, then return a tuple describing + the one named option. + + If one or more keyword arguments are specified or a dictionary + is specified, then modify the widget option(s) to have the given + value(s). """ return self._configure('configure', cnf, kw) config = configure def cget(self, key): - """Return the resource value for a KEY given as string.""" + """Return the current value of the configuration option.""" return self.tk.call(self._w, 'cget', '-' + key) __getitem__ = cget @@ -1847,7 +1853,7 @@ def __setitem__(self, key, value): self.configure({key: value}) def keys(self): - """Return a list of all resource names of this widget.""" + """Return a list of all option names of this widget.""" splitlist = self.tk.splitlist return [splitlist(x)[0][1:] for x in splitlist(self.tk.call(self._w, 'configure'))] @@ -1966,7 +1972,7 @@ def _grid_configure(self, command, index, cnf, kw): def grid_columnconfigure(self, index, cnf={}, **kw): """Configure column INDEX of a grid. - Valid resources are minsize (minimum size of the column), + Valid options are minsize (minimum size of the column), weight (how much does additional space propagate to this column) and pad (how much space to let additionally).""" return self._grid_configure('columnconfigure', index, cnf, kw) @@ -1997,7 +2003,7 @@ def grid_propagate(self, flag=_noarg_): def grid_rowconfigure(self, index, cnf={}, **kw): """Configure row INDEX of a grid. - Valid resources are minsize (minimum size of the row), + Valid options are minsize (minimum size of the row), weight (how much does additional space propagate to this row) and pad (how much space to let additionally).""" return self._grid_configure('rowconfigure', index, cnf, kw) @@ -2817,7 +2823,7 @@ class Toplevel(BaseWidget, Wm): def __init__(self, master=None, cnf={}, **kw): """Construct a toplevel widget with the parent MASTER. - Valid resource names: background, bd, bg, borderwidth, class, + Valid option names: background, bd, bg, borderwidth, class, colormap, container, cursor, height, highlightbackground, highlightcolor, highlightthickness, menu, relief, screen, takefocus, use, visual, width.""" @@ -2894,7 +2900,7 @@ class Canvas(Widget, XView, YView): def __init__(self, master=None, cnf={}, **kw): """Construct a canvas widget with the parent MASTER. - Valid resource names: background, bd, bg, borderwidth, closeenough, + Valid option names: background, bd, bg, borderwidth, closeenough, confine, cursor, height, highlightbackground, highlightcolor, highlightthickness, insertbackground, insertborderwidth, insertofftime, insertontime, insertwidth, offset, relief, @@ -3103,16 +3109,14 @@ def insert(self, *args): self.tk.call((self._w, 'insert') + args) def itemcget(self, tagOrId, option): - """Return the resource value for an OPTION for item TAGORID.""" + """Return the value of OPTION for item TAGORID.""" return self.tk.call( (self._w, 'itemcget') + (tagOrId, '-'+option)) def itemconfigure(self, tagOrId, cnf=None, **kw): - """Configure resources of an item TAGORID. + """Query or modify the configuration options of an item TAGORID. - The values for resources are specified as keyword - arguments. To get an overview about - the allowed keyword arguments call the method without arguments. + Similar to configure() except that it applies to the specified item. """ return self._configure(('itemconfigure', tagOrId), cnf, kw) @@ -3204,7 +3208,7 @@ class Checkbutton(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct a checkbutton widget with the parent MASTER. - Valid resource names: activebackground, activeforeground, anchor, + Valid option names: activebackground, activeforeground, anchor, background, bd, bg, bitmap, borderwidth, command, cursor, disabledforeground, fg, font, foreground, height, highlightbackground, highlightcolor, highlightthickness, image, @@ -3235,7 +3239,7 @@ def flash(self): self.tk.call(self._w, 'flash') def invoke(self): - """Toggle the button and invoke a command if given as resource.""" + """Toggle the button and invoke a command if given as option.""" return self.tk.call(self._w, 'invoke') def select(self): @@ -3253,7 +3257,7 @@ class Entry(Widget, XView): def __init__(self, master=None, cnf={}, **kw): """Construct an entry widget with the parent MASTER. - Valid resource names: background, bd, bg, borderwidth, cursor, + Valid option names: background, bd, bg, borderwidth, cursor, exportselection, fg, font, foreground, highlightbackground, highlightcolor, highlightthickness, insertbackground, insertborderwidth, insertofftime, insertontime, insertwidth, @@ -3339,7 +3343,7 @@ class Frame(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct a frame widget with the parent MASTER. - Valid resource names: background, bd, bg, borderwidth, class, + Valid option names: background, bd, bg, borderwidth, class, colormap, container, cursor, height, highlightbackground, highlightcolor, highlightthickness, relief, takefocus, visual, width.""" cnf = _cnfmerge((cnf, kw)) @@ -3383,7 +3387,7 @@ class Listbox(Widget, XView, YView): def __init__(self, master=None, cnf={}, **kw): """Construct a listbox widget with the parent MASTER. - Valid resource names: background, bd, bg, borderwidth, cursor, + Valid option names: background, bd, bg, borderwidth, cursor, exportselection, fg, font, foreground, height, highlightbackground, highlightcolor, highlightthickness, relief, selectbackground, selectborderwidth, selectforeground, selectmode, setgrid, takefocus, @@ -3476,18 +3480,15 @@ def size(self): return self.tk.getint(self.tk.call(self._w, 'size')) def itemcget(self, index, option): - """Return the resource value for an ITEM and an OPTION.""" + """Return the value of OPTION for item at INDEX.""" return self.tk.call( (self._w, 'itemcget') + (index, '-'+option)) def itemconfigure(self, index, cnf=None, **kw): - """Configure resources of an ITEM. + """Query or modify the configuration options of an item at INDEX. - The values for resources are specified as keyword arguments. - To get an overview about the allowed keyword arguments - call the method without arguments. - Valid resource names: background, bg, foreground, fg, - selectbackground, selectforeground.""" + Similar to configure() except that it applies to the specified item. + """ return self._configure(('itemconfigure', index), cnf, kw) itemconfig = itemconfigure @@ -3499,7 +3500,7 @@ class Menu(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct menu widget with the parent MASTER. - Valid resource names: activebackground, activeborderwidth, + Valid option names: activebackground, activeborderwidth, activeforeground, background, bd, bg, borderwidth, cursor, disabledforeground, fg, font, foreground, postcommand, relief, selectcolor, takefocus, tearoff, tearoffcommand, title, type.""" @@ -3580,11 +3581,15 @@ def delete(self, index1, index2=None): self.tk.call(self._w, 'delete', index1, index2) def entrycget(self, index, option): - """Return the resource value of a menu item for OPTION at INDEX.""" + """Return the value of OPTION for a menu item at INDEX.""" return self.tk.call(self._w, 'entrycget', index, '-' + option) def entryconfigure(self, index, cnf=None, **kw): - """Configure a menu item at INDEX.""" + """Query or modify the configuration options of a menu item at INDEX. + + Similar to configure() except that it applies to the specified + menu item. + """ return self._configure(('entryconfigure', index), cnf, kw) entryconfig = entryconfigure @@ -3642,7 +3647,7 @@ class Radiobutton(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct a radiobutton widget with the parent MASTER. - Valid resource names: activebackground, activeforeground, anchor, + Valid option names: activebackground, activeforeground, anchor, background, bd, bg, bitmap, borderwidth, command, cursor, disabledforeground, fg, font, foreground, height, highlightbackground, highlightcolor, highlightthickness, image, @@ -3661,7 +3666,7 @@ def flash(self): self.tk.call(self._w, 'flash') def invoke(self): - """Toggle the button and invoke a command if given as resource.""" + """Toggle the button and invoke a command if given as option.""" return self.tk.call(self._w, 'invoke') def select(self): @@ -3675,7 +3680,7 @@ class Scale(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct a scale widget with the parent MASTER. - Valid resource names: activebackground, background, bigincrement, bd, + Valid option names: activebackground, background, bigincrement, bd, bg, borderwidth, command, cursor, digits, fg, font, foreground, from, highlightbackground, highlightcolor, highlightthickness, label, length, orient, relief, repeatdelay, repeatinterval, resolution, @@ -3714,7 +3719,7 @@ class Scrollbar(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct a scrollbar widget with the parent MASTER. - Valid resource names: activebackground, activerelief, + Valid option names: activebackground, activerelief, background, bd, bg, borderwidth, command, cursor, elementborderwidth, highlightbackground, highlightcolor, highlightthickness, jump, orient, @@ -3958,7 +3963,11 @@ def image_cget(self, index, option): return self.tk.call(self._w, "image", "cget", index, option) def image_configure(self, index, cnf=None, **kw): - """Configure an embedded image at INDEX.""" + """Query or modify the configuration options of an embedded image at INDEX. + + Similar to configure() except that it applies to the specified + embedded image. + """ return self._configure(('image', 'configure', index), cnf, kw) def image_create(self, index, cnf={}, **kw): @@ -4096,7 +4105,10 @@ def tag_cget(self, tagName, option): return self.tk.call(self._w, 'tag', 'cget', tagName, option) def tag_configure(self, tagName, cnf=None, **kw): - """Configure a tag TAGNAME.""" + """Query or modify the configuration options of a tag TAGNAME. + + Similar to configure() except that it applies to the specified tag. + """ return self._configure(('tag', 'configure', tagName), cnf, kw) tag_config = tag_configure @@ -4154,7 +4166,11 @@ def window_cget(self, index, option): return self.tk.call(self._w, 'window', 'cget', index, option) def window_configure(self, index, cnf=None, **kw): - """Configure an embedded window at INDEX.""" + """Query or modify the configuration options of an embedded window at INDEX. + + Similar to configure() except that it applies to the specified + embedded window. + """ return self._configure(('window', 'configure', index), cnf, kw) window_config = window_configure @@ -4194,7 +4210,7 @@ class OptionMenu(Menubutton): def __init__(self, master, variable, value, *values, **kwargs): """Construct an optionmenu widget with the parent MASTER, with - the resource textvariable set to VARIABLE, the initially selected + the option textvariable set to VARIABLE, the initially selected value VALUE, the other menu values VALUES and an additional keyword argument command.""" kw = {"borderwidth": 2, "textvariable": variable, @@ -4296,7 +4312,7 @@ class PhotoImage(Image): def __init__(self, name=None, cnf={}, master=None, **kw): """Create an image with NAME. - Valid resource names: data, format, file, gamma, height, palette, + Valid option names: data, format, file, gamma, height, palette, width.""" Image.__init__(self, 'photo', name, cnf, master, **kw) @@ -4559,7 +4575,7 @@ class BitmapImage(Image): def __init__(self, name=None, cnf={}, master=None, **kw): """Create a bitmap with NAME. - Valid resource names: background, data, file, foreground, maskdata, maskfile.""" + Valid option names: background, data, file, foreground, maskdata, maskfile.""" Image.__init__(self, 'bitmap', name, cnf, master, **kw) @@ -4877,26 +4893,17 @@ def sash_place(self, index, x, y): return self.sash("place", index, x, y) def panecget(self, child, option): - """Query a management option for window. - - Option may be any value allowed by the paneconfigure subcommand - """ + """Return the value of option for a child window.""" return self.tk.call( (self._w, 'panecget') + (child, '-'+option)) def paneconfigure(self, tagOrId, cnf=None, **kw): - """Query or modify the management options for window. - - If no option is specified, returns a list describing all - of the available options for pathName. If option is - specified with no value, then the command returns a list - describing the one named option (this list will be identical - to the corresponding sublist of the value returned if no - option is specified). If one or more option-value pairs are - specified, then the command modifies the given widget - option(s) to have the given value(s); in this case the - command returns an empty string. The following options - are supported: + """Query or modify the configuration options for a child window. + + Similar to configure() except that it applies to the specified + window. + + The following options are supported: after window Insert the window after the window specified. window diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index c0cf1e787fa9ad..ef2b91dfbb6638 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -1589,7 +1589,7 @@ class OptionMenu(Menubutton): def __init__(self, master, variable, default=None, *values, **kwargs): """Construct a themed OptionMenu widget with master as the parent, - the resource textvariable set to variable, the initially selected + the option textvariable set to variable, the initially selected value specified by the default parameter, the menu values given by *values and additional keywords. diff --git a/Lib/traceback.py b/Lib/traceback.py index f0dbb6352f7760..318ec13cf91121 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -137,8 +137,9 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ BUILTIN_EXCEPTION_LIMIT = object() -def _print_exception_bltin(exc, /): - file = sys.stderr if sys.stderr is not None else sys.__stderr__ +def _print_exception_bltin(exc, file=None, /): + if file is None: + file = sys.stderr if sys.stderr is not None else sys.__stderr__ colorize = _colorize.can_colorize(file=file) return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file, colorize=colorize) diff --git a/Lib/typing.py b/Lib/typing.py index f1455c273d31ca..8c1d265019bb94 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -256,16 +256,27 @@ def _type_repr(obj): return _lazy_annotationlib.type_repr(obj) -def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): +def _collect_type_parameters( + args, + *, + enforce_default_ordering: bool = True, + validate_all: bool = False, +): """Collect all type parameters in args in order of first appearance (lexicographic order). + Having an explicit `Generic` or `Protocol` base class determines + the exact parameter order. + For example:: >>> P = ParamSpec('P') >>> T = TypeVar('T') >>> _collect_type_parameters((T, Callable[P, T])) (~T, ~P) + >>> _collect_type_parameters((list[T], Generic[P, T])) + (~P, ~T) + """ # required type parameter cannot appear after parameter with default default_encountered = False @@ -297,6 +308,17 @@ def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): ' follows type parameter with a default') parameters.append(t) + elif ( + not validate_all + and isinstance(t, _GenericAlias) + and t.__origin__ in (Generic, Protocol) + ): + # If we see explicit `Generic[...]` or `Protocol[...]` base classes, + # we need to just copy them as-is. + # Unless `validate_all` is passed, in this case it means that + # we are doing a validation of `Generic` subclasses, + # then we collect all unique parameters to be able to inspect them. + parameters = t.__parameters__ else: if _is_unpacked_typevartuple(t): type_var_tuple_encountered = True @@ -1156,20 +1178,22 @@ def _generic_init_subclass(cls, *args, **kwargs): if error: raise TypeError("Cannot inherit from plain Generic") if '__orig_bases__' in cls.__dict__: - tvars = _collect_type_parameters(cls.__orig_bases__) + tvars = _collect_type_parameters(cls.__orig_bases__, validate_all=True) # Look for Generic[T1, ..., Tn]. # If found, tvars must be a subset of it. # If not found, tvars is it. # Also check for and reject plain Generic, # and reject multiple Generic[...]. gvars = None + basename = None for base in cls.__orig_bases__: if (isinstance(base, _GenericAlias) and - base.__origin__ is Generic): + base.__origin__ in (Generic, Protocol)): if gvars is not None: raise TypeError( "Cannot inherit from Generic[...] multiple times.") gvars = base.__parameters__ + basename = base.__origin__.__name__ if gvars is not None: tvarset = set(tvars) gvarset = set(gvars) @@ -1177,7 +1201,7 @@ def _generic_init_subclass(cls, *args, **kwargs): s_vars = ', '.join(str(t) for t in tvars if t not in gvarset) s_args = ', '.join(str(g) for g in gvars) raise TypeError(f"Some type variables ({s_vars}) are" - f" not listed in Generic[{s_args}]") + f" not listed in {basename}[{s_args}]") tvars = gvars cls.__parameters__ = tuple(tvars) @@ -2342,10 +2366,13 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, # *base_globals* first rather than *base_locals*. # This only affects ForwardRefs. base_globals, base_locals = base_locals, base_globals + type_params = base.__type_params__ + base_globals, base_locals = _add_type_params_to_scope( + type_params, base_globals, base_locals, True) for name, value in ann.items(): if isinstance(value, str): value = _make_forward_ref(value, is_argument=False, is_class=True) - value = _eval_type(value, base_globals, base_locals, base.__type_params__, + value = _eval_type(value, base_globals, base_locals, (), format=format, owner=obj) if value is None: value = type(None) @@ -2381,6 +2408,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, elif localns is None: localns = globalns type_params = getattr(obj, "__type_params__", ()) + globalns, localns = _add_type_params_to_scope(type_params, globalns, localns, False) for name, value in hints.items(): if isinstance(value, str): # class-level forward refs were handled above, this must be either @@ -2390,13 +2418,27 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, is_argument=not isinstance(obj, types.ModuleType), is_class=False, ) - value = _eval_type(value, globalns, localns, type_params, format=format, owner=obj) + value = _eval_type(value, globalns, localns, (), format=format, owner=obj) if value is None: value = type(None) hints[name] = value return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} +# Add type parameters to the globals and locals scope. This is needed for +# compatibility. +def _add_type_params_to_scope(type_params, globalns, localns, is_class): + if not type_params: + return globalns, localns + globalns = dict(globalns) + localns = dict(localns) + for param in type_params: + if not is_class or param.__name__ not in globalns: + globalns[param.__name__] = param + localns.pop(param.__name__, None) + return globalns, localns + + def _strip_annotations(t): """Strip the annotations from a given type.""" if isinstance(t, _AnnotatedAlias): diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index b31cb766a468f4..4da6d924848889 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.49.1", - url="https://sqlite.org/2025/sqlite-autoconf-3490100.tar.gz", - checksum="106642d8ccb36c5f7323b64e4152e9b719f7c0215acf5bfeac3d5e7f97b59254", + name="SQLite 3.50.4", + url="https://www.sqlite.org/2025/sqlite-autoconf-3500400.tar.gz", + checksum="a3db587a1b92ee5ddac2f66b3edb41b26f9c867275782d46c3a088977d6a5b18", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' @@ -1747,7 +1747,7 @@ def main(): fn = os.path.join(folder, "ReadMe.rtf") patchFile("resources/ReadMe.rtf", fn) fn = os.path.join(folder, "Update Shell Profile.command") - patchScript("scripts/postflight.patch-profile", fn) + patchScript("resources/update_shell_profile.command", fn) fn = os.path.join(folder, "Install Certificates.command") patchScript("resources/install_certificates.command", fn) os.chmod(folder, STAT_0o755) diff --git a/Mac/BuildScript/resources/update_shell_profile.command b/Mac/BuildScript/resources/update_shell_profile.command new file mode 100755 index 00000000000000..3cf4d74de9f09a --- /dev/null +++ b/Mac/BuildScript/resources/update_shell_profile.command @@ -0,0 +1,116 @@ +#!/bin/sh + +echo "This script will update your shell profile when the 'bin' directory" +echo "of python is not early enough of the PATH of your shell." +echo "These changes will be effective only in shell windows that you open" +echo "after running this script." + +PYVER=@PYVER@ +PYTHON_ROOT="/Library/Frameworks/Python.framework/Versions/@PYVER@" + +if [ `id -ur` = 0 ]; then + # Run from the installer, do some trickery to fetch the information + # we need. + theShell="`finger $USER | grep Shell: | head -1 | awk '{ print $NF }'`" + +else + theShell="${SHELL}" +fi + +# Make sure the directory ${PYTHON_ROOT}/bin is on the users PATH. +BSH="`basename "${theShell}"`" +case "${BSH}" in +bash|ksh|sh|*csh|zsh|fish) + if [ `id -ur` = 0 ]; then + P=`su - ${USER} -c 'echo A-X-4-X@@$PATH@@X-4-X-A' | grep 'A-X-4-X@@.*@@X-4-X-A' | sed -e 's/^A-X-4-X@@//g' -e 's/@@X-4-X-A$//g'` + else + P="`(exec -l ${theShell} -c 'echo $PATH')`" + fi + ;; +*) + echo "Sorry, I don't know how to patch $BSH shells" + exit 0 + ;; +esac + +# Now ensure that our bin directory is on $P and before /usr/bin at that +for elem in `echo $P | tr ':' ' '` +do + if [ "${elem}" = "${PYTHON_ROOT}/bin" ]; then + echo "All right, you're a python lover already" + exit 0 + elif [ "${elem}" = "/usr/bin" ]; then + break + fi +done + +echo "${PYTHON_ROOT}/bin is not on your PATH or at least not early enough" +case "${BSH}" in +*csh) + if [ -f "${HOME}/.tcshrc" ]; then + RC="${HOME}/.tcshrc" + else + RC="${HOME}/.cshrc" + fi + # Create backup copy before patching + if [ -f "${RC}" ]; then + cp -fp "${RC}" "${RC}.pysave" + fi + echo "" >> "${RC}" + echo "# Setting PATH for Python ${PYVER}" >> "${RC}" + echo "# The original version is saved in .cshrc.pysave" >> "${RC}" + echo "set path=(${PYTHON_ROOT}/bin "'$path'")" >> "${RC}" + if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${RC}" + fi + exit 0 + ;; +bash) + if [ -e "${HOME}/.bash_profile" ]; then + PR="${HOME}/.bash_profile" + elif [ -e "${HOME}/.bash_login" ]; then + PR="${HOME}/.bash_login" + elif [ -e "${HOME}/.profile" ]; then + PR="${HOME}/.profile" + else + PR="${HOME}/.bash_profile" + fi + ;; +fish) + CONFIG_DIR="${HOME}/.config/fish/conf.d/" + RC="${CONFIG_DIR}/python-${PYVER}.fish" + mkdir -p "$CONFIG_DIR" + if [ -f "${RC}" ]; then + cp -fp "${RC}" "${RC}.pysave" + fi + echo "# Setting PATH for Python ${PYVER}" > "${RC}" + if [ -f "${RC}.pysave" ]; then + echo "# The original version is saved in ${RC}.pysave" >> "${RC}" + fi + echo "fish_add_path -g \"${PYTHON_ROOT}/bin\"" >> "${RC}" + if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${RC}" + fi + exit 0 + ;; +zsh) + PR="${HOME}/.zprofile" + ;; +*sh) + PR="${HOME}/.profile" + ;; +esac + +# Create backup copy before patching +if [ -f "${PR}" ]; then + cp -fp "${PR}" "${PR}.pysave" +fi +echo "" >> "${PR}" +echo "# Setting PATH for Python ${PYVER}" >> "${PR}" +echo "# The original version is saved in `basename ${PR}`.pysave" >> "${PR}" +echo 'PATH="'"${PYTHON_ROOT}/bin"':${PATH}"' >> "${PR}" +echo 'export PATH' >> "${PR}" +if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${PR}" +fi +exit 0 diff --git a/Mac/BuildScript/scripts/postflight.patch-profile b/Mac/BuildScript/scripts/postflight.patch-profile index 9caf62211ddd16..ce8720f895d1b5 100755 --- a/Mac/BuildScript/scripts/postflight.patch-profile +++ b/Mac/BuildScript/scripts/postflight.patch-profile @@ -1,116 +1,104 @@ #!/bin/sh -echo "This script will update your shell profile when the 'bin' directory" -echo "of python is not early enough of the PATH of your shell." -echo "These changes will be effective only in shell windows that you open" -echo "after running this script." - PYVER=@PYVER@ PYTHON_ROOT="/Library/Frameworks/Python.framework/Versions/@PYVER@" -if [ `id -ur` = 0 ]; then - # Run from the installer, do some trickery to fetch the information - # we need. - theShell="`finger $USER | grep Shell: | head -1 | awk '{ print $NF }'`" -else - theShell="${SHELL}" -fi +# Run from the installer, do some trickery to fetch the information +# we need. +theShell="`finger $USER | grep Shell: | head -1 | awk '{ print $NF }'`" # Make sure the directory ${PYTHON_ROOT}/bin is on the users PATH. BSH="`basename "${theShell}"`" case "${BSH}" in bash|ksh|sh|*csh|zsh|fish) - if [ `id -ur` = 0 ]; then - P=`su - ${USER} -c 'echo A-X-4-X@@$PATH@@X-4-X-A' | grep 'A-X-4-X@@.*@@X-4-X-A' | sed -e 's/^A-X-4-X@@//g' -e 's/@@X-4-X-A$//g'` - else - P="`(exec -l ${theShell} -c 'echo $PATH')`" - fi - ;; + true + ;; *) - echo "Sorry, I don't know how to patch $BSH shells" - exit 0 - ;; + exit 0 + ;; esac -# Now ensure that our bin directory is on $P and before /usr/bin at that -for elem in `echo $P | tr ':' ' '` -do - if [ "${elem}" = "${PYTHON_ROOT}/bin" ]; then - echo "All right, you're a python lover already" - exit 0 - elif [ "${elem}" = "/usr/bin" ]; then - break - fi -done - -echo "${PYTHON_ROOT}/bin is not on your PATH or at least not early enough" case "${BSH}" in *csh) - if [ -f "${HOME}/.tcshrc" ]; then - RC="${HOME}/.tcshrc" - else - RC="${HOME}/.cshrc" - fi - # Create backup copy before patching - if [ -f "${RC}" ]; then - cp -fp "${RC}" "${RC}.pysave" - fi - echo "" >> "${RC}" - echo "# Setting PATH for Python ${PYVER}" >> "${RC}" - echo "# The original version is saved in .cshrc.pysave" >> "${RC}" - echo "set path=(${PYTHON_ROOT}/bin "'$path'")" >> "${RC}" - if [ `id -ur` = 0 ]; then - chown "${USER}" "${RC}" - fi - exit 0 - ;; + if [ -f "${HOME}/.tcshrc" ]; then + RC="${HOME}/.tcshrc" + else + RC="${HOME}/.cshrc" + fi + + # Drop privileges while writing files. + su -m ${USER} <> "${RC}" + echo "# Setting PATH for Python ${PYVER}" >> "${RC}" + echo "# The original version is saved in .cshrc.pysave" >> "${RC}" + echo "set path=(${PYTHON_ROOT}/bin "'\$path'")" >> "${RC}" +EOFC + + if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${RC}" + fi + exit 0 + ;; bash) - if [ -e "${HOME}/.bash_profile" ]; then - PR="${HOME}/.bash_profile" - elif [ -e "${HOME}/.bash_login" ]; then - PR="${HOME}/.bash_login" - elif [ -e "${HOME}/.profile" ]; then - PR="${HOME}/.profile" - else - PR="${HOME}/.bash_profile" - fi - ;; + if [ -e "${HOME}/.bash_profile" ]; then + PR="${HOME}/.bash_profile" + elif [ -e "${HOME}/.bash_login" ]; then + PR="${HOME}/.bash_login" + elif [ -e "${HOME}/.profile" ]; then + PR="${HOME}/.profile" + else + PR="${HOME}/.bash_profile" + fi + ;; fish) - CONFIG_DIR="${HOME}/.config/fish/conf.d/" - RC="${CONFIG_DIR}/python-${PYVER}.fish" - mkdir -p "$CONFIG_DIR" - if [ -f "${RC}" ]; then - cp -fp "${RC}" "${RC}.pysave" - fi - echo "# Setting PATH for Python ${PYVER}" > "${RC}" - if [ -f "${RC}.pysave" ]; then - echo "# The original version is saved in ${RC}.pysave" >> "${RC}" - fi - echo "fish_add_path -g \"${PYTHON_ROOT}/bin\"" >> "${RC}" - if [ `id -ur` = 0 ]; then - chown "${USER}" "${RC}" - fi - exit 0 - ;; + CONFIG_DIR="${HOME}/.config/fish/conf.d/" + RC="${CONFIG_DIR}/python-${PYVER}.fish" + + # Drop privileges while writing files. + su -m ${USER} < "${RC}" + if [ -f "${RC}.pysave" ]; then + echo "# The original version is saved in ${RC}.pysave" >> "${RC}" + fi + echo "fish_add_path -g \"${PYTHON_ROOT}/bin\"" >> "${RC}" +EOFF + + if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${RC}" + fi + exit 0 + ;; zsh) - PR="${HOME}/.zprofile" - ;; + PR="${HOME}/.zprofile" + ;; *sh) - PR="${HOME}/.profile" - ;; + PR="${HOME}/.profile" + ;; esac +# Drop privileges while writing files. +su -m ${USER} <> "${PR}" echo "# Setting PATH for Python ${PYVER}" >> "${PR}" echo "# The original version is saved in `basename ${PR}`.pysave" >> "${PR}" -echo 'PATH="'"${PYTHON_ROOT}/bin"':${PATH}"' >> "${PR}" +echo 'PATH="'"${PYTHON_ROOT}/bin"':\${PATH}"' >> "${PR}" echo 'export PATH' >> "${PR}" +EOFS + if [ `id -ur` = 0 ]; then - chown "${USER}" "${PR}" + chown -h "${USER}" "${PR}" fi exit 0 diff --git a/Makefile.pre.in b/Makefile.pre.in index e2253d3e35b3e6..bcf19654adfb35 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -227,7 +227,6 @@ ENSUREPIP= @ENSUREPIP@ # Internal static libraries LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a LIBEXPAT_A= Modules/expat/libexpat.a -LIBHASHLIB_INTERNAL_A=Modules/_hashlib/libhashlib.a # HACL* build configuration LIBHACL_CFLAGS=@LIBHACL_CFLAGS@ @@ -762,17 +761,6 @@ LIBHACL_HMAC_HEADERS= \ $(LIBHACL_BLAKE2_HEADERS) \ $(LIBHACL_HEADERS) -########################################################################## -# Internal library for cryptographic primitives - -LIBHASHLIB_INTERNAL_OBJS= \ - Modules/_hashlib/hashlib_buffer.o - -LIBHASHLIB_INTERNAL_HEADERS= \ - Modules/_hashlib/hashlib_buffer.h \ - Modules/_hashlib/hashlib_fetch.h \ - Modules/_hashlib/hashlib_mutex.h - ######################################################################### # Rules @@ -1527,17 +1515,6 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS) -rm -f $@ $(AR) $(ARFLAGS) $@ $(LIBEXPAT_OBJS) -########################################################################## -# '_hashlib', '_hmac' and HACL*-based modules helpers -LIBHASHLIB_INTERNAL_CFLAGS=@LIBHASHLIB_INTERNAL_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED) - -Modules/_hashlib/hashlib_buffer.o: Modules/_hashlib/hashlib_buffer.c $(LIBHASHLIB_INTERNAL_HEADERS) $(PYTHON_HEADERS) - $(CC) -I$(srcdir)/Modules/_hashlib -c $(LIBHASHLIB_INTERNAL_CFLAGS) -o $@ $(srcdir)/Modules/_hashlib/hashlib_buffer.c - -$(LIBHASHLIB_INTERNAL_A): $(LIBHASHLIB_INTERNAL_OBJS) - -rm -f $@ - $(AR) $(ARFLAGS) $@ $(LIBHASHLIB_INTERNAL_OBJS) - ########################################################################## # HACL* library build # @@ -2253,7 +2230,7 @@ Python/frozen.o: $(FROZEN_FILES_OUT) # an include guard, so we can't use a pipeline to transform its output. Include/pydtrace_probes.h: $(srcdir)/Include/pydtrace.d $(MKDIR_P) Include - CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -h -s $< + CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -h -s $(srcdir)/Include/pydtrace.d : sed in-place edit with POSIX-only tools sed 's/PYTHON_/PyDTrace_/' $@ > $@.tmp mv $@.tmp $@ @@ -2263,7 +2240,7 @@ Python/gc.o: $(srcdir)/Include/pydtrace.h Python/import.o: $(srcdir)/Include/pydtrace.h Python/pydtrace.o: $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) - CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -G -s $< $(DTRACE_DEPS) + CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -G -s $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) Objects/typeobject.o: Objects/typeslots.inc @@ -3380,21 +3357,21 @@ MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ -MODULE__HASHLIB_DEPS=@LIBHASHLIB_INTERNAL@ +MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h # HACL*-based cryptographic primitives -MODULE__MD5_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__MD5_LDEPS=$(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA1_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA1_LDEPS=$(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA2_LDEPS=$(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA3_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA3_LDEPS=$(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__BLAKE2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__BLAKE2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__BLAKE2_LDEPS=$(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__HMAC_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__HMAC_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__HMAC_LDEPS=$(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c diff --git a/Misc/ACKS b/Misc/ACKS index 745f472474cd9d..dc28ccf8f57eda 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1902,6 +1902,7 @@ Nicolas M. Thiéry James Thomas Reuben Thomas Robin Thomas +Douglas Thor Brian Thorne Christopher Thorne Stephen Thorne diff --git a/Misc/NEWS.d/next/Build/2025-07-22-14-47-45.gh-issue-131876.oaYEEP.rst b/Misc/NEWS.d/next/Build/2025-07-22-14-47-45.gh-issue-131876.oaYEEP.rst deleted file mode 100644 index 304f2c30f664db..00000000000000 --- a/Misc/NEWS.d/next/Build/2025-07-22-14-47-45.gh-issue-131876.oaYEEP.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove :file:`!Modules/hashlib.h` and move its content into dedicated files -now located in ``Modules/_hashlib``. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/C_API/2025-07-31-04-30-42.gh-issue-128813.opL-Pv.rst b/Misc/NEWS.d/next/C_API/2025-07-31-04-30-42.gh-issue-128813.opL-Pv.rst new file mode 100644 index 00000000000000..caa8f3e9c985cd --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-07-31-04-30-42.gh-issue-128813.opL-Pv.rst @@ -0,0 +1,5 @@ +Functions :c:func:`_Py_c_sum`, :c:func:`_Py_c_diff`, :c:func:`_Py_c_neg`, +:c:func:`_Py_c_prod`, :c:func:`_Py_c_quot`, :c:func:`_Py_c_pow` and previously +undocumented :c:func:`_Py_c_abs` are :term:`soft deprecated`. Deprecate also +:c:member:`~PyComplexObject.cval` field of the :c:type:`PyComplexObject` type. +Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-18-10-50-46.gh-issue-134170.J0Hvmi.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-18-10-50-46.gh-issue-134170.J0Hvmi.rst new file mode 100644 index 00000000000000..f33a30c7e120dc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-18-10-50-46.gh-issue-134170.J0Hvmi.rst @@ -0,0 +1 @@ +Add colorization to :func:`sys.unraisablehook` by default. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-03-06-04-42.gh-issue-135552.CbBQof.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-03-06-04-42.gh-issue-135552.CbBQof.rst new file mode 100644 index 00000000000000..ea30a43fc25d41 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-03-06-04-42.gh-issue-135552.CbBQof.rst @@ -0,0 +1,7 @@ +Fix a bug caused by the garbage collector clearing weakrefs too early. The +weakrefs in the ``tp_subclasses`` dictionary are needed in order to correctly +invalidate type caches (for example, by calling ``PyType_Modified()``). +Clearing weakrefs before calling finalizers causes the caches to not be +correctly invalidated. That can cause crashes since the caches can refer to +invalid objects. Defer the clearing of weakrefs without callbacks until after +finalizers are executed. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-12-29-09.gh-issue-107545.ipfl7U.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-12-29-09.gh-issue-107545.ipfl7U.rst new file mode 100644 index 00000000000000..23122415e8a46f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-12-29-09.gh-issue-107545.ipfl7U.rst @@ -0,0 +1,2 @@ +Improve the error messages that may be raised by +:meth:`~socket.socket.setsockopt`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-31-23-02-02.gh-issue-137291.kIxVZd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-31-23-02-02.gh-issue-137291.kIxVZd.rst new file mode 100644 index 00000000000000..0995e3b4644e7c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-31-23-02-02.gh-issue-137291.kIxVZd.rst @@ -0,0 +1 @@ +The perf profiler can now be used if a previous frame evaluation API has been provided. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-01-18-54-31.gh-issue-137288.FhE7ku.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-01-18-54-31.gh-issue-137288.FhE7ku.rst new file mode 100644 index 00000000000000..37c143f18e76f7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-01-18-54-31.gh-issue-137288.FhE7ku.rst @@ -0,0 +1,2 @@ +Fix bug where some bytecode instructions of a boolean expression are not +associated with the correct exception handler. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-02-10-27-53.gh-issue-137308.at05p_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-02-10-27-53.gh-issue-137308.at05p_.rst new file mode 100644 index 00000000000000..8003de422b2919 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-02-10-27-53.gh-issue-137308.at05p_.rst @@ -0,0 +1,3 @@ +A standalone docstring in a node body is optimized as a :keyword:`pass` +statement to ensure that the node's body is never empty. There was a +:exc:`ValueError` in :func:`compile` otherwise. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-02-23-04-57.gh-issue-137314.wjEdzD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-02-23-04-57.gh-issue-137314.wjEdzD.rst new file mode 100644 index 00000000000000..09d0c3e68fc1ed --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-02-23-04-57.gh-issue-137314.wjEdzD.rst @@ -0,0 +1,5 @@ +Fixed a regression where raw f-strings incorrectly interpreted +escape sequences in format specifications. Raw f-strings now properly preserve +literal backslashes in format specs, matching the behavior from Python 3.11. +For example, ``rf"{obj:\xFF}"`` now correctly produces ``'\\xFF'`` instead of +``'ÿ'``. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-05-17-22-24.gh-issue-58124.q1__53.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-05-17-22-24.gh-issue-58124.q1__53.rst new file mode 100644 index 00000000000000..f875d4c5e785c6 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-05-17-22-24.gh-issue-58124.q1__53.rst @@ -0,0 +1,3 @@ +Fix name of the Python encoding in Unicode errors of the code page codec: +use "cp65000" and "cp65001" instead of "CP_UTF7" and "CP_UTF8" which are not +valid Python code names. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-06-15-39-54.gh-issue-137400.xIw0zs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-06-15-39-54.gh-issue-137400.xIw0zs.rst new file mode 100644 index 00000000000000..a464cf48948f9a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-06-15-39-54.gh-issue-137400.xIw0zs.rst @@ -0,0 +1,4 @@ +Fix a crash in the :term:`free threading` build when disabling profiling or tracing +across all threads with :c:func:`PyEval_SetProfileAllThreads` or +:c:func:`PyEval_SetTraceAllThreads` or their Python equivalents +:func:`threading.settrace_all_threads` and :func:`threading.setprofile_all_threads`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-07-09-52-19.gh-issue-137400.AK1dy-.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-07-09-52-19.gh-issue-137400.AK1dy-.rst new file mode 100644 index 00000000000000..406d6528840ba5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-07-09-52-19.gh-issue-137400.AK1dy-.rst @@ -0,0 +1,5 @@ +Fix a crash in the :term:`free threading` build when disabling profiling or +tracing across all threads with :c:func:`PyEval_SetProfileAllThreads` or +:c:func:`PyEval_SetTraceAllThreads` or their Python equivalents +:func:`threading.settrace_all_threads` and +:func:`threading.setprofile_all_threads`. diff --git a/Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst b/Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst deleted file mode 100644 index 0341b5f7f0d5e6..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst +++ /dev/null @@ -1 +0,0 @@ -EPUB builds are fixed by excluding non-XHTML-compatible tags. diff --git a/Misc/NEWS.d/next/Library/2021-09-21-17-17-29.gh-issue-84683.wDSRsG.rst b/Misc/NEWS.d/next/Library/2021-09-21-17-17-29.gh-issue-84683.wDSRsG.rst new file mode 100644 index 00000000000000..66f76bda6ad7a7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-09-21-17-17-29.gh-issue-84683.wDSRsG.rst @@ -0,0 +1 @@ +:mod:`zoneinfo`: Check in ``/share/zoneinfo`` for data files on Windows diff --git a/Misc/NEWS.d/next/Library/2025-03-27-08-13-32.gh-issue-131788.0RWiFc.rst b/Misc/NEWS.d/next/Library/2025-03-27-08-13-32.gh-issue-131788.0RWiFc.rst new file mode 100644 index 00000000000000..525802405bd8bd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-27-08-13-32.gh-issue-131788.0RWiFc.rst @@ -0,0 +1 @@ +Make ``ResourceTracker.send`` from :mod:`multiprocessing` re-entrant safe diff --git a/Misc/NEWS.d/next/Library/2025-05-08-20-03-20.gh-issue-133722.1-B82a.rst b/Misc/NEWS.d/next/Library/2025-05-08-20-03-20.gh-issue-133722.1-B82a.rst new file mode 100644 index 00000000000000..86f244412498c4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-08-20-03-20.gh-issue-133722.1-B82a.rst @@ -0,0 +1,2 @@ +Added a *color* option to :func:`difflib.unified_diff` that colors output +similar to :program:`git diff`. diff --git a/Misc/NEWS.d/next/Library/2025-05-29-19-00-37.gh-issue-134861.y2-fu-.rst b/Misc/NEWS.d/next/Library/2025-05-29-19-00-37.gh-issue-134861.y2-fu-.rst new file mode 100644 index 00000000000000..07e4c61b404ba4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-29-19-00-37.gh-issue-134861.y2-fu-.rst @@ -0,0 +1 @@ +Add CSV as an output format for :program:`python -m asyncio ps`. diff --git a/Misc/NEWS.d/next/Library/2025-06-11-15-08-02.gh-issue-135336.6Gq6MI.rst b/Misc/NEWS.d/next/Library/2025-06-11-15-08-02.gh-issue-135336.6Gq6MI.rst new file mode 100644 index 00000000000000..8a1d492ff08944 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-11-15-08-02.gh-issue-135336.6Gq6MI.rst @@ -0,0 +1 @@ +:mod:`json` now encodes strings up to 2.2x faster if they consist solely of characters that don’t require escaping. diff --git a/Misc/NEWS.d/next/Library/2025-07-12-14-15-47.gh-issue-136571.muHmBv.rst b/Misc/NEWS.d/next/Library/2025-07-12-14-15-47.gh-issue-136571.muHmBv.rst new file mode 100644 index 00000000000000..37f535f564883c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-12-14-15-47.gh-issue-136571.muHmBv.rst @@ -0,0 +1,2 @@ +:meth:`datetime.date.fromisocalendar` can now raise OverflowError for out of +range arguments. diff --git a/Misc/NEWS.d/next/Library/2025-07-20-16-56-55.gh-issue-135228.n_XIao.rst b/Misc/NEWS.d/next/Library/2025-07-20-16-56-55.gh-issue-135228.n_XIao.rst new file mode 100644 index 00000000000000..517a37feb37ae6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-20-16-56-55.gh-issue-135228.n_XIao.rst @@ -0,0 +1,4 @@ +When :mod:`dataclasses` replaces a class with a slotted dataclass, the +original class can now be garbage collected again. Earlier changes in Python +3.14 caused this class to always remain in existence together with the replacement +class synthesized by :mod:`dataclasses`. diff --git a/Misc/NEWS.d/next/Library/2025-07-28-23-11-29.gh-issue-81325.jMJFBe.rst b/Misc/NEWS.d/next/Library/2025-07-28-23-11-29.gh-issue-81325.jMJFBe.rst new file mode 100644 index 00000000000000..3d89b6eb92a0d6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-28-23-11-29.gh-issue-81325.jMJFBe.rst @@ -0,0 +1,2 @@ +:class:`tarfile.TarFile` now accepts a :term:`path-like ` when working on a tar archive. +(Contributed by Alexander Enrique Urieles Nieto in :gh:`81325`.) diff --git a/Misc/NEWS.d/next/Library/2025-07-29-21-18-31.gh-issue-137226.B_4lpu.rst b/Misc/NEWS.d/next/Library/2025-07-29-21-18-31.gh-issue-137226.B_4lpu.rst new file mode 100644 index 00000000000000..522943cdd376dc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-29-21-18-31.gh-issue-137226.B_4lpu.rst @@ -0,0 +1,3 @@ +Fix behavior of :meth:`annotationlib.ForwardRef.evaluate` when the +*type_params* parameter is passed and the name of a type param is also +present in an enclosing scope. diff --git a/Misc/NEWS.d/next/Library/2025-07-30-18-07-33.gh-issue-137257.XBtzf2.rst b/Misc/NEWS.d/next/Library/2025-07-30-18-07-33.gh-issue-137257.XBtzf2.rst new file mode 100644 index 00000000000000..fad609854025c0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-30-18-07-33.gh-issue-137257.XBtzf2.rst @@ -0,0 +1 @@ +Bump the version of pip bundled in ensurepip to version 25.2 diff --git a/Misc/NEWS.d/next/Library/2025-07-31-16-43-16.gh-issue-137191.FIogE8.rst b/Misc/NEWS.d/next/Library/2025-07-31-16-43-16.gh-issue-137191.FIogE8.rst new file mode 100644 index 00000000000000..b2dba81251eed6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-31-16-43-16.gh-issue-137191.FIogE8.rst @@ -0,0 +1,4 @@ +Fix how type parameters are collected, when :class:`typing.Protocol` are +specified with explicit parameters. Now, :class:`typing.Generic` and +:class:`typing.Protocol` always dictate the parameter number +and parameter ordering of types. Previous behavior was a bug. diff --git a/Misc/NEWS.d/next/Library/2025-08-01-15-07-59.gh-issue-137273.4V8Xmv.rst b/Misc/NEWS.d/next/Library/2025-08-01-15-07-59.gh-issue-137273.4V8Xmv.rst new file mode 100644 index 00000000000000..f344877955fea0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-01-15-07-59.gh-issue-137273.4V8Xmv.rst @@ -0,0 +1 @@ +Fix debug assertion failure in :func:`locale.setlocale` on Windows. diff --git a/Misc/NEWS.d/next/Library/2025-08-01-23-52-49.gh-issue-75989.5aYXNJ.rst b/Misc/NEWS.d/next/Library/2025-08-01-23-52-49.gh-issue-75989.5aYXNJ.rst new file mode 100644 index 00000000000000..00b15503b50ba3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-01-23-52-49.gh-issue-75989.5aYXNJ.rst @@ -0,0 +1,3 @@ +:func:`tarfile.TarFile.extractall` and :func:`tarfile.TarFile.extract` now +overwrite symlinks when extracting hardlinks. +(Contributed by Alexander Enrique Urieles Nieto in :gh:`75989`.) diff --git a/Misc/NEWS.d/next/Library/2025-08-03-00-36-57.gh-issue-115766.nJCFkW.rst b/Misc/NEWS.d/next/Library/2025-08-03-00-36-57.gh-issue-115766.nJCFkW.rst new file mode 100644 index 00000000000000..d8d2203847caf6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-03-00-36-57.gh-issue-115766.nJCFkW.rst @@ -0,0 +1 @@ +Fix :attr:`!ipaddress.IPv4Interface.is_unspecified`. diff --git a/Misc/NEWS.d/next/Library/2025-08-03-13-16-39.gh-issue-137044.0hPVL_.rst b/Misc/NEWS.d/next/Library/2025-08-03-13-16-39.gh-issue-137044.0hPVL_.rst new file mode 100644 index 00000000000000..f5f96263823e86 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-03-13-16-39.gh-issue-137044.0hPVL_.rst @@ -0,0 +1,4 @@ +Return large limit values as positive integers instead of negative integers +in :func:`resource.getrlimit`. Accept large values and reject negative +values (except :data:`~resource.RLIM_INFINITY`) for limits in +:func:`resource.setrlimit`. diff --git a/Misc/NEWS.d/next/Library/2025-08-06-16-13-47.gh-issue-137466.Whv0-A.rst b/Misc/NEWS.d/next/Library/2025-08-06-16-13-47.gh-issue-137466.Whv0-A.rst new file mode 100644 index 00000000000000..918019aa8c03eb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-06-16-13-47.gh-issue-137466.Whv0-A.rst @@ -0,0 +1,3 @@ +Remove undocumented :func:`!glob.glob0` and :func:`!glob.glob1` functions, +which have been deprecated since Python 3.13. Use :func:`glob.glob` and pass +a directory to its *root_dir* argument instead. diff --git a/Misc/NEWS.d/next/Library/2025-08-08-15-00-38.gh-issue-137426.lW-Rk2.rst b/Misc/NEWS.d/next/Library/2025-08-08-15-00-38.gh-issue-137426.lW-Rk2.rst new file mode 100644 index 00000000000000..6d05c8a3bf4e4e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-08-15-00-38.gh-issue-137426.lW-Rk2.rst @@ -0,0 +1,3 @@ +Remove the code deprecation of ``importlib.abc.ResourceLoader``. It is +documented as deprecated, but left for backwards compatibility with other +classes in ``importlib.abc``. diff --git a/Misc/NEWS.d/next/Library/2025-08-08-21-20-14.gh-issue-92936.rOgG1S.rst b/Misc/NEWS.d/next/Library/2025-08-08-21-20-14.gh-issue-92936.rOgG1S.rst new file mode 100644 index 00000000000000..906c442b64f438 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-08-21-20-14.gh-issue-92936.rOgG1S.rst @@ -0,0 +1,2 @@ +Update regex used by ``http.cookies.SimpleCookie`` to handle values containing +double quotes. diff --git a/Misc/NEWS.d/next/Library/2025-08-09-08-53-32.gh-issue-137583.s6OZud.rst b/Misc/NEWS.d/next/Library/2025-08-09-08-53-32.gh-issue-137583.s6OZud.rst new file mode 100644 index 00000000000000..3843cc7c8c5524 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-09-08-53-32.gh-issue-137583.s6OZud.rst @@ -0,0 +1,4 @@ +Fix a deadlock introduced in 3.13.6 when a call to +:meth:`ssl.SSLSocket.recv ` was blocked in one thread, +and then another method on the object (such as :meth:`ssl.SSLSocket.send `) +was subsequently called in another thread. diff --git a/Misc/NEWS.d/next/Library/2025-08-11-05-05-08.gh-issue-137630.9lmqyc.rst b/Misc/NEWS.d/next/Library/2025-08-11-05-05-08.gh-issue-137630.9lmqyc.rst new file mode 100644 index 00000000000000..94d836c17834fd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-11-05-05-08.gh-issue-137630.9lmqyc.rst @@ -0,0 +1,2 @@ +The :mod:`!_interpreters` module now uses Argument Clinic to parse arguments. +Patch by Adam Turner. diff --git a/Misc/NEWS.d/next/Library/2025-08-13-10-50-22.gh-issue-73487.DUHbBq.rst b/Misc/NEWS.d/next/Library/2025-08-13-10-50-22.gh-issue-73487.DUHbBq.rst new file mode 100644 index 00000000000000..8cca50d91cd198 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-13-10-50-22.gh-issue-73487.DUHbBq.rst @@ -0,0 +1,3 @@ +Speedup processing arguments (up to 1.5x) in the :mod:`decimal` module +methods, that now using :c:macro:`METH_FASTCALL` calling convention. Patch +by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-08-06-11-54-55.gh-issue-137484.8iFAQs.rst b/Misc/NEWS.d/next/Tools-Demos/2025-08-06-11-54-55.gh-issue-137484.8iFAQs.rst new file mode 100644 index 00000000000000..bd7bc0984ecfcc --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-08-06-11-54-55.gh-issue-137484.8iFAQs.rst @@ -0,0 +1,2 @@ +Have ``Tools/wasm/wasi`` put the build Python into a directory named after +the build triple instead of "build". diff --git a/Misc/NEWS.d/next/Windows/2025-07-27-02-16-53.gh-issue-137134.W0WpDF.rst b/Misc/NEWS.d/next/Windows/2025-07-27-02-16-53.gh-issue-137134.W0WpDF.rst new file mode 100644 index 00000000000000..ddccf95b7d039a --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2025-07-27-02-16-53.gh-issue-137134.W0WpDF.rst @@ -0,0 +1 @@ +Update Windows installer to ship with SQLite 3.50.4. diff --git a/Misc/NEWS.d/next/macOS/2025-07-27-02-17-40.gh-issue-137134.pjgITs.rst b/Misc/NEWS.d/next/macOS/2025-07-27-02-17-40.gh-issue-137134.pjgITs.rst new file mode 100644 index 00000000000000..957270f5abae93 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2025-07-27-02-17-40.gh-issue-137134.pjgITs.rst @@ -0,0 +1 @@ +Update macOS installer to ship with SQLite version 3.50.4. diff --git a/Misc/NEWS.d/next/macOS/2025-08-06-06-29-12.gh-issue-137450.JZypb7.rst b/Misc/NEWS.d/next/macOS/2025-08-06-06-29-12.gh-issue-137450.JZypb7.rst new file mode 100644 index 00000000000000..5efd74660c95d2 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2025-08-06-06-29-12.gh-issue-137450.JZypb7.rst @@ -0,0 +1,3 @@ +macOS installer shell path management improvements: separate the installer +``Shell profile updater`` postinstall script from the +``Update Shell Profile.command`` to enable more robust error handling. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 69f3beec82ed34..a87af7f9173780 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -91,21 +91,21 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "e335aeb44fa36cde60ecbb6a9f8be6f5d449d645ce9b0199ee53a7e6728d19d2" + "checksumValue": "fb5ab81f27612b0a7b4861ba655906c76dc85ee969e7a4905d2075aff931e8d0" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.49.1.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.50.4.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.49.1.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.50.4.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "sqlite", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.49.1.0" + "versionInfo": "3.50.4.0" }, { "SPDXID": "SPDXRef-PACKAGE-tcl-core", diff --git a/Modules/_ctypes/malloc_closure.c b/Modules/_ctypes/malloc_closure.c index 80ba96614bff79..db405acf8727b5 100644 --- a/Modules/_ctypes/malloc_closure.c +++ b/Modules/_ctypes/malloc_closure.c @@ -30,11 +30,6 @@ #ifdef Py_GIL_DISABLED static PyMutex malloc_closure_lock; -# define MALLOC_CLOSURE_LOCK() PyMutex_Lock(&malloc_closure_lock) -# define MALLOC_CLOSURE_UNLOCK() PyMutex_Unlock(&malloc_closure_lock) -#else -# define MALLOC_CLOSURE_LOCK() ((void)0) -# define MALLOC_CLOSURE_UNLOCK() ((void)0) #endif typedef union _tagITEM { @@ -120,11 +115,11 @@ void Py_ffi_closure_free(void *p) } #endif #endif - MALLOC_CLOSURE_LOCK(); + FT_MUTEX_LOCK(&malloc_closure_lock); ITEM *item = (ITEM *)p; item->next = free_list; free_list = item; - MALLOC_CLOSURE_UNLOCK(); + FT_MUTEX_UNLOCK(&malloc_closure_lock); } /* return one item from the free list, allocating more if needed */ @@ -143,13 +138,13 @@ void *Py_ffi_closure_alloc(size_t size, void** codeloc) } #endif #endif - MALLOC_CLOSURE_LOCK(); + FT_MUTEX_LOCK(&malloc_closure_lock); ITEM *item; if (!free_list) { more_core(); } if (!free_list) { - MALLOC_CLOSURE_UNLOCK(); + FT_MUTEX_UNLOCK(&malloc_closure_lock); return NULL; } item = free_list; @@ -160,6 +155,6 @@ void *Py_ffi_closure_alloc(size_t size, void** codeloc) #else *codeloc = (void *)item; #endif - MALLOC_CLOSURE_UNLOCK(); + FT_MUTEX_UNLOCK(&malloc_closure_lock); return (void *)item; } diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 01039dfeec0719..e0bd4993817277 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -214,7 +214,7 @@ clear_current_module(PyInterpreterState *interp, PyObject *expected) if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { goto error; } - if (ref != NULL) { + if (ref != NULL && ref != Py_None) { PyObject *current = NULL; int rc = PyWeakref_GetRef(ref, ¤t); /* We only need "current" for pointer comparison. */ @@ -331,8 +331,10 @@ class datetime.datetime "PyDateTime_DateTime *" "get_datetime_state()->datetime_ class datetime.date "PyDateTime_Date *" "get_datetime_state()->date_type" class datetime.time "PyDateTime_Time *" "get_datetime_state()->time_type" class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "get_datetime_state()->isocalendar_date_type" +class datetime.timedelta "PyDateTime_Delta *" "&PyDateTime_DeltaType" +class datetime.timezone "PyDateTime_TimeZone *" "&PyDateTime_TimeZoneType" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c8f3d834a860d50a]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c54b9adf60082f0d]*/ #include "clinic/_datetimemodule.c.h" @@ -1171,19 +1173,18 @@ new_datetime_ex(int, int, int, int, int, int, int, PyObject *, PyTypeObject *); /* Create date instance with no range checking, or call subclass constructor */ static PyObject * -new_date_subclass_ex(int year, int month, int day, PyObject *cls) +new_date_subclass_ex(int year, int month, int day, PyTypeObject *cls) { PyObject *result; // We have "fast path" constructors for two subclasses: date and datetime - if ((PyTypeObject *)cls == DATE_TYPE(NO_STATE)) { - result = new_date_ex(year, month, day, (PyTypeObject *)cls); + if (cls == DATE_TYPE(NO_STATE)) { + result = new_date_ex(year, month, day, cls); } - else if ((PyTypeObject *)cls == DATETIME_TYPE(NO_STATE)) { - result = new_datetime_ex(year, month, day, 0, 0, 0, 0, Py_None, - (PyTypeObject *)cls); + else if (cls == DATETIME_TYPE(NO_STATE)) { + result = new_datetime_ex(year, month, day, 0, 0, 0, 0, Py_None, cls); } else { - result = PyObject_CallFunction(cls, "iii", year, month, day); + result = PyObject_CallFunction((PyObject *)cls, "iii", year, month, day); } return result; @@ -1235,7 +1236,7 @@ new_datetime_ex(int year, int month, int day, int hour, int minute, new_datetime_ex2(y, m, d, hh, mm, ss, us, tzinfo, fold, DATETIME_TYPE(NO_STATE)) static PyObject * -call_subclass_fold(PyObject *cls, int fold, const char *format, ...) +call_subclass_fold(PyTypeObject *cls, int fold, const char *format, ...) { PyObject *kwargs = NULL, *res = NULL; va_list va; @@ -1261,7 +1262,7 @@ call_subclass_fold(PyObject *cls, int fold, const char *format, ...) goto Done; } } - res = PyObject_Call(cls, args, kwargs); + res = PyObject_Call((PyObject *)cls, args, kwargs); Done: Py_DECREF(args); Py_XDECREF(kwargs); @@ -1271,10 +1272,10 @@ call_subclass_fold(PyObject *cls, int fold, const char *format, ...) static PyObject * new_datetime_subclass_fold_ex(int year, int month, int day, int hour, int minute, int second, int usecond, PyObject *tzinfo, - int fold, PyObject *cls) + int fold, PyTypeObject *cls) { PyObject* dt; - if ((PyTypeObject*)cls == DATETIME_TYPE(NO_STATE)) { + if (cls == DATETIME_TYPE(NO_STATE)) { // Use the fast path constructor dt = new_datetime(year, month, day, hour, minute, second, usecond, tzinfo, fold); @@ -1291,7 +1292,7 @@ new_datetime_subclass_fold_ex(int year, int month, int day, int hour, int minute static PyObject * new_datetime_subclass_ex(int year, int month, int day, int hour, int minute, int second, int usecond, PyObject *tzinfo, - PyObject *cls) { + PyTypeObject *cls) { return new_datetime_subclass_fold_ex(year, month, day, hour, minute, second, usecond, tzinfo, 0, cls); @@ -1340,10 +1341,10 @@ new_time_ex(int hour, int minute, int second, int usecond, static PyObject * new_time_subclass_fold_ex(int hour, int minute, int second, int usecond, - PyObject *tzinfo, int fold, PyObject *cls) + PyObject *tzinfo, int fold, PyTypeObject *cls) { PyObject *t; - if ((PyTypeObject*)cls == TIME_TYPE(NO_STATE)) { + if (cls == TIME_TYPE(NO_STATE)) { // Use the fast path constructor t = new_time(hour, minute, second, usecond, tzinfo, fold); } @@ -2768,38 +2769,39 @@ accum(const char* tag, PyObject *sofar, PyObject *num, PyObject *factor, return NULL; } +/*[clinic input] +@classmethod +datetime.timedelta.__new__ as delta_new + + days: object(c_default="NULL") = 0 + seconds: object(c_default="NULL") = 0 + microseconds: object(c_default="NULL") = 0 + milliseconds: object(c_default="NULL") = 0 + minutes: object(c_default="NULL") = 0 + hours: object(c_default="NULL") = 0 + weeks: object(c_default="NULL") = 0 + +Difference between two datetime values. + +All arguments are optional and default to 0. +Arguments may be integers or floats, and may be positive or negative. +[clinic start generated code]*/ + static PyObject * -delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) +delta_new_impl(PyTypeObject *type, PyObject *days, PyObject *seconds, + PyObject *microseconds, PyObject *milliseconds, + PyObject *minutes, PyObject *hours, PyObject *weeks) +/*[clinic end generated code: output=61d7e02a92a97700 input=e8cd54819295d34b]*/ { PyObject *self = NULL; PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); - /* Argument objects. */ - PyObject *day = NULL; - PyObject *second = NULL; - PyObject *us = NULL; - PyObject *ms = NULL; - PyObject *minute = NULL; - PyObject *hour = NULL; - PyObject *week = NULL; - PyObject *x = NULL; /* running sum of microseconds */ PyObject *y = NULL; /* temp sum of microseconds */ double leftover_us = 0.0; - static char *keywords[] = { - "days", "seconds", "microseconds", "milliseconds", - "minutes", "hours", "weeks", NULL - }; - - if (PyArg_ParseTupleAndKeywords(args, kw, "|OOOOOOO:__new__", - keywords, - &day, &second, &us, - &ms, &minute, &hour, &week) == 0) - goto Done; - x = PyLong_FromLong(0); if (x == NULL) goto Done; @@ -2810,32 +2812,32 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (x == NULL) \ goto Done - if (us) { - y = accum("microseconds", x, us, _PyLong_GetOne(), &leftover_us); + if (microseconds) { + y = accum("microseconds", x, microseconds, _PyLong_GetOne(), &leftover_us); CLEANUP; } - if (ms) { - y = accum("milliseconds", x, ms, CONST_US_PER_MS(st), &leftover_us); + if (milliseconds) { + y = accum("milliseconds", x, milliseconds, CONST_US_PER_MS(st), &leftover_us); CLEANUP; } - if (second) { - y = accum("seconds", x, second, CONST_US_PER_SECOND(st), &leftover_us); + if (seconds) { + y = accum("seconds", x, seconds, CONST_US_PER_SECOND(st), &leftover_us); CLEANUP; } - if (minute) { - y = accum("minutes", x, minute, CONST_US_PER_MINUTE(st), &leftover_us); + if (minutes) { + y = accum("minutes", x, minutes, CONST_US_PER_MINUTE(st), &leftover_us); CLEANUP; } - if (hour) { - y = accum("hours", x, hour, CONST_US_PER_HOUR(st), &leftover_us); + if (hours) { + y = accum("hours", x, hours, CONST_US_PER_HOUR(st), &leftover_us); CLEANUP; } - if (day) { - y = accum("days", x, day, CONST_US_PER_DAY(st), &leftover_us); + if (days) { + y = accum("days", x, days, CONST_US_PER_DAY(st), &leftover_us); CLEANUP; } - if (week) { - y = accum("weeks", x, week, CONST_US_PER_WEEK(st), &leftover_us); + if (weeks) { + y = accum("weeks", x, weeks, CONST_US_PER_WEEK(st), &leftover_us); CLEANUP; } if (leftover_us) { @@ -3034,13 +3036,6 @@ static PyMethodDef delta_methods[] = { {NULL, NULL}, }; -static const char delta_doc[] = -PyDoc_STR("Difference between two datetime values.\n\n" - "timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, " - "minutes=0, hours=0, weeks=0)\n\n" - "All arguments are optional and default to 0.\n" - "Arguments may be integers or floats, and may be positive or negative."); - static PyNumberMethods delta_as_number = { delta_add, /* nb_add */ delta_subtract, /* nb_subtract */ @@ -3098,7 +3093,7 @@ static PyTypeObject PyDateTime_DeltaType = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - delta_doc, /* tp_doc */ + delta_new__doc__, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ delta_richcompare, /* tp_richcompare */ @@ -3174,8 +3169,6 @@ static PyGetSetDef date_getset[] = { /* Constructors. */ -static char *date_kws[] = {"year", "month", "day", NULL}; - static PyObject * date_from_pickle(PyTypeObject *type, PyObject *state) { @@ -3193,11 +3186,6 @@ date_from_pickle(PyTypeObject *type, PyObject *state) static PyObject * date_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - PyObject *self = NULL; - int year; - int month; - int day; - /* Check for invocation from pickle with __getstate__ state */ if (PyTuple_GET_SIZE(args) == 1) { PyObject *state = PyTuple_GET_ITEM(args, 0); @@ -3223,22 +3211,36 @@ date_new(PyTypeObject *type, PyObject *args, PyObject *kw) } return NULL; } - self = date_from_pickle(type, state); + PyObject *self = date_from_pickle(type, state); Py_DECREF(state); return self; } } } - if (PyArg_ParseTupleAndKeywords(args, kw, "iii", date_kws, - &year, &month, &day)) { - self = new_date_ex(year, month, day, type); - } - return self; + return datetime_date(type, args, kw); +} + +/*[clinic input] +@classmethod +datetime.date.__new__ + + year: int + month: int + day: int + +Concrete date type. +[clinic start generated code]*/ + +static PyObject * +datetime_date_impl(PyTypeObject *type, int year, int month, int day) +/*[clinic end generated code: output=6654caa3dea7d518 input=fd1bac0658690455]*/ +{ + return new_date_ex(year, month, day, type); } static PyObject * -date_fromtimestamp(PyObject *cls, PyObject *obj) +date_fromtimestamp(PyTypeObject *cls, PyObject *obj) { struct tm tm; time_t t; @@ -3260,8 +3262,18 @@ date_fromtimestamp(PyObject *cls, PyObject *obj) * only way to be sure of that is to *call* time.time(). That's not * generally the same as calling C's time. */ +/*[clinic input] +@classmethod +datetime.date.today + +Current date or datetime. + +Equivalent to fromtimestamp(time.time()). +[clinic start generated code]*/ + static PyObject * -date_today(PyObject *cls, PyObject *Py_UNUSED(dummy)) +datetime_date_today_impl(PyTypeObject *type) +/*[clinic end generated code: output=d5474697df6b251c input=21688afa289c0a06]*/ { PyObject *time; PyObject *result; @@ -3275,7 +3287,7 @@ date_today(PyObject *cls, PyObject *Py_UNUSED(dummy)) * time.time() delivers; if someone were gonzo about optimization, * date.today() could get away with plain C time(). */ - result = PyObject_CallMethodOneArg(cls, &_Py_ID(fromtimestamp), time); + result = PyObject_CallMethodOneArg((PyObject*)type, &_Py_ID(fromtimestamp), time); Py_DECREF(time); return result; } @@ -3297,7 +3309,7 @@ static PyObject * datetime_date_fromtimestamp_impl(PyTypeObject *type, PyObject *timestamp) /*[clinic end generated code: output=59def4e32c028fb6 input=eabb3fe7f40491fe]*/ { - return date_fromtimestamp((PyObject *) type, timestamp); + return date_fromtimestamp(type, timestamp); } /* bpo-36025: This is a wrapper for API compatibility with the public C API, @@ -3311,52 +3323,58 @@ datetime_date_fromtimestamp_capi(PyObject *cls, PyObject *args) PyObject *result = NULL; if (PyArg_UnpackTuple(args, "fromtimestamp", 1, 1, ×tamp)) { - result = date_fromtimestamp(cls, timestamp); + result = date_fromtimestamp((PyTypeObject *)cls, timestamp); } return result; } -/* Return new date from proleptic Gregorian ordinal. Raises ValueError if - * the ordinal is out of range. - */ -static PyObject * -date_fromordinal(PyObject *cls, PyObject *args) -{ - PyObject *result = NULL; - int ordinal; +/*[clinic input] +@classmethod +datetime.date.fromordinal - if (PyArg_ParseTuple(args, "i:fromordinal", &ordinal)) { - int year; - int month; - int day; + ordinal: int + / - if (ordinal < 1) - PyErr_SetString(PyExc_ValueError, "ordinal must be " - ">= 1"); - else { - ord_to_ymd(ordinal, &year, &month, &day); - result = new_date_subclass_ex(year, month, day, cls); - } - } - return result; -} +Construct a date from a proleptic Gregorian ordinal. + +January 1 of year 1 is day 1. Only the year, month and day are +non-zero in the result. +[clinic start generated code]*/ -/* Return the new date from a string as generated by date.isoformat() */ static PyObject * -date_fromisoformat(PyObject *cls, PyObject *dtstr) +datetime_date_fromordinal_impl(PyTypeObject *type, int ordinal) +/*[clinic end generated code: output=ea5cc69d86614a6b input=a3a4eedf582f145e]*/ { - assert(dtstr != NULL); + int year; + int month; + int day; - if (!PyUnicode_Check(dtstr)) { - PyErr_SetString(PyExc_TypeError, - "fromisoformat: argument must be str"); + if (ordinal < 1) { + PyErr_SetString(PyExc_ValueError, "ordinal must be >= 1"); return NULL; } + ord_to_ymd(ordinal, &year, &month, &day); + return new_date_subclass_ex(year, month, day, type); +} + +/*[clinic input] +@classmethod +datetime.date.fromisoformat + string: unicode + / + +Construct a date from a string in ISO 8601 format. +[clinic start generated code]*/ + +static PyObject * +datetime_date_fromisoformat_impl(PyTypeObject *type, PyObject *string) +/*[clinic end generated code: output=8b9f9324904fca02 input=73c64216c10bcc8e]*/ +{ Py_ssize_t len; - const char *dt_ptr = PyUnicode_AsUTF8AndSize(dtstr, &len); + const char *dt_ptr = PyUnicode_AsUTF8AndSize(string, &len); if (dt_ptr == NULL) { goto invalid_string_error; } @@ -3375,33 +3393,32 @@ date_fromisoformat(PyObject *cls, PyObject *dtstr) goto invalid_string_error; } - return new_date_subclass_ex(year, month, day, cls); + return new_date_subclass_ex(year, month, day, type); invalid_string_error: - PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", dtstr); + PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", string); return NULL; } -static PyObject * -date_fromisocalendar(PyObject *cls, PyObject *args, PyObject *kw) -{ - static char *keywords[] = { - "year", "week", "day", NULL - }; +/*[clinic input] +@classmethod +datetime.date.fromisocalendar - int year, week, day; - if (PyArg_ParseTupleAndKeywords(args, kw, "iii:fromisocalendar", - keywords, - &year, &week, &day) == 0) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) { - PyErr_Format(PyExc_ValueError, - "ISO calendar component out of range"); + year: int + week: int + day: int - } - return NULL; - } +Construct a date from the ISO year, week number and weekday. + +This is the inverse of the date.isocalendar() function. +[clinic start generated code]*/ +static PyObject * +datetime_date_fromisocalendar_impl(PyTypeObject *type, int year, int week, + int day) +/*[clinic end generated code: output=7b26e15115d24df6 input=fbb05b53d6fb51d8]*/ +{ int month; int rv = iso_to_ymd(year, week, day, &year, &month, &day); @@ -3422,26 +3439,34 @@ date_fromisocalendar(PyObject *cls, PyObject *args, PyObject *kw) return NULL; } - return new_date_subclass_ex(year, month, day, cls); + return new_date_subclass_ex(year, month, day, type); } -/* Return new date from _strptime.strptime_datetime_date(). */ +/*[clinic input] +@classmethod +datetime.date.strptime + + string: unicode + format: unicode + / + +Parse string according to the given date format (like time.strptime()). +[clinic start generated code]*/ + static PyObject * -date_strptime(PyObject *cls, PyObject *args) +datetime_date_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format) +/*[clinic end generated code: output=454d473bee2d5161 input=001904ab34f594a1]*/ { - PyObject *string, *format, *result; - - if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) { - return NULL; - } + PyObject *result; PyObject *module = PyImport_Import(&_Py_ID(_strptime)); if (module == NULL) { return NULL; } result = PyObject_CallMethodObjArgs(module, - &_Py_ID(_strptime_datetime_date), cls, - string, format, NULL); + &_Py_ID(_strptime_datetime_date), + (PyObject *)type, string, format, NULL); Py_DECREF(module); return result; } @@ -3465,8 +3490,7 @@ add_date_timedelta(PyDateTime_Date *date, PyDateTime_Delta *delta, int negate) int day = GET_DAY(date) + (negate ? -deltadays : deltadays); if (normalize_date(&year, &month, &day) >= 0) - result = new_date_subclass_ex(year, month, day, - (PyObject* )Py_TYPE(date)); + result = new_date_subclass_ex(year, month, day, Py_TYPE(date)); return result; } @@ -3558,20 +3582,26 @@ date_ctime(PyObject *self, PyObject *Py_UNUSED(dummy)) return format_ctime(self, 0, 0, 0); } +/*[clinic input] +datetime.date.strftime + + self: self(type="PyObject *") + format: unicode + +Format using strftime(). + +Example: "%d/%m/%Y, %H:%M:%S". +[clinic start generated code]*/ + static PyObject * -date_strftime(PyObject *self, PyObject *args, PyObject *kw) +datetime_date_strftime_impl(PyObject *self, PyObject *format) +/*[clinic end generated code: output=6529b70095e16778 input=72af55077e606ed8]*/ { /* This method can be inherited, and needs to call the * timetuple() method appropriate to self's class. */ PyObject *result; PyObject *tuple; - PyObject *format; - static char *keywords[] = {"format", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kw, "U:strftime", keywords, - &format)) - return NULL; tuple = PyObject_CallMethodNoArgs(self, &_Py_ID(timetuple)); if (tuple == NULL) @@ -3581,14 +3611,20 @@ date_strftime(PyObject *self, PyObject *args, PyObject *kw) return result; } -static PyObject * -date_format(PyObject *self, PyObject *args) -{ - PyObject *format; +/*[clinic input] +datetime.date.__format__ - if (!PyArg_ParseTuple(args, "U:__format__", &format)) - return NULL; + self: self(type="PyObject *") + format: unicode + / + +Formats self with strftime. +[clinic start generated code]*/ +static PyObject * +datetime_date___format___impl(PyObject *self, PyObject *format) +/*[clinic end generated code: output=efa0223d000a93b7 input=e417a7c84e1abaf9]*/ +{ /* if the format is zero length, return str(self) */ if (PyUnicode_GetLength(format) == 0) return PyObject_Str(self); @@ -3834,7 +3870,7 @@ datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, int day) /*[clinic end generated code: output=2a9430d1e6318aeb input=0d1f02685b3e90f6]*/ { - return new_date_subclass_ex(year, month, day, (PyObject *)Py_TYPE(self)); + return new_date_subclass_ex(year, month, day, Py_TYPE(self)); } static Py_hash_t @@ -3895,38 +3931,19 @@ static PyMethodDef date_methods[] = { /* Class methods: */ DATETIME_DATE_FROMTIMESTAMP_METHODDEF - - {"fromordinal", date_fromordinal, METH_VARARGS | METH_CLASS, - PyDoc_STR("int -> date corresponding to a proleptic Gregorian " - "ordinal.")}, - - {"fromisoformat", date_fromisoformat, METH_O | METH_CLASS, - PyDoc_STR("str -> Construct a date from a string in ISO 8601 format.")}, - - {"fromisocalendar", _PyCFunction_CAST(date_fromisocalendar), - METH_VARARGS | METH_KEYWORDS | METH_CLASS, - PyDoc_STR("int, int, int -> Construct a date from the ISO year, week " - "number and weekday.\n\n" - "This is the inverse of the date.isocalendar() function")}, - - {"strptime", date_strptime, METH_VARARGS | METH_CLASS, - PyDoc_STR("string, format -> new date parsed from a string " - "(like time.strptime()).")}, - - {"today", date_today, METH_NOARGS | METH_CLASS, - PyDoc_STR("Current date or datetime: same as " - "self.__class__.fromtimestamp(time.time()).")}, + DATETIME_DATE_FROMORDINAL_METHODDEF + DATETIME_DATE_FROMISOFORMAT_METHODDEF + DATETIME_DATE_FROMISOCALENDAR_METHODDEF + DATETIME_DATE_STRPTIME_METHODDEF + DATETIME_DATE_TODAY_METHODDEF /* Instance methods: */ {"ctime", date_ctime, METH_NOARGS, PyDoc_STR("Return ctime() style string.")}, - {"strftime", _PyCFunction_CAST(date_strftime), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("format -> strftime() style string.")}, - - {"__format__", date_format, METH_VARARGS, - PyDoc_STR("Formats self with strftime.")}, + DATETIME_DATE_STRFTIME_METHODDEF + DATETIME_DATE___FORMAT___METHODDEF {"timetuple", date_timetuple, METH_NOARGS, PyDoc_STR("Return time tuple, compatible with time.localtime().")}, @@ -3961,9 +3978,6 @@ static PyMethodDef date_methods[] = { {NULL, NULL} }; -static const char date_doc[] = -PyDoc_STR("date(year, month, day) --> date object"); - static PyNumberMethods date_as_number = { date_add, /* nb_add */ date_subtract, /* nb_subtract */ @@ -3998,7 +4012,7 @@ static PyTypeObject PyDateTime_DateType = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - date_doc, /* tp_doc */ + datetime_date__doc__, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ date_richcompare, /* tp_richcompare */ @@ -4201,7 +4215,8 @@ static PyMethodDef tzinfo_methods[] = { }; static const char tzinfo_doc[] = -PyDoc_STR("Abstract base class for time zone info objects."); +PyDoc_STR("Abstract base class for time zone info objects.\n\n" + "Subclasses must override the tzname(), utcoffset() and dst() methods."); static PyTypeObject PyDateTime_TZInfoType = { PyVarObject_HEAD_INIT(NULL, 0) @@ -4245,18 +4260,21 @@ static PyTypeObject PyDateTime_TZInfoType = { 0, /* tp_free */ }; -static char *timezone_kws[] = {"offset", "name", NULL}; +/*[clinic input] +@classmethod +datetime.timezone.__new__ as timezone_new + + offset: object(subclass_of="DELTA_TYPE(NO_STATE)") + name: unicode = NULL + +Fixed offset from UTC implementation of tzinfo. +[clinic start generated code]*/ static PyObject * -timezone_new(PyTypeObject *type, PyObject *args, PyObject *kw) +timezone_new_impl(PyTypeObject *type, PyObject *offset, PyObject *name) +/*[clinic end generated code: output=41a2dda500424187 input=d51255afe60382cd]*/ { - PyObject *offset; - PyObject *name = NULL; - if (PyArg_ParseTupleAndKeywords(args, kw, "O!|U:timezone", timezone_kws, - DELTA_TYPE(NO_STATE), &offset, &name)) - return new_timezone(offset, name); - - return NULL; + return new_timezone(offset, name); } static void @@ -4444,9 +4462,6 @@ static PyMethodDef timezone_methods[] = { {NULL, NULL} }; -static const char timezone_doc[] = -PyDoc_STR("Fixed offset from UTC implementation of tzinfo."); - static PyTypeObject PyDateTime_TimeZoneType = { PyVarObject_HEAD_INIT(NULL, 0) "datetime.timezone", /* tp_name */ @@ -4468,7 +4483,7 @@ static PyTypeObject PyDateTime_TimeZoneType = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT, /* tp_flags */ - timezone_doc, /* tp_doc */ + timezone_new__doc__, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ timezone_richcompare, /* tp_richcompare */ @@ -4570,9 +4585,6 @@ static PyGetSetDef time_getset[] = { * Constructors. */ -static char *time_kws[] = {"hour", "minute", "second", "microsecond", - "tzinfo", "fold", NULL}; - static PyObject * time_from_pickle(PyTypeObject *type, PyObject *state, PyObject *tzinfo) { @@ -4608,17 +4620,10 @@ time_from_pickle(PyTypeObject *type, PyObject *state, PyObject *tzinfo) static PyObject * time_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - PyObject *self = NULL; - int hour = 0; - int minute = 0; - int second = 0; - int usecond = 0; - PyObject *tzinfo = Py_None; - int fold = 0; - /* Check for invocation from pickle with __getstate__ state */ if (PyTuple_GET_SIZE(args) >= 1 && PyTuple_GET_SIZE(args) <= 2) { PyObject *state = PyTuple_GET_ITEM(args, 0); + PyObject *tzinfo = Py_None; if (PyTuple_GET_SIZE(args) == 2) { tzinfo = PyTuple_GET_ITEM(args, 1); } @@ -4644,40 +4649,67 @@ time_new(PyTypeObject *type, PyObject *args, PyObject *kw) } return NULL; } - self = time_from_pickle(type, state, tzinfo); + PyObject *self = time_from_pickle(type, state, tzinfo); Py_DECREF(state); return self; } } - tzinfo = Py_None; } - if (PyArg_ParseTupleAndKeywords(args, kw, "|iiiiO$i", time_kws, - &hour, &minute, &second, &usecond, - &tzinfo, &fold)) { - self = new_time_ex2(hour, minute, second, usecond, tzinfo, fold, - type); - } - return self; + return datetime_time(type, args, kw); } -/* Return new time from _strptime.strptime_datetime_time(). */ +/*[clinic input] +@classmethod +datetime.time.__new__ + + hour: int = 0 + minute: int = 0 + second: int = 0 + microsecond: int = 0 + tzinfo: object = None + * + fold: int = 0 + +Time with time zone. + +All arguments are optional. tzinfo may be None, or an instance of +a tzinfo subclass. The remaining arguments may be ints. +[clinic start generated code]*/ + static PyObject * -time_strptime(PyObject *cls, PyObject *args) +datetime_time_impl(PyTypeObject *type, int hour, int minute, int second, + int microsecond, PyObject *tzinfo, int fold) +/*[clinic end generated code: output=f06bb4315225e7f6 input=0148df5e8138fe7b]*/ { - PyObject *string, *format, *result; + return new_time_ex2(hour, minute, second, microsecond, tzinfo, fold, type); +} - if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) { - return NULL; - } +/*[clinic input] +@classmethod +datetime.time.strptime + + string: unicode + format: unicode + / + +Parse string according to the given time format (like time.strptime()). +[clinic start generated code]*/ + +static PyObject * +datetime_time_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format) +/*[clinic end generated code: output=ae05a9bc0241d3bf input=6d0f263a5f94d78d]*/ +{ + PyObject *result; PyObject *module = PyImport_Import(&_Py_ID(_strptime)); if (module == NULL) { return NULL; } result = PyObject_CallMethodObjArgs(module, - &_Py_ID(_strptime_datetime_time), cls, - string, format, NULL); + &_Py_ID(_strptime_datetime_time), + (PyObject *)type, string, format, NULL); Py_DECREF(module); return result; } @@ -4756,17 +4788,30 @@ time_str(PyObject *op) return PyObject_CallMethodNoArgs(op, &_Py_ID(isoformat)); } +/*[clinic input] +datetime.time.isoformat + + timespec: str(c_default="NULL") = 'auto' + +Return the time formatted according to ISO. + +The full format is 'HH:MM:SS.mmmmmm+zz:zz'. By default, the fractional +part is omitted if self.microsecond == 0. + +The optional argument timespec specifies the number of additional +terms of the time to include. Valid options are 'auto', 'hours', +'minutes', 'seconds', 'milliseconds' and 'microseconds'. +[clinic start generated code]*/ + static PyObject * -time_isoformat(PyObject *op, PyObject *args, PyObject *kw) +datetime_time_isoformat_impl(PyDateTime_Time *self, const char *timespec) +/*[clinic end generated code: output=2bcc7cab65c35545 input=afbbbd953d10ad07]*/ { char buf[100]; - const char *timespec = NULL; - static char *keywords[] = {"timespec", NULL}; - PyDateTime_Time *self = PyTime_CAST(op); PyObject *result; int us = TIME_GET_MICROSECOND(self); - static const char *specs[][2] = { + static const char * const specs[][2] = { {"hours", "%02d"}, {"minutes", "%02d:%02d"}, {"seconds", "%02d:%02d:%02d"}, @@ -4775,9 +4820,6 @@ time_isoformat(PyObject *op, PyObject *args, PyObject *kw) }; size_t given_spec; - if (!PyArg_ParseTupleAndKeywords(args, kw, "|s:isoformat", keywords, ×pec)) - return NULL; - if (timespec == NULL || strcmp(timespec, "auto") == 0) { if (us == 0) { /* seconds */ @@ -4823,18 +4865,22 @@ time_isoformat(PyObject *op, PyObject *args, PyObject *kw) return result; } +/*[clinic input] +datetime.time.strftime + + format: unicode + +Format using strftime(). + +The date part of the timestamp passed to underlying strftime should not be used. +[clinic start generated code]*/ + static PyObject * -time_strftime(PyObject *op, PyObject *args, PyObject *kw) +datetime_time_strftime_impl(PyDateTime_Time *self, PyObject *format) +/*[clinic end generated code: output=10f65af20e2a78c7 input=7dd9df1acbf37b50]*/ { PyObject *result; PyObject *tuple; - PyObject *format; - static char *keywords[] = {"format", NULL}; - PyDateTime_Time *self = PyTime_CAST(op); - - if (! PyArg_ParseTupleAndKeywords(args, kw, "U:strftime", keywords, - &format)) - return NULL; /* Python's strftime does insane things with the year part of the * timetuple. The year is forced to (the otherwise nonsensical) @@ -4855,6 +4901,27 @@ time_strftime(PyObject *op, PyObject *args, PyObject *kw) return result; } +/*[clinic input] +datetime.time.__format__ + + self: self(type="PyObject *") + format: unicode + / + +Formats self with strftime. +[clinic start generated code]*/ + +static PyObject * +datetime_time___format___impl(PyObject *self, PyObject *format) +/*[clinic end generated code: output=4646451f7a5d2156 input=6a858ae787d20230]*/ +{ + /* if the format is zero length, return str(self) */ + if (PyUnicode_GetLength(format) == 0) + return PyObject_Str(self); + + return PyObject_CallMethodOneArg(self, &_Py_ID(strftime), format); +} + /* * Miscellaneous methods. */ @@ -5007,20 +5074,25 @@ datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, /*[clinic end generated code: output=0b89a44c299e4f80 input=abf23656e8df4e97]*/ { return new_time_subclass_fold_ex(hour, minute, second, microsecond, tzinfo, - fold, (PyObject *)Py_TYPE(self)); + fold, Py_TYPE(self)); } -static PyObject * -time_fromisoformat(PyObject *cls, PyObject *tstr) { - assert(tstr != NULL); +/*[clinic input] +@classmethod +datetime.time.fromisoformat - if (!PyUnicode_Check(tstr)) { - PyErr_SetString(PyExc_TypeError, "fromisoformat: argument must be str"); - return NULL; - } + string: unicode + / + +Construct a time from a string in ISO 8601 format. +[clinic start generated code]*/ +static PyObject * +datetime_time_fromisoformat_impl(PyTypeObject *type, PyObject *string) +/*[clinic end generated code: output=97c57e896e7f2535 input=bdb4b8abea9cd688]*/ +{ Py_ssize_t len; - const char *p = PyUnicode_AsUTF8AndSize(tstr, &len); + const char *p = PyUnicode_AsUTF8AndSize(string, &len); if (p == NULL) { goto invalid_string_error; @@ -5063,10 +5135,10 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { } PyObject *t; - if ( (PyTypeObject *)cls == TIME_TYPE(NO_STATE)) { + if (type == TIME_TYPE(NO_STATE)) { t = new_time(hour, minute, second, microsecond, tzinfo, 0); } else { - t = PyObject_CallFunction(cls, "iiiiO", + t = PyObject_CallFunction((PyObject *)type, "iiiiO", hour, minute, second, microsecond, tzinfo); } @@ -5078,7 +5150,7 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { return NULL; invalid_string_error: - PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", tstr); + PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", string); return NULL; error: @@ -5114,21 +5186,28 @@ time_getstate(PyDateTime_Time *self, int proto) return result; } +/*[clinic input] +datetime.time.__reduce_ex__ + + proto: int + / +[clinic start generated code]*/ + static PyObject * -time_reduce_ex(PyObject *op, PyObject *args) +datetime_time___reduce_ex___impl(PyDateTime_Time *self, int proto) +/*[clinic end generated code: output=ccfab65f5c320c1b input=4cd06bb3ac3657bb]*/ { - int proto; - if (!PyArg_ParseTuple(args, "i:__reduce_ex__", &proto)) - return NULL; - - PyDateTime_Time *self = PyTime_CAST(op); return Py_BuildValue("(ON)", Py_TYPE(self), time_getstate(self, proto)); } +/*[clinic input] +datetime.time.__reduce__ +[clinic start generated code]*/ + static PyObject * -time_reduce(PyObject *op, PyObject *Py_UNUSED(dummy)) +datetime_time___reduce___impl(PyDateTime_Time *self) +/*[clinic end generated code: output=9a2fcc87e64ce300 input=0fb8dd14d275857f]*/ { - PyDateTime_Time *self = PyTime_CAST(op); return Py_BuildValue("(ON)", Py_TYPE(self), time_getstate(self, 2)); } @@ -5136,26 +5215,14 @@ static PyMethodDef time_methods[] = { /* Class method: */ - {"strptime", time_strptime, - METH_VARARGS | METH_CLASS, - PyDoc_STR("string, format -> new time parsed from a string " - "(like time.strptime()).")}, + DATETIME_TIME_FROMISOFORMAT_METHODDEF + DATETIME_TIME_STRPTIME_METHODDEF /* Instance methods: */ - {"isoformat", _PyCFunction_CAST(time_isoformat), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("Return string in ISO 8601 format, [HH[:MM[:SS[.mmm[uuu]]]]]" - "[+HH:MM].\n\n" - "The optional argument timespec specifies the number " - "of additional terms\nof the time to include. Valid " - "options are 'auto', 'hours', 'minutes',\n'seconds', " - "'milliseconds' and 'microseconds'.\n")}, - - {"strftime", _PyCFunction_CAST(time_strftime), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("format -> strftime() style string.")}, - - {"__format__", date_format, METH_VARARGS, - PyDoc_STR("Formats self with strftime.")}, + DATETIME_TIME_ISOFORMAT_METHODDEF + DATETIME_TIME_STRFTIME_METHODDEF + DATETIME_TIME___FORMAT___METHODDEF {"utcoffset", time_utcoffset, METH_NOARGS, PyDoc_STR("Return self.tzinfo.utcoffset(self).")}, @@ -5171,24 +5238,12 @@ static PyMethodDef time_methods[] = { {"__replace__", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, - {"fromisoformat", time_fromisoformat, METH_O | METH_CLASS, - PyDoc_STR("string -> time from a string in ISO 8601 format")}, - - {"__reduce_ex__", time_reduce_ex, METH_VARARGS, - PyDoc_STR("__reduce_ex__(proto) -> (cls, state)")}, - - {"__reduce__", time_reduce, METH_NOARGS, - PyDoc_STR("__reduce__() -> (cls, state)")}, + DATETIME_TIME___REDUCE_EX___METHODDEF + DATETIME_TIME___REDUCE___METHODDEF {NULL, NULL} }; -static const char time_doc[] = -PyDoc_STR("time([hour[, minute[, second[, microsecond[, tzinfo]]]]]) --> a time object\n\ -\n\ -All arguments are optional. tzinfo may be None, or an instance of\n\ -a tzinfo subclass. The remaining arguments may be ints.\n"); - static PyTypeObject PyDateTime_TimeType = { PyVarObject_HEAD_INIT(NULL, 0) "datetime.time", /* tp_name */ @@ -5209,8 +5264,8 @@ static PyTypeObject PyDateTime_TimeType = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - time_doc, /* tp_doc */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + datetime_time__doc__, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ time_richcompare, /* tp_richcompare */ @@ -5296,11 +5351,6 @@ static PyGetSetDef datetime_getset[] = { * Constructors. */ -static char *datetime_kws[] = { - "year", "month", "day", "hour", "minute", "second", - "microsecond", "tzinfo", "fold", NULL -}; - static PyObject * datetime_from_pickle(PyTypeObject *type, PyObject *state, PyObject *tzinfo) { @@ -5336,20 +5386,10 @@ datetime_from_pickle(PyTypeObject *type, PyObject *state, PyObject *tzinfo) static PyObject * datetime_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - PyObject *self = NULL; - int year; - int month; - int day; - int hour = 0; - int minute = 0; - int second = 0; - int usecond = 0; - int fold = 0; - PyObject *tzinfo = Py_None; - /* Check for invocation from pickle with __getstate__ state */ if (PyTuple_GET_SIZE(args) >= 1 && PyTuple_GET_SIZE(args) <= 2) { PyObject *state = PyTuple_GET_ITEM(args, 0); + PyObject *tzinfo = Py_None; if (PyTuple_GET_SIZE(args) == 2) { tzinfo = PyTuple_GET_ITEM(args, 1); } @@ -5375,22 +5415,46 @@ datetime_new(PyTypeObject *type, PyObject *args, PyObject *kw) } return NULL; } - self = datetime_from_pickle(type, state, tzinfo); + PyObject *self = datetime_from_pickle(type, state, tzinfo); Py_DECREF(state); return self; } } - tzinfo = Py_None; } - if (PyArg_ParseTupleAndKeywords(args, kw, "iii|iiiiO$i", datetime_kws, - &year, &month, &day, &hour, &minute, - &second, &usecond, &tzinfo, &fold)) { - self = new_datetime_ex2(year, month, day, - hour, minute, second, usecond, - tzinfo, fold, type); - } - return self; + return datetime_datetime(type, args, kw); +} + +/*[clinic input] +@classmethod +datetime.datetime.__new__ + + year: int + month: int + day: int + hour: int = 0 + minute: int = 0 + second: int = 0 + microsecond: int = 0 + tzinfo: object = None + * + fold: int = 0 + +A combination of a date and a time. + +The year, month and day arguments are required. tzinfo may be None, or an +instance of a tzinfo subclass. The remaining arguments may be ints. +[clinic start generated code]*/ + +static PyObject * +datetime_datetime_impl(PyTypeObject *type, int year, int month, int day, + int hour, int minute, int second, int microsecond, + PyObject *tzinfo, int fold) +/*[clinic end generated code: output=47983ddb47d36037 input=2af468d7a9c1e568]*/ +{ + return new_datetime_ex2(year, month, day, + hour, minute, second, microsecond, + tzinfo, fold, type); } /* TM_FUNC is the shared type of _PyTime_localtime() and @@ -5447,7 +5511,7 @@ local(long long u) * Pass localtime or gmtime for f, to control the interpretation of timet. */ static PyObject * -datetime_from_timet_and_us(PyObject *cls, TM_FUNC f, time_t timet, int us, +datetime_from_timet_and_us(PyTypeObject *cls, TM_FUNC f, time_t timet, int us, PyObject *tzinfo) { struct tm tm; @@ -5519,7 +5583,7 @@ datetime_from_timet_and_us(PyObject *cls, TM_FUNC f, time_t timet, int us, * to get that much precision (e.g., C time() isn't good enough). */ static PyObject * -datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, +datetime_from_timestamp(PyTypeObject *cls, TM_FUNC f, PyObject *timestamp, PyObject *tzinfo) { time_t timet; @@ -5537,7 +5601,7 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, * gmtime for f as appropriate. */ static PyObject * -datetime_best_possible(PyObject *cls, TM_FUNC f, PyObject *tzinfo) +datetime_best_possible(PyTypeObject *cls, TM_FUNC f, PyObject *tzinfo) { PyTime_t ts; if (PyTime_Time(&ts) < 0) { @@ -5580,7 +5644,7 @@ datetime_datetime_now_impl(PyTypeObject *type, PyObject *tz) if (check_tzinfo_subclass(tz) < 0) return NULL; - self = datetime_best_possible((PyObject *)type, + self = datetime_best_possible(type, tz == Py_None ? _PyTime_localtime : _PyTime_gmtime, tz); @@ -5596,8 +5660,16 @@ datetime_datetime_now_impl(PyTypeObject *type, PyObject *tz) /* Return best possible UTC time -- this isn't constrained by the * precision of a timestamp. */ +/*[clinic input] +@classmethod +datetime.datetime.utcnow + +Return a new datetime representing UTC day and time. +[clinic start generated code]*/ + static PyObject * -datetime_utcnow(PyObject *cls, PyObject *dummy) +datetime_datetime_utcnow_impl(PyTypeObject *type) +/*[clinic end generated code: output=cfcfe71c6c916ba9 input=576eff2b222b80a1]*/ { if (PyErr_WarnEx(PyExc_DeprecationWarning, "datetime.datetime.utcnow() is deprecated and scheduled for removal in a " @@ -5606,25 +5678,32 @@ datetime_utcnow(PyObject *cls, PyObject *dummy) { return NULL; } - return datetime_best_possible(cls, _PyTime_gmtime, Py_None); + return datetime_best_possible(type, _PyTime_gmtime, Py_None); } -/* Return new local datetime from timestamp (Python timestamp -- a double). */ +/*[clinic input] +@classmethod +datetime.datetime.fromtimestamp + + timestamp: object + tz as tzinfo: object = None + +Create a datetime from a POSIX timestamp. + +The timestamp is a number, e.g. created via time.time(), that is interpreted +as local time. +[clinic start generated code]*/ + static PyObject * -datetime_fromtimestamp(PyObject *cls, PyObject *args, PyObject *kw) +datetime_datetime_fromtimestamp_impl(PyTypeObject *type, PyObject *timestamp, + PyObject *tzinfo) +/*[clinic end generated code: output=9c47ea2b2ebdaded input=34721a5facc94215]*/ { PyObject *self; - PyObject *timestamp; - PyObject *tzinfo = Py_None; - static char *keywords[] = {"timestamp", "tz", NULL}; - - if (! PyArg_ParseTupleAndKeywords(args, kw, "O|O:fromtimestamp", - keywords, ×tamp, &tzinfo)) - return NULL; if (check_tzinfo_subclass(tzinfo) < 0) return NULL; - self = datetime_from_timestamp(cls, + self = datetime_from_timestamp(type, tzinfo == Py_None ? _PyTime_localtime : _PyTime_gmtime, timestamp, @@ -5638,9 +5717,35 @@ datetime_fromtimestamp(PyObject *cls, PyObject *args, PyObject *kw) return self; } -/* Return new UTC datetime from timestamp (Python timestamp -- a double). */ +/* This is a wrapper for API compatibility with the public C API. */ +static PyObject * +datetime_datetime_fromtimestamp_capi(PyObject *cls, PyObject *args, PyObject *kw) +{ + PyObject *timestamp; + PyObject *tzinfo = Py_None; + static char *keywords[] = {"timestamp", "tz", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kw, "O|O:fromtimestamp", + keywords, ×tamp, &tzinfo)) + return NULL; + return datetime_datetime_fromtimestamp_impl((PyTypeObject *)cls, + timestamp, tzinfo); +} + +/*[clinic input] +@classmethod +datetime.datetime.utcfromtimestamp + + timestamp: object + / + +Create a naive UTC datetime from a POSIX timestamp. +[clinic start generated code]*/ + static PyObject * -datetime_utcfromtimestamp(PyObject *cls, PyObject *args) +datetime_datetime_utcfromtimestamp_impl(PyTypeObject *type, + PyObject *timestamp) +/*[clinic end generated code: output=66d0b1741d788fd2 input=13fabd4296b1c206]*/ { if (PyErr_WarnEx(PyExc_DeprecationWarning, "datetime.datetime.utcfromtimestamp() is deprecated and scheduled for removal " @@ -5649,23 +5754,27 @@ datetime_utcfromtimestamp(PyObject *cls, PyObject *args) { return NULL; } - PyObject *timestamp; - PyObject *result = NULL; - if (PyArg_ParseTuple(args, "O:utcfromtimestamp", ×tamp)) - result = datetime_from_timestamp(cls, _PyTime_gmtime, timestamp, - Py_None); - return result; + return datetime_from_timestamp(type, _PyTime_gmtime, timestamp, Py_None); } -/* Return new datetime from _strptime.strptime_datetime_datetime(). */ +/*[clinic input] +@classmethod +datetime.datetime.strptime + + string: unicode + format: unicode + / + +Parse string according to the given date and time format (like time.strptime()). +[clinic start generated code]*/ + static PyObject * -datetime_strptime(PyObject *cls, PyObject *args) +datetime_datetime_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format) +/*[clinic end generated code: output=af2c2d024f3203f5 input=b3918835524a1f22]*/ { - PyObject *string, *format, *result; - - if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) - return NULL; + PyObject *result; PyObject *module = PyImport_Import(&_Py_ID(_strptime)); if (module == NULL) { @@ -5673,42 +5782,43 @@ datetime_strptime(PyObject *cls, PyObject *args) } result = PyObject_CallMethodObjArgs(module, &_Py_ID(_strptime_datetime_datetime), - cls, string, format, NULL); + (PyObject *)type, string, format, NULL); Py_DECREF(module); return result; } -/* Return new datetime from date/datetime and time arguments. */ +/*[clinic input] +@classmethod +datetime.datetime.combine + + date: object(subclass_of="DATE_TYPE(NO_STATE)") + time: object(subclass_of="TIME_TYPE(NO_STATE)") + tzinfo: object = NULL + +Construct a datetime from a given date and a given time. +[clinic start generated code]*/ + static PyObject * -datetime_combine(PyObject *cls, PyObject *args, PyObject *kw) +datetime_datetime_combine_impl(PyTypeObject *type, PyObject *date, + PyObject *time, PyObject *tzinfo) +/*[clinic end generated code: output=a10f3cbb90f4d0aa input=4fcf0743288d0bab]*/ { - static char *keywords[] = {"date", "time", "tzinfo", NULL}; - PyObject *date; - PyObject *time; - PyObject *tzinfo = NULL; - PyObject *result = NULL; - - if (PyArg_ParseTupleAndKeywords(args, kw, "O!O!|O:combine", keywords, - DATE_TYPE(NO_STATE), &date, - TIME_TYPE(NO_STATE), &time, &tzinfo)) { - if (tzinfo == NULL) { - if (HASTZINFO(time)) - tzinfo = ((PyDateTime_Time *)time)->tzinfo; - else - tzinfo = Py_None; - } - result = new_datetime_subclass_fold_ex(GET_YEAR(date), - GET_MONTH(date), - GET_DAY(date), - TIME_GET_HOUR(time), - TIME_GET_MINUTE(time), - TIME_GET_SECOND(time), - TIME_GET_MICROSECOND(time), - tzinfo, - TIME_GET_FOLD(time), - cls); + if (tzinfo == NULL) { + if (HASTZINFO(time)) + tzinfo = ((PyDateTime_Time *)time)->tzinfo; + else + tzinfo = Py_None; } - return result; + return new_datetime_subclass_fold_ex(GET_YEAR(date), + GET_MONTH(date), + GET_DAY(date), + TIME_GET_HOUR(time), + TIME_GET_MINUTE(time), + TIME_GET_SECOND(time), + TIME_GET_MICROSECOND(time), + tzinfo, + TIME_GET_FOLD(time), + type); } static PyObject * @@ -5866,23 +5976,26 @@ _find_isoformat_datetime_separator(const char *dtstr, Py_ssize_t len) { } } -static PyObject * -datetime_fromisoformat(PyObject *cls, PyObject *dtstr) -{ - assert(dtstr != NULL); +/*[clinic input] +@classmethod +datetime.datetime.fromisoformat - if (!PyUnicode_Check(dtstr)) { - PyErr_SetString(PyExc_TypeError, - "fromisoformat: argument must be str"); - return NULL; - } + string: unicode + / +Construct a date from a string in ISO 8601 format. +[clinic start generated code]*/ + +static PyObject * +datetime_datetime_fromisoformat_impl(PyTypeObject *type, PyObject *string) +/*[clinic end generated code: output=1800a952fcab79d9 input=d517b158209ded42]*/ +{ // We only need to sanitize this string if the separator is a surrogate // character. In the situation where the separator location is ambiguous, // we don't have to sanitize it anything because that can only happen when // the separator is either '-' or a number. This should mostly be a noop // but it makes the reference counting easier if we still sanitize. - PyObject *dtstr_clean = _sanitize_isoformat_str(dtstr); + PyObject *dtstr_clean = _sanitize_isoformat_str(string); if (dtstr_clean == NULL) { goto invalid_string_error; } @@ -5970,7 +6083,7 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) } } PyObject *dt = new_datetime_subclass_ex(year, month, day, hour, minute, - second, microsecond, tzinfo, cls); + second, microsecond, tzinfo, type); Py_DECREF(tzinfo); Py_DECREF(dtstr_clean); @@ -5983,7 +6096,7 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) return NULL; invalid_string_error: - PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", dtstr); + PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", string); error: Py_XDECREF(dtstr_clean); @@ -6060,7 +6173,7 @@ add_datetime_timedelta(PyDateTime_DateTime *date, PyDateTime_Delta *delta, return new_datetime_subclass_ex(year, month, day, hour, minute, second, microsecond, HASTZINFO(date) ? date->tzinfo : Py_None, - (PyObject *)Py_TYPE(date)); + Py_TYPE(date)); } static PyObject * @@ -6222,18 +6335,38 @@ datetime_str(PyObject *op) return res; } +/*[clinic input] +datetime.datetime.isoformat + + sep: int(accept={str}, c_default="'T'", py_default="'T'") = ord('T') + timespec: str(c_default="NULL") = 'auto' + +Return the time formatted according to ISO. + +The full format looks like 'YYYY-MM-DD HH:MM:SS.mmmmmm'. +By default, the fractional part is omitted if self.microsecond == 0. + +If self.tzinfo is not None, the UTC offset is also attached, giving +a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'. + +Optional argument sep specifies the separator between date and +time, default 'T'. + +The optional argument timespec specifies the number of additional +terms of the time to include. Valid options are 'auto', 'hours', +'minutes', 'seconds', 'milliseconds' and 'microseconds'. +[clinic start generated code]*/ + static PyObject * -datetime_isoformat(PyObject *op, PyObject *args, PyObject *kw) +datetime_datetime_isoformat_impl(PyDateTime_DateTime *self, int sep, + const char *timespec) +/*[clinic end generated code: output=9b6ce1383189b0bf input=2fa2512172ccf5d5]*/ { - int sep = 'T'; - char *timespec = NULL; - static char *keywords[] = {"sep", "timespec", NULL}; char buffer[100]; - PyDateTime_DateTime *self = PyDateTime_CAST(op); PyObject *result = NULL; int us = DATE_GET_MICROSECOND(self); - static const char *specs[][2] = { + static const char * const specs[][2] = { {"hours", "%04d-%02d-%02d%c%02d"}, {"minutes", "%04d-%02d-%02d%c%02d:%02d"}, {"seconds", "%04d-%02d-%02d%c%02d:%02d:%02d"}, @@ -6242,9 +6375,6 @@ datetime_isoformat(PyObject *op, PyObject *args, PyObject *kw) }; size_t given_spec; - if (!PyArg_ParseTupleAndKeywords(args, kw, "|Cs:isoformat", keywords, &sep, ×pec)) - return NULL; - if (timespec == NULL || strcmp(timespec, "auto") == 0) { if (us == 0) { /* seconds */ @@ -6282,7 +6412,7 @@ datetime_isoformat(PyObject *op, PyObject *args, PyObject *kw) return result; /* We need to append the UTC offset. */ - if (format_utcoffset(buffer, sizeof(buffer), ":", self->tzinfo, op) < 0) { + if (format_utcoffset(buffer, sizeof(buffer), ":", self->tzinfo, (PyObject *)self) < 0) { Py_DECREF(result); return NULL; } @@ -6412,8 +6542,7 @@ datetime_richcompare(PyObject *self, PyObject *other, int op) PyDateTime_Delta *delta; assert(offset1 != offset2); /* else last "if" handled it */ - delta = (PyDateTime_Delta *)datetime_subtract((PyObject *)self, - other); + delta = (PyDateTime_Delta *)datetime_subtract(self, other); if (delta == NULL) goto done; diff = GET_TD_DAYS(delta); @@ -6537,7 +6666,7 @@ datetime_datetime_replace_impl(PyDateTime_DateTime *self, int year, { return new_datetime_subclass_fold_ex(year, month, day, hour, minute, second, microsecond, tzinfo, fold, - (PyObject *)Py_TYPE(self)); + Py_TYPE(self)); } static PyObject * @@ -6673,20 +6802,23 @@ local_timezone_from_local(PyDateTime_DateTime *local_dt) return local_timezone_from_timestamp(timestamp); } +/*[clinic input] +datetime.datetime.astimezone + + tz as tzinfo: object = None + +Convert to local time in new timezone tz. +[clinic start generated code]*/ + static PyObject * -datetime_astimezone(PyObject *op, PyObject *args, PyObject *kw) +datetime_datetime_astimezone_impl(PyDateTime_DateTime *self, + PyObject *tzinfo) +/*[clinic end generated code: output=ae2263d04e944537 input=9c675c8595009935]*/ { - PyDateTime_DateTime *self = PyDateTime_CAST(op); PyDateTime_DateTime *result; PyObject *offset; PyObject *temp; PyObject *self_tzinfo; - PyObject *tzinfo = Py_None; - static char *keywords[] = {"tz", NULL}; - - if (! PyArg_ParseTupleAndKeywords(args, kw, "|O:astimezone", keywords, - &tzinfo)) - return NULL; if (check_tzinfo_subclass(tzinfo) == -1) return NULL; @@ -6983,22 +7115,29 @@ datetime_getstate(PyDateTime_DateTime *self, int proto) return result; } +/*[clinic input] +datetime.datetime.__reduce_ex__ + + proto: int + / +[clinic start generated code]*/ + static PyObject * -datetime_reduce_ex(PyObject *op, PyObject *args) +datetime_datetime___reduce_ex___impl(PyDateTime_DateTime *self, int proto) +/*[clinic end generated code: output=53d712ce3e927735 input=bab748e49ffb30c3]*/ { - int proto; - if (!PyArg_ParseTuple(args, "i:__reduce_ex__", &proto)) - return NULL; - - PyDateTime_DateTime *self = PyDateTime_CAST(op); return Py_BuildValue("(ON)", Py_TYPE(self), datetime_getstate(self, proto)); } +/*[clinic input] +datetime.datetime.__reduce__ +[clinic start generated code]*/ + static PyObject * -datetime_reduce(PyObject *op, PyObject *Py_UNUSED(arg)) +datetime_datetime___reduce___impl(PyDateTime_DateTime *self) +/*[clinic end generated code: output=6794df9ea75666cf input=cadbbeb3bf3bf94c]*/ { - PyDateTime_DateTime *self = PyDateTime_CAST(op); return Py_BuildValue("(ON)", Py_TYPE(self), datetime_getstate(self, 2)); } @@ -7008,31 +7147,12 @@ static PyMethodDef datetime_methods[] = { /* Class methods: */ DATETIME_DATETIME_NOW_METHODDEF - - {"utcnow", datetime_utcnow, - METH_NOARGS | METH_CLASS, - PyDoc_STR("Return a new datetime representing UTC day and time.")}, - - {"fromtimestamp", _PyCFunction_CAST(datetime_fromtimestamp), - METH_VARARGS | METH_KEYWORDS | METH_CLASS, - PyDoc_STR("timestamp[, tz] -> tz's local time from POSIX timestamp.")}, - - {"utcfromtimestamp", datetime_utcfromtimestamp, - METH_VARARGS | METH_CLASS, - PyDoc_STR("Construct a naive UTC datetime from a POSIX timestamp.")}, - - {"strptime", datetime_strptime, - METH_VARARGS | METH_CLASS, - PyDoc_STR("string, format -> new datetime parsed from a string " - "(like time.strptime()).")}, - - {"combine", _PyCFunction_CAST(datetime_combine), - METH_VARARGS | METH_KEYWORDS | METH_CLASS, - PyDoc_STR("date, time -> datetime with same date and time fields")}, - - {"fromisoformat", datetime_fromisoformat, - METH_O | METH_CLASS, - PyDoc_STR("string -> datetime from a string in most ISO 8601 formats")}, + DATETIME_DATETIME_UTCNOW_METHODDEF + DATETIME_DATETIME_FROMTIMESTAMP_METHODDEF + DATETIME_DATETIME_UTCFROMTIMESTAMP_METHODDEF + DATETIME_DATETIME_STRPTIME_METHODDEF + DATETIME_DATETIME_COMBINE_METHODDEF + DATETIME_DATETIME_FROMISOFORMAT_METHODDEF /* Instance methods: */ @@ -7057,15 +7177,7 @@ static PyMethodDef datetime_methods[] = { {"utctimetuple", datetime_utctimetuple, METH_NOARGS, PyDoc_STR("Return UTC time tuple, compatible with time.localtime().")}, - {"isoformat", _PyCFunction_CAST(datetime_isoformat), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("[sep] -> string in ISO 8601 format, " - "YYYY-MM-DDT[HH[:MM[:SS[.mmm[uuu]]]]][+HH:MM].\n" - "sep is used to separate the year from the time, and " - "defaults to 'T'.\n" - "The optional argument timespec specifies the number " - "of additional terms\nof the time to include. Valid " - "options are 'auto', 'hours', 'minutes',\n'seconds', " - "'milliseconds' and 'microseconds'.\n")}, + DATETIME_DATETIME_ISOFORMAT_METHODDEF {"utcoffset", datetime_utcoffset, METH_NOARGS, PyDoc_STR("Return self.tzinfo.utcoffset(self).")}, @@ -7081,24 +7193,13 @@ static PyMethodDef datetime_methods[] = { {"__replace__", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, - {"astimezone", _PyCFunction_CAST(datetime_astimezone), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("tz -> convert to local time in new timezone tz\n")}, - - {"__reduce_ex__", datetime_reduce_ex, METH_VARARGS, - PyDoc_STR("__reduce_ex__(proto) -> (cls, state)")}, - - {"__reduce__", datetime_reduce, METH_NOARGS, - PyDoc_STR("__reduce__() -> (cls, state)")}, + DATETIME_DATETIME_ASTIMEZONE_METHODDEF + DATETIME_DATETIME___REDUCE_EX___METHODDEF + DATETIME_DATETIME___REDUCE___METHODDEF {NULL, NULL} }; -static const char datetime_doc[] = -PyDoc_STR("datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]])\n\ -\n\ -The year, month and day arguments are required. tzinfo may be None, or an\n\ -instance of a tzinfo subclass. The remaining arguments may be ints.\n"); - static PyNumberMethods datetime_as_number = { datetime_add, /* nb_add */ datetime_subtract, /* nb_subtract */ @@ -7132,8 +7233,8 @@ static PyTypeObject PyDateTime_DateTimeType = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - datetime_doc, /* tp_doc */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + datetime_datetime__doc__, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ datetime_richcompare, /* tp_richcompare */ @@ -7190,7 +7291,7 @@ static PyDateTime_CAPI capi = { .Time_FromTime = new_time_ex, .Delta_FromDelta = new_delta_ex, .TimeZone_FromTimeZone = new_timezone, - .DateTime_FromTimestamp = datetime_fromtimestamp, + .DateTime_FromTimestamp = datetime_datetime_fromtimestamp_capi, .Date_FromTimestamp = datetime_date_fromtimestamp_capi, .DateTime_FromDateAndTimeAndFold = new_datetime_ex2, .Time_FromTimeAndFold = new_time_ex2, diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 602b23cfca8945..46cedf83df1f00 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -58,6 +58,14 @@ #define _PY_DEC_ROUND_GUARD (MPD_ROUND_GUARD-1) #endif +#include "clinic/_decimal.c.h" + +/*[clinic input] +module _decimal +class _decimal.Decimal "PyObject *" "&dec_spec" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e0e1f68f1f413f5f]*/ + struct PyDecContextObject; struct DecCondMap; @@ -2869,18 +2877,41 @@ PyDecType_FromSequenceExact(PyTypeObject *type, PyObject *v, #define PyDec_FromSequenceExact(st, sequence, context) \ PyDecType_FromSequenceExact((st)->PyDec_Type, sequence, context) -/* class method */ +/*[clinic input] +@classmethod +_decimal.Decimal.from_float + + f as pyfloat: object + / + +Class method that converts a float to a decimal number, exactly. + +Since 0.1 is not exactly representable in binary floating point, +Decimal.from_float(0.1) is not the same as Decimal('0.1'). + + >>> Decimal.from_float(0.1) + Decimal('0.1000000000000000055511151231257827021181583404541015625') + >>> Decimal.from_float(float('nan')) + Decimal('NaN') + >>> Decimal.from_float(float('inf')) + Decimal('Infinity') + >>> Decimal.from_float(float('-inf')) + Decimal('-Infinity') +[clinic start generated code]*/ + static PyObject * -dec_from_float(PyObject *type, PyObject *pyfloat) +_decimal_Decimal_from_float_impl(PyTypeObject *type, PyObject *pyfloat) +/*[clinic end generated code: output=e62775271ac469e6 input=052036648342f8c8]*/ { PyObject *context; PyObject *result; - decimal_state *state = get_module_state_by_def((PyTypeObject *)type); + decimal_state *state = get_module_state_by_def(type); CURRENT_CONTEXT(state, context); result = PyDecType_FromFloatExact(state->PyDec_Type, pyfloat, context); - if (type != (PyObject *)state->PyDec_Type && result != NULL) { - Py_SETREF(result, PyObject_CallFunctionObjArgs(type, result, NULL)); + if (type != state->PyDec_Type && result != NULL) { + Py_SETREF(result, + PyObject_CallFunctionObjArgs((PyObject *)type, result, NULL)); } return result; @@ -2914,18 +2945,36 @@ PyDecType_FromNumberExact(PyTypeObject *type, PyObject *v, PyObject *context) } } -/* class method */ +/*[clinic input] +@classmethod +_decimal.Decimal.from_number + + number: object + / + +Class method that converts a real number to a decimal number, exactly. + + >>> Decimal.from_number(314) # int + Decimal('314') + >>> Decimal.from_number(0.1) # float + Decimal('0.1000000000000000055511151231257827021181583404541015625') + >>> Decimal.from_number(Decimal('3.14')) # another decimal instance + Decimal('3.14') +[clinic start generated code]*/ + static PyObject * -dec_from_number(PyObject *type, PyObject *number) +_decimal_Decimal_from_number_impl(PyTypeObject *type, PyObject *number) +/*[clinic end generated code: output=41885304e5beea0a input=c58b678e8916f66b]*/ { PyObject *context; PyObject *result; - decimal_state *state = get_module_state_by_def((PyTypeObject *)type); + decimal_state *state = get_module_state_by_def(type); CURRENT_CONTEXT(state, context); result = PyDecType_FromNumberExact(state->PyDec_Type, number, context); - if (type != (PyObject *)state->PyDec_Type && result != NULL) { - Py_SETREF(result, PyObject_CallFunctionObjArgs(type, result, NULL)); + if (type != state->PyDec_Type && result != NULL) { + Py_SETREF(result, + PyObject_CallFunctionObjArgs((PyObject *)type, result, NULL)); } return result; @@ -3750,9 +3799,18 @@ dec_as_long(PyObject *dec, PyObject *context, int round) return PyLongWriter_Finish(writer); } -/* Convert a Decimal to its exact integer ratio representation. */ +/*[clinic input] +_decimal.Decimal.as_integer_ratio + +Return a pair of integers whose ratio is exactly equal to the original. + +The ratio is in lowest terms and with a positive denominator. +Raise OverflowError on infinities and a ValueError on NaNs. +[clinic start generated code]*/ + static PyObject * -dec_as_integer_ratio(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal_as_integer_ratio_impl(PyObject *self) +/*[clinic end generated code: output=c5d88e900080c264 input=7861cb643f01525a]*/ { PyObject *numerator = NULL; PyObject *denominator = NULL; @@ -3854,21 +3912,29 @@ dec_as_integer_ratio(PyObject *self, PyObject *Py_UNUSED(dummy)) return result; } +/*[clinic input] +_decimal.Decimal.to_integral_value + + rounding: object = None + context: object = None + +Round to the nearest integer without signaling Inexact or Rounded. + +The rounding mode is determined by the rounding parameter if given, +else by the given context. If neither parameter is given, then the +rounding mode of the current default context is used. +[clinic start generated code]*/ + static PyObject * -PyDec_ToIntegralValue(PyObject *dec, PyObject *args, PyObject *kwds) +_decimal_Decimal_to_integral_value_impl(PyObject *self, PyObject *rounding, + PyObject *context) +/*[clinic end generated code: output=7301465765f48b6b input=04e2312d5ed19f77]*/ { - static char *kwlist[] = {"rounding", "context", NULL}; PyObject *result; - PyObject *rounding = Py_None; - PyObject *context = Py_None; uint32_t status = 0; mpd_context_t workctx; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OO", kwlist, - &rounding, &context)) { - return NULL; - } - decimal_state *state = get_module_state_by_def(Py_TYPE(dec)); + decimal_state *state = get_module_state_by_def(Py_TYPE(self)); CONTEXT_CHECK_VA(state, context); workctx = *CTX(context); @@ -3887,7 +3953,7 @@ PyDec_ToIntegralValue(PyObject *dec, PyObject *args, PyObject *kwds) return NULL; } - mpd_qround_to_int(MPD(result), MPD(dec), &workctx, &status); + mpd_qround_to_int(MPD(result), MPD(self), &workctx, &status); if (dec_addstatus(context, status)) { Py_DECREF(result); return NULL; @@ -3896,21 +3962,50 @@ PyDec_ToIntegralValue(PyObject *dec, PyObject *args, PyObject *kwds) return result; } +/*[clinic input] +_decimal.Decimal.to_integral + + rounding: object = None + context: object = None + +Identical to the to_integral_value() method. + +The to_integral() name has been kept for compatibility with older +versions. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_to_integral_impl(PyObject *self, PyObject *rounding, + PyObject *context) +/*[clinic end generated code: output=a0c7188686ee7f5c input=8eac6def038d13b9]*/ +{ + return _decimal_Decimal_to_integral_value_impl(self, rounding, context); +} + +/*[clinic input] +_decimal.Decimal.to_integral_exact + + rounding: object = None + context: object = None + +Round to the nearest integer. + +Decimal.to_integral_exact() signals Inexact or Rounded as appropriate +if rounding occurs. The rounding mode is determined by the rounding +parameter if given, else by the given context. If neither parameter is +given, then the rounding mode of the current default context is used. +[clinic start generated code]*/ + static PyObject * -PyDec_ToIntegralExact(PyObject *dec, PyObject *args, PyObject *kwds) +_decimal_Decimal_to_integral_exact_impl(PyObject *self, PyObject *rounding, + PyObject *context) +/*[clinic end generated code: output=8b004f9b45ac7746 input=c290166f59c1d6ab]*/ { - static char *kwlist[] = {"rounding", "context", NULL}; PyObject *result; - PyObject *rounding = Py_None; - PyObject *context = Py_None; uint32_t status = 0; mpd_context_t workctx; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OO", kwlist, - &rounding, &context)) { - return NULL; - } - decimal_state *state = get_module_state_by_def(Py_TYPE(dec)); + decimal_state *state = get_module_state_by_def(Py_TYPE(self)); CONTEXT_CHECK_VA(state, context); workctx = *CTX(context); @@ -3929,7 +4024,7 @@ PyDec_ToIntegralExact(PyObject *dec, PyObject *args, PyObject *kwds) return NULL; } - mpd_qround_to_intx(MPD(result), MPD(dec), &workctx, &status); + mpd_qround_to_intx(MPD(result), MPD(self), &workctx, &status); if (dec_addstatus(context, status)) { Py_DECREF(result); return NULL; @@ -4018,9 +4113,15 @@ PyDec_Round(PyObject *dec, PyObject *args) } } -/* Return the DecimalTuple representation of a PyDecObject. */ +/*[clinic input] +_decimal.Decimal.as_tuple + +Return a tuple representation of the number. +[clinic start generated code]*/ + static PyObject * -PyDec_AsTuple(PyObject *dec, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal_as_tuple_impl(PyObject *self) +/*[clinic end generated code: output=c6e8e2420c515eca input=e26f2151d78ff59d]*/ { PyObject *result = NULL; PyObject *sign = NULL; @@ -4032,13 +4133,13 @@ PyDec_AsTuple(PyObject *dec, PyObject *Py_UNUSED(dummy)) Py_ssize_t intlen, i; - x = mpd_qncopy(MPD(dec)); + x = mpd_qncopy(MPD(self)); if (x == NULL) { PyErr_NoMemory(); goto out; } - sign = PyLong_FromUnsignedLong(mpd_sign(MPD(dec))); + sign = PyLong_FromUnsignedLong(mpd_sign(MPD(self))); if (sign == NULL) { goto out; } @@ -4059,7 +4160,7 @@ PyDec_AsTuple(PyObject *dec, PyObject *Py_UNUSED(dummy)) expt = PyUnicode_FromString(mpd_isqnan(x)?"n":"N"); } else { - expt = PyLong_FromSsize_t(MPD(dec)->exp); + expt = PyLong_FromSsize_t(MPD(self)->exp); } if (expt == NULL) { goto out; @@ -4100,7 +4201,7 @@ PyDec_AsTuple(PyObject *dec, PyObject *Py_UNUSED(dummy)) } } - decimal_state *state = get_module_state_by_def(Py_TYPE(dec)); + decimal_state *state = get_module_state_by_def(Py_TYPE(self)); result = PyObject_CallFunctionObjArgs((PyObject *)state->DecimalTuple, sign, coeff, expt, NULL); @@ -4504,8 +4605,16 @@ Dec_BoolFuncVA(mpd_isnormal) Dec_BoolFuncVA(mpd_issubnormal) /* Unary functions, no context arg */ + +/*[clinic input] +_decimal.Decimal.adjusted + +Return the adjusted exponent (exp + digits - 1) of the number. +[clinic start generated code]*/ + static PyObject * -dec_mpd_adjexp(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal_adjusted_impl(PyObject *self) +/*[clinic end generated code: output=21ea2c9f23994c52 input=8ba2029d8d906b18]*/ { mpd_ssize_t retval; @@ -4519,14 +4628,31 @@ dec_mpd_adjexp(PyObject *self, PyObject *Py_UNUSED(dummy)) return PyLong_FromSsize_t(retval); } +/*[clinic input] +_decimal.Decimal.canonical + +Return the canonical encoding of the argument. + +Currently, the encoding of a Decimal instance is always canonical, +so this operation returns its argument unchanged. +[clinic start generated code]*/ + static PyObject * -dec_canonical(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal_canonical_impl(PyObject *self) +/*[clinic end generated code: output=3cbeb47d91e6da2d input=8a4719d14c52d521]*/ { return Py_NewRef(self); } +/*[clinic input] +_decimal.Decimal.conjugate + +Return self. +[clinic start generated code]*/ + static PyObject * -dec_conjugate(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal_conjugate_impl(PyObject *self) +/*[clinic end generated code: output=9a37bf633f25a291 input=c7179975ef74fd84]*/ { return Py_NewRef(self); } @@ -4545,15 +4671,35 @@ _dec_mpd_radix(decimal_state *state) return result; } +/*[clinic input] +_decimal.Decimal.radix + +Return Decimal(10). + +This is the radix (base) in which the Decimal class does +all its arithmetic. Included for compatibility with the specification. +[clinic start generated code]*/ + static PyObject * -dec_mpd_radix(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal_radix_impl(PyObject *self) +/*[clinic end generated code: output=6b1db4c3fcdb5ee1 input=18b72393549ca8fd]*/ { decimal_state *state = get_module_state_by_def(Py_TYPE(self)); return _dec_mpd_radix(state); } +/*[clinic input] +_decimal.Decimal.copy_abs + +Return the absolute value of the argument. + +This operation is unaffected by context and is quiet: no flags are +changed and no rounding is performed. +[clinic start generated code]*/ + static PyObject * -dec_mpd_qcopy_abs(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal_copy_abs_impl(PyObject *self) +/*[clinic end generated code: output=fff53742cca94d70 input=a263c2e71d421f1b]*/ { PyObject *result; uint32_t status = 0; @@ -4573,8 +4719,18 @@ dec_mpd_qcopy_abs(PyObject *self, PyObject *Py_UNUSED(dummy)) return result; } +/*[clinic input] +_decimal.Decimal.copy_negate + +Return the negation of the argument. + +This operation is unaffected by context and is quiet: no flags are +changed and no rounding is performed. +[clinic start generated code]*/ + static PyObject * -dec_mpd_qcopy_negate(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal_copy_negate_impl(PyObject *self) +/*[clinic end generated code: output=8551bc26dbc5d01d input=13d47ed3a5d228b1]*/ { PyObject *result; uint32_t status = 0; @@ -4598,17 +4754,37 @@ dec_mpd_qcopy_negate(PyObject *self, PyObject *Py_UNUSED(dummy)) Dec_UnaryFuncVA(mpd_qinvert) Dec_UnaryFuncVA(mpd_qlogb) +/*[clinic input] +_decimal.Decimal.number_class + + context: object = None + +Return a string describing the class of the operand. + +The returned value is one of the following ten strings: + + * '-Infinity', indicating that the operand is negative infinity. + * '-Normal', indicating that the operand is a negative normal + number. + * '-Subnormal', indicating that the operand is negative and + subnormal. + * '-Zero', indicating that the operand is a negative zero. + * '+Zero', indicating that the operand is a positive zero. + * '+Subnormal', indicating that the operand is positive and + subnormal. + * '+Normal', indicating that the operand is a positive normal + number. + * '+Infinity', indicating that the operand is positive infinity. + * 'NaN', indicating that the operand is a quiet NaN (Not a Number). + * 'sNaN', indicating that the operand is a signaling NaN. +[clinic start generated code]*/ + static PyObject * -dec_mpd_class(PyObject *self, PyObject *args, PyObject *kwds) +_decimal_Decimal_number_class_impl(PyObject *self, PyObject *context) +/*[clinic end generated code: output=3044cd45966b4949 input=f3d6cdda603e8b89]*/ { - static char *kwlist[] = {"context", NULL}; - PyObject *context = Py_None; const char *cp; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, - &context)) { - return NULL; - } decimal_state *state = get_module_state_by_def(Py_TYPE(self)); CONTEXT_CHECK_VA(state, context); @@ -4616,19 +4792,30 @@ dec_mpd_class(PyObject *self, PyObject *args, PyObject *kwds) return PyUnicode_FromString(cp); } +/*[clinic input] +_decimal.Decimal.to_eng_string + + context: object = None + +Convert to an engineering-type string. + +Engineering notation has an exponent which is a multiple of 3, so there +are up to 3 digits left of the decimal place. For example, +Decimal('123E+1') is converted to Decimal('1.23E+3'). + +The value of context.capitals determines whether the exponent sign is +lower or upper case. Otherwise, the context does not affect the +operation. +[clinic start generated code]*/ + static PyObject * -dec_mpd_to_eng(PyObject *self, PyObject *args, PyObject *kwds) +_decimal_Decimal_to_eng_string_impl(PyObject *self, PyObject *context) +/*[clinic end generated code: output=d386194c25ffffa7 input=2e13e7c7c1bad2ad]*/ { - static char *kwlist[] = {"context", NULL}; PyObject *result; - PyObject *context = Py_None; mpd_ssize_t size; char *s; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, - &context)) { - return NULL; - } decimal_state *state = get_module_state_by_def(Py_TYPE(self)); CONTEXT_CHECK_VA(state, context); @@ -4648,20 +4835,34 @@ dec_mpd_to_eng(PyObject *self, PyObject *args, PyObject *kwds) Dec_BinaryFuncVA_NO_CTX(mpd_compare_total) Dec_BinaryFuncVA_NO_CTX(mpd_compare_total_mag) +/*[clinic input] +_decimal.Decimal.copy_sign + + other: object + context: object = None + +Return a copy of *self* with the sign of *other*. + +For example: + + >>> Decimal('2.3').copy_sign(Decimal('-1.5')) + Decimal('-2.3') + +This operation is unaffected by context and is quiet: no flags are +changed and no rounding is performed. As an exception, the C version +may raise InvalidOperation if the second operand cannot be converted +exactly. +[clinic start generated code]*/ + static PyObject * -dec_mpd_qcopy_sign(PyObject *self, PyObject *args, PyObject *kwds) +_decimal_Decimal_copy_sign_impl(PyObject *self, PyObject *other, + PyObject *context) +/*[clinic end generated code: output=72c62177763e012e input=8410238d533a06eb]*/ { - static char *kwlist[] = {"other", "context", NULL}; - PyObject *other; PyObject *a, *b; PyObject *result; - PyObject *context = Py_None; uint32_t status = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O", kwlist, - &other, &context)) { - return NULL; - } decimal_state *state = get_module_state_by_def(Py_TYPE(self)); CONTEXT_CHECK_VA(state, context); CONVERT_BINOP_RAISE(&a, &b, self, other, context); @@ -4684,19 +4885,28 @@ dec_mpd_qcopy_sign(PyObject *self, PyObject *args, PyObject *kwds) return result; } +/*[clinic input] +_decimal.Decimal.same_quantum + + other: object + context: object = None + +Test whether self and other have the same exponent or both are NaN. + +This operation is unaffected by context and is quiet: no flags are +changed and no rounding is performed. As an exception, the C version +may raise InvalidOperation if the second operand cannot be converted +exactly. +[clinic start generated code]*/ + static PyObject * -dec_mpd_same_quantum(PyObject *self, PyObject *args, PyObject *kwds) +_decimal_Decimal_same_quantum_impl(PyObject *self, PyObject *other, + PyObject *context) +/*[clinic end generated code: output=c0a3a046c662a7e2 input=3ae45df81d6edb73]*/ { - static char *kwlist[] = {"other", "context", NULL}; - PyObject *other; PyObject *a, *b; PyObject *result; - PyObject *context = Py_None; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O", kwlist, - &other, &context)) { - return NULL; - } decimal_state *state = get_module_state_by_def(Py_TYPE(self)); CONTEXT_CHECK_VA(state, context); CONVERT_BINOP_RAISE(&a, &b, self, other, context); @@ -4717,22 +4927,48 @@ Dec_BinaryFuncVA(mpd_qrotate) Dec_BinaryFuncVA(mpd_qscaleb) Dec_BinaryFuncVA(mpd_qshift) +/*[clinic input] +_decimal.Decimal.quantize + + exp as w: object + rounding: object = None + context: object = None + +Quantize *self* so its exponent is the same as that of *exp*. + +Return a value equal to *self* after rounding, with the exponent +of *exp*. + + >>> Decimal('1.41421356').quantize(Decimal('1.000')) + Decimal('1.414') + +Unlike other operations, if the length of the coefficient after the +quantize operation would be greater than precision, then an +InvalidOperation is signaled. This guarantees that, unless there +is an error condition, the quantized exponent is always equal to +that of the right-hand operand. + +Also unlike other operations, quantize never signals Underflow, even +if the result is subnormal and inexact. + +If the exponent of the second operand is larger than that of the first, +then rounding may be necessary. In this case, the rounding mode is +determined by the rounding argument if given, else by the given context +argument; if neither argument is given, the rounding mode of the +current thread's context is used. +[clinic start generated code]*/ + static PyObject * -dec_mpd_qquantize(PyObject *v, PyObject *args, PyObject *kwds) +_decimal_Decimal_quantize_impl(PyObject *self, PyObject *w, + PyObject *rounding, PyObject *context) +/*[clinic end generated code: output=5e84581f96dc685c input=4c7d28d36948e9aa]*/ { - static char *kwlist[] = {"exp", "rounding", "context", NULL}; - PyObject *rounding = Py_None; - PyObject *context = Py_None; - PyObject *w, *a, *b; + PyObject *a, *b; PyObject *result; uint32_t status = 0; mpd_context_t workctx; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO", kwlist, - &w, &rounding, &context)) { - return NULL; - } - decimal_state *state = get_module_state_by_def(Py_TYPE(v)); + decimal_state *state = get_module_state_by_def(Py_TYPE(self)); CONTEXT_CHECK_VA(state, context); workctx = *CTX(context); @@ -4746,7 +4982,7 @@ dec_mpd_qquantize(PyObject *v, PyObject *args, PyObject *kwds) } } - CONVERT_BINOP_RAISE(&a, &b, v, w, context); + CONVERT_BINOP_RAISE(&a, &b, self, w, context); result = dec_alloc(state); if (result == NULL) { @@ -5081,9 +5317,9 @@ static PyMethodDef dec_methods [] = { "next_minus", _PyCFunction_CAST(dec_mpd_qnext_minus), METH_VARARGS|METH_KEYWORDS, doc_next_minus }, { "next_plus", _PyCFunction_CAST(dec_mpd_qnext_plus), METH_VARARGS|METH_KEYWORDS, doc_next_plus }, { "normalize", _PyCFunction_CAST(dec_mpd_qreduce), METH_VARARGS|METH_KEYWORDS, doc_normalize }, - { "to_integral", _PyCFunction_CAST(PyDec_ToIntegralValue), METH_VARARGS|METH_KEYWORDS, doc_to_integral }, - { "to_integral_exact", _PyCFunction_CAST(PyDec_ToIntegralExact), METH_VARARGS|METH_KEYWORDS, doc_to_integral_exact }, - { "to_integral_value", _PyCFunction_CAST(PyDec_ToIntegralValue), METH_VARARGS|METH_KEYWORDS, doc_to_integral_value }, + _DECIMAL_DECIMAL_TO_INTEGRAL_METHODDEF + _DECIMAL_DECIMAL_TO_INTEGRAL_EXACT_METHODDEF + _DECIMAL_DECIMAL_TO_INTEGRAL_VALUE_METHODDEF { "sqrt", _PyCFunction_CAST(dec_mpd_qsqrt), METH_VARARGS|METH_KEYWORDS, doc_sqrt }, /* Binary arithmetic functions, optional context arg */ @@ -5094,7 +5330,7 @@ static PyMethodDef dec_methods [] = { "min", _PyCFunction_CAST(dec_mpd_qmin), METH_VARARGS|METH_KEYWORDS, doc_min }, { "min_mag", _PyCFunction_CAST(dec_mpd_qmin_mag), METH_VARARGS|METH_KEYWORDS, doc_min_mag }, { "next_toward", _PyCFunction_CAST(dec_mpd_qnext_toward), METH_VARARGS|METH_KEYWORDS, doc_next_toward }, - { "quantize", _PyCFunction_CAST(dec_mpd_qquantize), METH_VARARGS|METH_KEYWORDS, doc_quantize }, + _DECIMAL_DECIMAL_QUANTIZE_METHODDEF { "remainder_near", _PyCFunction_CAST(dec_mpd_qrem_near), METH_VARARGS|METH_KEYWORDS, doc_remainder_near }, /* Ternary arithmetic functions, optional context arg */ @@ -5115,26 +5351,26 @@ static PyMethodDef dec_methods [] = { "is_subnormal", _PyCFunction_CAST(dec_mpd_issubnormal), METH_VARARGS|METH_KEYWORDS, doc_is_subnormal }, /* Unary functions, no context arg */ - { "adjusted", dec_mpd_adjexp, METH_NOARGS, doc_adjusted }, - { "canonical", dec_canonical, METH_NOARGS, doc_canonical }, - { "conjugate", dec_conjugate, METH_NOARGS, doc_conjugate }, - { "radix", dec_mpd_radix, METH_NOARGS, doc_radix }, + _DECIMAL_DECIMAL_ADJUSTED_METHODDEF + _DECIMAL_DECIMAL_CANONICAL_METHODDEF + _DECIMAL_DECIMAL_CONJUGATE_METHODDEF + _DECIMAL_DECIMAL_RADIX_METHODDEF /* Unary functions, optional context arg for conversion errors */ - { "copy_abs", dec_mpd_qcopy_abs, METH_NOARGS, doc_copy_abs }, - { "copy_negate", dec_mpd_qcopy_negate, METH_NOARGS, doc_copy_negate }, + _DECIMAL_DECIMAL_COPY_ABS_METHODDEF + _DECIMAL_DECIMAL_COPY_NEGATE_METHODDEF /* Unary functions, optional context arg */ { "logb", _PyCFunction_CAST(dec_mpd_qlogb), METH_VARARGS|METH_KEYWORDS, doc_logb }, { "logical_invert", _PyCFunction_CAST(dec_mpd_qinvert), METH_VARARGS|METH_KEYWORDS, doc_logical_invert }, - { "number_class", _PyCFunction_CAST(dec_mpd_class), METH_VARARGS|METH_KEYWORDS, doc_number_class }, - { "to_eng_string", _PyCFunction_CAST(dec_mpd_to_eng), METH_VARARGS|METH_KEYWORDS, doc_to_eng_string }, + _DECIMAL_DECIMAL_NUMBER_CLASS_METHODDEF + _DECIMAL_DECIMAL_TO_ENG_STRING_METHODDEF /* Binary functions, optional context arg for conversion errors */ { "compare_total", _PyCFunction_CAST(dec_mpd_compare_total), METH_VARARGS|METH_KEYWORDS, doc_compare_total }, { "compare_total_mag", _PyCFunction_CAST(dec_mpd_compare_total_mag), METH_VARARGS|METH_KEYWORDS, doc_compare_total_mag }, - { "copy_sign", _PyCFunction_CAST(dec_mpd_qcopy_sign), METH_VARARGS|METH_KEYWORDS, doc_copy_sign }, - { "same_quantum", _PyCFunction_CAST(dec_mpd_same_quantum), METH_VARARGS|METH_KEYWORDS, doc_same_quantum }, + _DECIMAL_DECIMAL_COPY_SIGN_METHODDEF + _DECIMAL_DECIMAL_SAME_QUANTUM_METHODDEF /* Binary functions, optional context arg */ { "logical_and", _PyCFunction_CAST(dec_mpd_qand), METH_VARARGS|METH_KEYWORDS, doc_logical_and }, @@ -5145,10 +5381,10 @@ static PyMethodDef dec_methods [] = { "shift", _PyCFunction_CAST(dec_mpd_qshift), METH_VARARGS|METH_KEYWORDS, doc_shift }, /* Miscellaneous */ - { "from_float", dec_from_float, METH_O|METH_CLASS, doc_from_float }, - { "from_number", dec_from_number, METH_O|METH_CLASS, doc_from_number }, - { "as_tuple", PyDec_AsTuple, METH_NOARGS, doc_as_tuple }, - { "as_integer_ratio", dec_as_integer_ratio, METH_NOARGS, doc_as_integer_ratio }, + _DECIMAL_DECIMAL_FROM_FLOAT_METHODDEF + _DECIMAL_DECIMAL_FROM_NUMBER_METHODDEF + _DECIMAL_DECIMAL_AS_TUPLE_METHODDEF + _DECIMAL_DECIMAL_AS_INTEGER_RATIO_METHODDEF /* Special methods */ { "__copy__", dec_copy, METH_NOARGS, NULL }, diff --git a/Modules/_decimal/clinic/_decimal.c.h b/Modules/_decimal/clinic/_decimal.c.h new file mode 100644 index 00000000000000..441515edbf60f6 --- /dev/null +++ b/Modules/_decimal/clinic/_decimal.c.h @@ -0,0 +1,849 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(_decimal_Decimal_from_float__doc__, +"from_float($type, f, /)\n" +"--\n" +"\n" +"Class method that converts a float to a decimal number, exactly.\n" +"\n" +"Since 0.1 is not exactly representable in binary floating point,\n" +"Decimal.from_float(0.1) is not the same as Decimal(\'0.1\').\n" +"\n" +" >>> Decimal.from_float(0.1)\n" +" Decimal(\'0.1000000000000000055511151231257827021181583404541015625\')\n" +" >>> Decimal.from_float(float(\'nan\'))\n" +" Decimal(\'NaN\')\n" +" >>> Decimal.from_float(float(\'inf\'))\n" +" Decimal(\'Infinity\')\n" +" >>> Decimal.from_float(float(\'-inf\'))\n" +" Decimal(\'-Infinity\')"); + +#define _DECIMAL_DECIMAL_FROM_FLOAT_METHODDEF \ + {"from_float", (PyCFunction)_decimal_Decimal_from_float, METH_O|METH_CLASS, _decimal_Decimal_from_float__doc__}, + +static PyObject * +_decimal_Decimal_from_float_impl(PyTypeObject *type, PyObject *pyfloat); + +static PyObject * +_decimal_Decimal_from_float(PyObject *type, PyObject *pyfloat) +{ + PyObject *return_value = NULL; + + return_value = _decimal_Decimal_from_float_impl((PyTypeObject *)type, pyfloat); + + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_from_number__doc__, +"from_number($type, number, /)\n" +"--\n" +"\n" +"Class method that converts a real number to a decimal number, exactly.\n" +"\n" +" >>> Decimal.from_number(314) # int\n" +" Decimal(\'314\')\n" +" >>> Decimal.from_number(0.1) # float\n" +" Decimal(\'0.1000000000000000055511151231257827021181583404541015625\')\n" +" >>> Decimal.from_number(Decimal(\'3.14\')) # another decimal instance\n" +" Decimal(\'3.14\')"); + +#define _DECIMAL_DECIMAL_FROM_NUMBER_METHODDEF \ + {"from_number", (PyCFunction)_decimal_Decimal_from_number, METH_O|METH_CLASS, _decimal_Decimal_from_number__doc__}, + +static PyObject * +_decimal_Decimal_from_number_impl(PyTypeObject *type, PyObject *number); + +static PyObject * +_decimal_Decimal_from_number(PyObject *type, PyObject *number) +{ + PyObject *return_value = NULL; + + return_value = _decimal_Decimal_from_number_impl((PyTypeObject *)type, number); + + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_as_integer_ratio__doc__, +"as_integer_ratio($self, /)\n" +"--\n" +"\n" +"Return a pair of integers whose ratio is exactly equal to the original.\n" +"\n" +"The ratio is in lowest terms and with a positive denominator.\n" +"Raise OverflowError on infinities and a ValueError on NaNs."); + +#define _DECIMAL_DECIMAL_AS_INTEGER_RATIO_METHODDEF \ + {"as_integer_ratio", (PyCFunction)_decimal_Decimal_as_integer_ratio, METH_NOARGS, _decimal_Decimal_as_integer_ratio__doc__}, + +static PyObject * +_decimal_Decimal_as_integer_ratio_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_as_integer_ratio(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_as_integer_ratio_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_to_integral_value__doc__, +"to_integral_value($self, /, rounding=None, context=None)\n" +"--\n" +"\n" +"Round to the nearest integer without signaling Inexact or Rounded.\n" +"\n" +"The rounding mode is determined by the rounding parameter if given,\n" +"else by the given context. If neither parameter is given, then the\n" +"rounding mode of the current default context is used."); + +#define _DECIMAL_DECIMAL_TO_INTEGRAL_VALUE_METHODDEF \ + {"to_integral_value", _PyCFunction_CAST(_decimal_Decimal_to_integral_value), METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_to_integral_value__doc__}, + +static PyObject * +_decimal_Decimal_to_integral_value_impl(PyObject *self, PyObject *rounding, + PyObject *context); + +static PyObject * +_decimal_Decimal_to_integral_value(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(rounding), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"rounding", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "to_integral_value", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *rounding = Py_None; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + rounding = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_to_integral_value_impl(self, rounding, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_to_integral__doc__, +"to_integral($self, /, rounding=None, context=None)\n" +"--\n" +"\n" +"Identical to the to_integral_value() method.\n" +"\n" +"The to_integral() name has been kept for compatibility with older\n" +"versions."); + +#define _DECIMAL_DECIMAL_TO_INTEGRAL_METHODDEF \ + {"to_integral", _PyCFunction_CAST(_decimal_Decimal_to_integral), METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_to_integral__doc__}, + +static PyObject * +_decimal_Decimal_to_integral_impl(PyObject *self, PyObject *rounding, + PyObject *context); + +static PyObject * +_decimal_Decimal_to_integral(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(rounding), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"rounding", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "to_integral", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *rounding = Py_None; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + rounding = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_to_integral_impl(self, rounding, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_to_integral_exact__doc__, +"to_integral_exact($self, /, rounding=None, context=None)\n" +"--\n" +"\n" +"Round to the nearest integer.\n" +"\n" +"Decimal.to_integral_exact() signals Inexact or Rounded as appropriate\n" +"if rounding occurs. The rounding mode is determined by the rounding\n" +"parameter if given, else by the given context. If neither parameter is\n" +"given, then the rounding mode of the current default context is used."); + +#define _DECIMAL_DECIMAL_TO_INTEGRAL_EXACT_METHODDEF \ + {"to_integral_exact", _PyCFunction_CAST(_decimal_Decimal_to_integral_exact), METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_to_integral_exact__doc__}, + +static PyObject * +_decimal_Decimal_to_integral_exact_impl(PyObject *self, PyObject *rounding, + PyObject *context); + +static PyObject * +_decimal_Decimal_to_integral_exact(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(rounding), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"rounding", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "to_integral_exact", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *rounding = Py_None; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + rounding = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_to_integral_exact_impl(self, rounding, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_as_tuple__doc__, +"as_tuple($self, /)\n" +"--\n" +"\n" +"Return a tuple representation of the number."); + +#define _DECIMAL_DECIMAL_AS_TUPLE_METHODDEF \ + {"as_tuple", (PyCFunction)_decimal_Decimal_as_tuple, METH_NOARGS, _decimal_Decimal_as_tuple__doc__}, + +static PyObject * +_decimal_Decimal_as_tuple_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_as_tuple(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_as_tuple_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_adjusted__doc__, +"adjusted($self, /)\n" +"--\n" +"\n" +"Return the adjusted exponent (exp + digits - 1) of the number."); + +#define _DECIMAL_DECIMAL_ADJUSTED_METHODDEF \ + {"adjusted", (PyCFunction)_decimal_Decimal_adjusted, METH_NOARGS, _decimal_Decimal_adjusted__doc__}, + +static PyObject * +_decimal_Decimal_adjusted_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_adjusted(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_adjusted_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_canonical__doc__, +"canonical($self, /)\n" +"--\n" +"\n" +"Return the canonical encoding of the argument.\n" +"\n" +"Currently, the encoding of a Decimal instance is always canonical,\n" +"so this operation returns its argument unchanged."); + +#define _DECIMAL_DECIMAL_CANONICAL_METHODDEF \ + {"canonical", (PyCFunction)_decimal_Decimal_canonical, METH_NOARGS, _decimal_Decimal_canonical__doc__}, + +static PyObject * +_decimal_Decimal_canonical_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_canonical(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_canonical_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_conjugate__doc__, +"conjugate($self, /)\n" +"--\n" +"\n" +"Return self."); + +#define _DECIMAL_DECIMAL_CONJUGATE_METHODDEF \ + {"conjugate", (PyCFunction)_decimal_Decimal_conjugate, METH_NOARGS, _decimal_Decimal_conjugate__doc__}, + +static PyObject * +_decimal_Decimal_conjugate_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_conjugate(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_conjugate_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_radix__doc__, +"radix($self, /)\n" +"--\n" +"\n" +"Return Decimal(10).\n" +"\n" +"This is the radix (base) in which the Decimal class does\n" +"all its arithmetic. Included for compatibility with the specification."); + +#define _DECIMAL_DECIMAL_RADIX_METHODDEF \ + {"radix", (PyCFunction)_decimal_Decimal_radix, METH_NOARGS, _decimal_Decimal_radix__doc__}, + +static PyObject * +_decimal_Decimal_radix_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_radix(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_radix_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_copy_abs__doc__, +"copy_abs($self, /)\n" +"--\n" +"\n" +"Return the absolute value of the argument.\n" +"\n" +"This operation is unaffected by context and is quiet: no flags are\n" +"changed and no rounding is performed."); + +#define _DECIMAL_DECIMAL_COPY_ABS_METHODDEF \ + {"copy_abs", (PyCFunction)_decimal_Decimal_copy_abs, METH_NOARGS, _decimal_Decimal_copy_abs__doc__}, + +static PyObject * +_decimal_Decimal_copy_abs_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_copy_abs(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_copy_abs_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_copy_negate__doc__, +"copy_negate($self, /)\n" +"--\n" +"\n" +"Return the negation of the argument.\n" +"\n" +"This operation is unaffected by context and is quiet: no flags are\n" +"changed and no rounding is performed."); + +#define _DECIMAL_DECIMAL_COPY_NEGATE_METHODDEF \ + {"copy_negate", (PyCFunction)_decimal_Decimal_copy_negate, METH_NOARGS, _decimal_Decimal_copy_negate__doc__}, + +static PyObject * +_decimal_Decimal_copy_negate_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_copy_negate(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_copy_negate_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_number_class__doc__, +"number_class($self, /, context=None)\n" +"--\n" +"\n" +"Return a string describing the class of the operand.\n" +"\n" +"The returned value is one of the following ten strings:\n" +"\n" +" * \'-Infinity\', indicating that the operand is negative infinity.\n" +" * \'-Normal\', indicating that the operand is a negative normal\n" +" number.\n" +" * \'-Subnormal\', indicating that the operand is negative and\n" +" subnormal.\n" +" * \'-Zero\', indicating that the operand is a negative zero.\n" +" * \'+Zero\', indicating that the operand is a positive zero.\n" +" * \'+Subnormal\', indicating that the operand is positive and\n" +" subnormal.\n" +" * \'+Normal\', indicating that the operand is a positive normal\n" +" number.\n" +" * \'+Infinity\', indicating that the operand is positive infinity.\n" +" * \'NaN\', indicating that the operand is a quiet NaN (Not a Number).\n" +" * \'sNaN\', indicating that the operand is a signaling NaN."); + +#define _DECIMAL_DECIMAL_NUMBER_CLASS_METHODDEF \ + {"number_class", _PyCFunction_CAST(_decimal_Decimal_number_class), METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_number_class__doc__}, + +static PyObject * +_decimal_Decimal_number_class_impl(PyObject *self, PyObject *context); + +static PyObject * +_decimal_Decimal_number_class(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "number_class", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + context = args[0]; +skip_optional_pos: + return_value = _decimal_Decimal_number_class_impl(self, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_to_eng_string__doc__, +"to_eng_string($self, /, context=None)\n" +"--\n" +"\n" +"Convert to an engineering-type string.\n" +"\n" +"Engineering notation has an exponent which is a multiple of 3, so there\n" +"are up to 3 digits left of the decimal place. For example,\n" +"Decimal(\'123E+1\') is converted to Decimal(\'1.23E+3\').\n" +"\n" +"The value of context.capitals determines whether the exponent sign is\n" +"lower or upper case. Otherwise, the context does not affect the\n" +"operation."); + +#define _DECIMAL_DECIMAL_TO_ENG_STRING_METHODDEF \ + {"to_eng_string", _PyCFunction_CAST(_decimal_Decimal_to_eng_string), METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_to_eng_string__doc__}, + +static PyObject * +_decimal_Decimal_to_eng_string_impl(PyObject *self, PyObject *context); + +static PyObject * +_decimal_Decimal_to_eng_string(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "to_eng_string", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + context = args[0]; +skip_optional_pos: + return_value = _decimal_Decimal_to_eng_string_impl(self, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_copy_sign__doc__, +"copy_sign($self, /, other, context=None)\n" +"--\n" +"\n" +"Return a copy of *self* with the sign of *other*.\n" +"\n" +"For example:\n" +"\n" +" >>> Decimal(\'2.3\').copy_sign(Decimal(\'-1.5\'))\n" +" Decimal(\'-2.3\')\n" +"\n" +"This operation is unaffected by context and is quiet: no flags are\n" +"changed and no rounding is performed. As an exception, the C version\n" +"may raise InvalidOperation if the second operand cannot be converted\n" +"exactly."); + +#define _DECIMAL_DECIMAL_COPY_SIGN_METHODDEF \ + {"copy_sign", _PyCFunction_CAST(_decimal_Decimal_copy_sign), METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_copy_sign__doc__}, + +static PyObject * +_decimal_Decimal_copy_sign_impl(PyObject *self, PyObject *other, + PyObject *context); + +static PyObject * +_decimal_Decimal_copy_sign(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "copy_sign", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_copy_sign_impl(self, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_same_quantum__doc__, +"same_quantum($self, /, other, context=None)\n" +"--\n" +"\n" +"Test whether self and other have the same exponent or both are NaN.\n" +"\n" +"This operation is unaffected by context and is quiet: no flags are\n" +"changed and no rounding is performed. As an exception, the C version\n" +"may raise InvalidOperation if the second operand cannot be converted\n" +"exactly."); + +#define _DECIMAL_DECIMAL_SAME_QUANTUM_METHODDEF \ + {"same_quantum", _PyCFunction_CAST(_decimal_Decimal_same_quantum), METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_same_quantum__doc__}, + +static PyObject * +_decimal_Decimal_same_quantum_impl(PyObject *self, PyObject *other, + PyObject *context); + +static PyObject * +_decimal_Decimal_same_quantum(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "same_quantum", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_same_quantum_impl(self, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_quantize__doc__, +"quantize($self, /, exp, rounding=None, context=None)\n" +"--\n" +"\n" +"Quantize *self* so its exponent is the same as that of *exp*.\n" +"\n" +"Return a value equal to *self* after rounding, with the exponent\n" +"of *exp*.\n" +"\n" +" >>> Decimal(\'1.41421356\').quantize(Decimal(\'1.000\'))\n" +" Decimal(\'1.414\')\n" +"\n" +"Unlike other operations, if the length of the coefficient after the\n" +"quantize operation would be greater than precision, then an\n" +"InvalidOperation is signaled. This guarantees that, unless there\n" +"is an error condition, the quantized exponent is always equal to\n" +"that of the right-hand operand.\n" +"\n" +"Also unlike other operations, quantize never signals Underflow, even\n" +"if the result is subnormal and inexact.\n" +"\n" +"If the exponent of the second operand is larger than that of the first,\n" +"then rounding may be necessary. In this case, the rounding mode is\n" +"determined by the rounding argument if given, else by the given context\n" +"argument; if neither argument is given, the rounding mode of the\n" +"current thread\'s context is used."); + +#define _DECIMAL_DECIMAL_QUANTIZE_METHODDEF \ + {"quantize", _PyCFunction_CAST(_decimal_Decimal_quantize), METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_quantize__doc__}, + +static PyObject * +_decimal_Decimal_quantize_impl(PyObject *self, PyObject *w, + PyObject *rounding, PyObject *context); + +static PyObject * +_decimal_Decimal_quantize(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(exp), &_Py_ID(rounding), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"exp", "rounding", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "quantize", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *w; + PyObject *rounding = Py_None; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + w = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + rounding = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + context = args[2]; +skip_optional_pos: + return_value = _decimal_Decimal_quantize_impl(self, w, rounding, context); + +exit: + return return_value; +} +/*[clinic end generated code: output=f33166d1bf53e613 input=a9049054013a1b77]*/ diff --git a/Modules/_decimal/docstrings.h b/Modules/_decimal/docstrings.h index 77017a92252cb8..bd1c1d5295d717 100644 --- a/Modules/_decimal/docstrings.h +++ b/Modules/_decimal/docstrings.h @@ -57,32 +57,6 @@ context does not affect the conversion and is only passed to determine if\n\ the InvalidOperation trap is active.\n\ \n"); -PyDoc_STRVAR(doc_adjusted, -"adjusted($self, /)\n--\n\n\ -Return the adjusted exponent of the number. Defined as exp + digits - 1.\n\ -\n"); - -PyDoc_STRVAR(doc_as_tuple, -"as_tuple($self, /)\n--\n\n\ -Return a tuple representation of the number.\n\ -\n"); - -PyDoc_STRVAR(doc_as_integer_ratio, -"as_integer_ratio($self, /)\n--\n\n\ -Decimal.as_integer_ratio() -> (int, int)\n\ -\n\ -Return a pair of integers, whose ratio is exactly equal to the original\n\ -Decimal and with a positive denominator. The ratio is in lowest terms.\n\ -Raise OverflowError on infinities and a ValueError on NaNs.\n\ -\n"); - -PyDoc_STRVAR(doc_canonical, -"canonical($self, /)\n--\n\n\ -Return the canonical encoding of the argument. Currently, the encoding\n\ -of a Decimal instance is always canonical, so this operation returns its\n\ -argument unchanged.\n\ -\n"); - PyDoc_STRVAR(doc_compare, "compare($self, /, other, context=None)\n--\n\n\ Compare self to other. Return a decimal value:\n\ @@ -132,36 +106,6 @@ and no rounding is performed. As an exception, the C version may raise\n\ InvalidOperation if the second operand cannot be converted exactly.\n\ \n"); -PyDoc_STRVAR(doc_conjugate, -"conjugate($self, /)\n--\n\n\ -Return self.\n\ -\n"); - -PyDoc_STRVAR(doc_copy_abs, -"copy_abs($self, /)\n--\n\n\ -Return the absolute value of the argument. This operation is unaffected by\n\ -context and is quiet: no flags are changed and no rounding is performed.\n\ -\n"); - -PyDoc_STRVAR(doc_copy_negate, -"copy_negate($self, /)\n--\n\n\ -Return the negation of the argument. This operation is unaffected by context\n\ -and is quiet: no flags are changed and no rounding is performed.\n\ -\n"); - -PyDoc_STRVAR(doc_copy_sign, -"copy_sign($self, /, other, context=None)\n--\n\n\ -Return a copy of the first operand with the sign set to be the same as the\n\ -sign of the second operand. For example:\n\ -\n\ - >>> Decimal('2.3').copy_sign(Decimal('-1.5'))\n\ - Decimal('-2.3')\n\ -\n\ -This operation is unaffected by context and is quiet: no flags are changed\n\ -and no rounding is performed. As an exception, the C version may raise\n\ -InvalidOperation if the second operand cannot be converted exactly.\n\ -\n"); - PyDoc_STRVAR(doc_exp, "exp($self, /, context=None)\n--\n\n\ Return the value of the (natural) exponential function e**x at the given\n\ @@ -169,36 +113,6 @@ number. The function always uses the ROUND_HALF_EVEN mode and the result\n\ is correctly rounded.\n\ \n"); -PyDoc_STRVAR(doc_from_float, -"from_float($type, f, /)\n--\n\n\ -Class method that converts a float to a decimal number, exactly.\n\ -Since 0.1 is not exactly representable in binary floating point,\n\ -Decimal.from_float(0.1) is not the same as Decimal('0.1').\n\ -\n\ - >>> Decimal.from_float(0.1)\n\ - Decimal('0.1000000000000000055511151231257827021181583404541015625')\n\ - >>> Decimal.from_float(float('nan'))\n\ - Decimal('NaN')\n\ - >>> Decimal.from_float(float('inf'))\n\ - Decimal('Infinity')\n\ - >>> Decimal.from_float(float('-inf'))\n\ - Decimal('-Infinity')\n\ -\n\ -\n"); - -PyDoc_STRVAR(doc_from_number, -"from_number($type, number, /)\n--\n\n\ -Class method that converts a real number to a decimal number, exactly.\n\ -\n\ - >>> Decimal.from_number(314) # int\n\ - Decimal('314')\n\ - >>> Decimal.from_number(0.1) # float\n\ - Decimal('0.1000000000000000055511151231257827021181583404541015625')\n\ - >>> Decimal.from_number(Decimal('3.14')) # another decimal instance\n\ - Decimal('3.14')\n\ -\n\ -\n"); - PyDoc_STRVAR(doc_fma, "fma($self, /, other, third, context=None)\n--\n\n\ Fused multiply-add. Return self*other+third with no rounding of the\n\ @@ -365,52 +279,6 @@ For example, Decimal('32.100') and Decimal('0.321000e+2') both normalize\n\ to the equivalent value Decimal('32.1').\n\ \n"); -PyDoc_STRVAR(doc_number_class, -"number_class($self, /, context=None)\n--\n\n\ -Return a string describing the class of the operand. The returned value\n\ -is one of the following ten strings:\n\ -\n\ - * '-Infinity', indicating that the operand is negative infinity.\n\ - * '-Normal', indicating that the operand is a negative normal number.\n\ - * '-Subnormal', indicating that the operand is negative and subnormal.\n\ - * '-Zero', indicating that the operand is a negative zero.\n\ - * '+Zero', indicating that the operand is a positive zero.\n\ - * '+Subnormal', indicating that the operand is positive and subnormal.\n\ - * '+Normal', indicating that the operand is a positive normal number.\n\ - * '+Infinity', indicating that the operand is positive infinity.\n\ - * 'NaN', indicating that the operand is a quiet NaN (Not a Number).\n\ - * 'sNaN', indicating that the operand is a signaling NaN.\n\ -\n\ -\n"); - -PyDoc_STRVAR(doc_quantize, -"quantize($self, /, exp, rounding=None, context=None)\n--\n\n\ -Return a value equal to the first operand after rounding and having the\n\ -exponent of the second operand.\n\ -\n\ - >>> Decimal('1.41421356').quantize(Decimal('1.000'))\n\ - Decimal('1.414')\n\ -\n\ -Unlike other operations, if the length of the coefficient after the quantize\n\ -operation would be greater than precision, then an InvalidOperation is signaled.\n\ -This guarantees that, unless there is an error condition, the quantized exponent\n\ -is always equal to that of the right-hand operand.\n\ -\n\ -Also unlike other operations, quantize never signals Underflow, even if the\n\ -result is subnormal and inexact.\n\ -\n\ -If the exponent of the second operand is larger than that of the first, then\n\ -rounding may be necessary. In this case, the rounding mode is determined by the\n\ -rounding argument if given, else by the given context argument; if neither\n\ -argument is given, the rounding mode of the current thread's context is used.\n\ -\n"); - -PyDoc_STRVAR(doc_radix, -"radix($self, /)\n--\n\n\ -Return Decimal(10), the radix (base) in which the Decimal class does\n\ -all its arithmetic. Included for compatibility with the specification.\n\ -\n"); - PyDoc_STRVAR(doc_remainder_near, "remainder_near($self, /, other, context=None)\n--\n\n\ Return the remainder from dividing self by other. This differs from\n\ @@ -434,15 +302,6 @@ length precision if necessary. The sign and exponent of the first operand are\n\ unchanged.\n\ \n"); -PyDoc_STRVAR(doc_same_quantum, -"same_quantum($self, /, other, context=None)\n--\n\n\ -Test whether self and other have the same exponent or whether both are NaN.\n\ -\n\ -This operation is unaffected by context and is quiet: no flags are changed\n\ -and no rounding is performed. As an exception, the C version may raise\n\ -InvalidOperation if the second operand cannot be converted exactly.\n\ -\n"); - PyDoc_STRVAR(doc_scaleb, "scaleb($self, /, other, context=None)\n--\n\n\ Return the first operand with the exponent adjusted the second. Equivalently,\n\ @@ -467,38 +326,6 @@ Return the square root of the argument to full precision. The result is\n\ correctly rounded using the ROUND_HALF_EVEN rounding mode.\n\ \n"); -PyDoc_STRVAR(doc_to_eng_string, -"to_eng_string($self, /, context=None)\n--\n\n\ -Convert to an engineering-type string. Engineering notation has an exponent\n\ -which is a multiple of 3, so there are up to 3 digits left of the decimal\n\ -place. For example, Decimal('123E+1') is converted to Decimal('1.23E+3').\n\ -\n\ -The value of context.capitals determines whether the exponent sign is lower\n\ -or upper case. Otherwise, the context does not affect the operation.\n\ -\n"); - -PyDoc_STRVAR(doc_to_integral, -"to_integral($self, /, rounding=None, context=None)\n--\n\n\ -Identical to the to_integral_value() method. The to_integral() name has been\n\ -kept for compatibility with older versions.\n\ -\n"); - -PyDoc_STRVAR(doc_to_integral_exact, -"to_integral_exact($self, /, rounding=None, context=None)\n--\n\n\ -Round to the nearest integer, signaling Inexact or Rounded as appropriate if\n\ -rounding occurs. The rounding mode is determined by the rounding parameter\n\ -if given, else by the given context. If neither parameter is given, then the\n\ -rounding mode of the current default context is used.\n\ -\n"); - -PyDoc_STRVAR(doc_to_integral_value, -"to_integral_value($self, /, rounding=None, context=None)\n--\n\n\ -Round to the nearest integer without signaling Inexact or Rounded. The\n\ -rounding mode is determined by the rounding parameter if given, else by\n\ -the given context. If neither parameter is given, then the rounding mode\n\ -of the current default context is used.\n\ -\n"); - /******************************************************************************/ /* Context Object and Methods */ diff --git a/Modules/_hashlib/hashlib_buffer.c b/Modules/_hashlib/hashlib_buffer.c deleted file mode 100644 index 032f93ad53ad1b..00000000000000 --- a/Modules/_hashlib/hashlib_buffer.c +++ /dev/null @@ -1,65 +0,0 @@ -#include "hashlib_buffer.h" - -int -_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string) -{ - if (data != NULL && string == NULL) { - // called as H(data) or H(data=...) - *res = data; - return 1; - } - else if (data == NULL && string != NULL) { - // called as H(string=...) - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "the 'string' keyword parameter is deprecated since " - "Python 3.15 and slated for removal in Python 3.19; " - "use the 'data' keyword parameter or pass the data " - "to hash as a positional argument instead", 1) < 0) - { - *res = NULL; - return -1; - } - *res = string; - return 1; - } - else if (data == NULL && string == NULL) { - // fast path when no data is given - assert(!PyErr_Occurred()); - *res = NULL; - return 0; - } - else { - // called as H(data=..., string) - *res = NULL; - PyErr_SetString(PyExc_TypeError, - "'data' and 'string' are mutually exclusive " - "and support for 'string' keyword parameter " - "is slated for removal in a future version."); - return -1; - } -} - -int -_Py_hashlib_get_buffer_view(PyObject *obj, Py_buffer *view) -{ - if (PyUnicode_Check(obj)) { - PyErr_SetString(PyExc_TypeError, - "Strings must be encoded before hashing"); - return -1; - } - if (!PyObject_CheckBuffer(obj)) { - PyErr_SetString(PyExc_TypeError, - "object supporting the buffer API required"); - return -1; - } - if (PyObject_GetBuffer(obj, view, PyBUF_SIMPLE) == -1) { - return -1; - } - if (view->ndim > 1) { - PyErr_SetString(PyExc_BufferError, - "Buffer must be single dimension"); - PyBuffer_Release(view); - return -1; - } - return 0; -} diff --git a/Modules/_hashlib/hashlib_buffer.h b/Modules/_hashlib/hashlib_buffer.h deleted file mode 100644 index 809f19884f41b7..00000000000000 --- a/Modules/_hashlib/hashlib_buffer.h +++ /dev/null @@ -1,48 +0,0 @@ -#ifndef _HASHLIB_HASHLIB_BUFFER_H -#define _HASHLIB_HASHLIB_BUFFER_H - -#include "Python.h" - -/* - * Allow to use the 'data' or 'string' keyword in hashlib.new() - * and other hash functions named constructors. - * - * - If 'data' and 'string' are both non-NULL, set an exception and return -1. - * - If 'data' and 'string' are both NULL, set '*res' to NULL and return 0. - * - Otherwise, set '*res' to 'data' or 'string' and return 1. A deprecation - * warning is set when 'string' is specified. - * - * The symbol is exported for '_hashlib' and HACL*-based extension modules. - */ -PyAPI_FUNC(int) -_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string); - -/* - * Obtain a buffer view from a buffer-like object 'obj'. - * - * On success, store the result in 'view' and return 0. - * On error, set an exception and return -1. - * - * The symbol is exported for '_hashlib' and HACL*-based extension modules. - */ -PyAPI_FUNC(int) -_Py_hashlib_get_buffer_view(PyObject *obj, Py_buffer *view); - -/* - * Call _Py_hashlib_get_buffer_view() and check if it succeeded. - * - * On error, set an exception and execute the ERRACTION statements. - */ -#define GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, ERRACTION) \ - do { \ - if (_Py_hashlib_get_buffer_view(OBJ, VIEW) < 0) { \ - assert(PyErr_Occurred()); \ - ERRACTION; \ - } \ - } while (0) - -/* Specialization of GET_BUFFER_VIEW_OR_ERROR() returning NULL on error. */ -#define GET_BUFFER_VIEW_OR_ERROUT(OBJ, VIEW) \ - GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, return NULL) - -#endif // !_HASHLIB_HASHLIB_BUFFER_H diff --git a/Modules/_hashlib/hashlib_fetch.h b/Modules/_hashlib/hashlib_fetch.h deleted file mode 100644 index 09add71e0c798c..00000000000000 --- a/Modules/_hashlib/hashlib_fetch.h +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Utilities used when fetching a message digest from a digest-like identifier. - */ - -#ifndef _HASHLIB_HASHLIB_FETCH_H -#define _HASHLIB_HASHLIB_FETCH_H - -#include "Python.h" - -/* - * Internal error messages used for reporting an unsupported hash algorithm. - * The algorithm can be given by its name, a callable or a PEP-247 module. - * The same message is raised by Lib/hashlib.py::__get_builtin_constructor() - * and _hmacmodule.c::find_hash_info(). - */ -#define _Py_HASHLIB_UNSUPPORTED_ALGORITHM "unsupported hash algorithm %S" -#define _Py_HASHLIB_UNSUPPORTED_STR_ALGORITHM "unsupported hash algorithm %s" - -#endif // !_HASHLIB_HASHLIB_FETCH_H diff --git a/Modules/_hashlib/hashlib_mutex.h b/Modules/_hashlib/hashlib_mutex.h deleted file mode 100644 index d6924a2ef61e81..00000000000000 --- a/Modules/_hashlib/hashlib_mutex.h +++ /dev/null @@ -1,82 +0,0 @@ -#ifndef _HASHLIB_HASHLIB_MUTEX_H -#define _HASHLIB_HASHLIB_MUTEX_H - -#include "Python.h" -#include "pycore_lock.h" // PyMutex - -/* - * Message length above which the GIL is to be released - * when performing hashing operations. - */ -#define HASHLIB_GIL_MINSIZE 2048 - -/* - * Helper code to synchronize access to the hash object when the GIL is - * released around a CPU consuming hashlib operation. - * - * Code accessing a mutable part of the hash object must be enclosed in - * an HASHLIB_{ACQUIRE,RELEASE}_LOCK block or explicitly acquire and release - * the mutex inside a Py_BEGIN_ALLOW_THREADS -- Py_END_ALLOW_THREADS block if - * they wish to release the GIL for an operation. - */ - -#define HASHLIB_OBJECT_HEAD \ - PyObject_HEAD \ - /* Guard against race conditions during incremental update(). */ \ - PyMutex mutex; - -#define HASHLIB_INIT_MUTEX(OBJ) \ - do { \ - (OBJ)->mutex = (PyMutex){0}; \ - } while (0) - -#define HASHLIB_ACQUIRE_LOCK(OBJ) PyMutex_Lock(&(OBJ)->mutex) -#define HASHLIB_RELEASE_LOCK(OBJ) PyMutex_Unlock(&(OBJ)->mutex) - -// Macros for executing code while conditionally holding the GIL. -// -// These only drop the GIL if the lock acquisition itself is likely to -// block. Thus the non-blocking acquire gating the GIL release for a -// blocking lock acquisition. The intent of these macros is to surround -// the assumed always "fast" operations that you aren't releasing the -// GIL around. - -/* - * Execute a suite of C statements 'STATEMENTS'. - * - * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold. - */ -#define HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED(SIZE, STATEMENTS) \ - do { \ - if ((SIZE) > HASHLIB_GIL_MINSIZE) { \ - Py_BEGIN_ALLOW_THREADS \ - STATEMENTS; \ - Py_END_ALLOW_THREADS \ - } \ - else { \ - STATEMENTS; \ - } \ - } while (0) - -/* - * Lock 'OBJ' and execute a suite of C statements 'STATEMENTS'. - * - * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold. - */ -#define HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED(OBJ, SIZE, STATEMENTS) \ - do { \ - if ((SIZE) > HASHLIB_GIL_MINSIZE) { \ - Py_BEGIN_ALLOW_THREADS \ - HASHLIB_ACQUIRE_LOCK(OBJ); \ - STATEMENTS; \ - HASHLIB_RELEASE_LOCK(OBJ); \ - Py_END_ALLOW_THREADS \ - } \ - else { \ - HASHLIB_ACQUIRE_LOCK(OBJ); \ - STATEMENTS; \ - HASHLIB_RELEASE_LOCK(OBJ); \ - } \ - } while (0) - -#endif // !_HASHLIB_HASHLIB_MUTEX_H diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 26412cb62430c9..00f98c090b3952 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -24,17 +24,14 @@ #include "Python.h" #include "pycore_hashtable.h" -#include "pycore_strhex.h" // _Py_strhex() -#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_RELAXED - -#include "_hashlib/hashlib_buffer.h" -#include "_hashlib/hashlib_fetch.h" -#include "_hashlib/hashlib_mutex.h" +#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_RELAXED +#include "hashlib.h" /* EVP is the preferred interface to hashing in OpenSSL */ #include #include -#include // FIPS_mode() +#include // FIPS_mode() /* We use the object interface to discover what hashes OpenSSL supports. */ #include #include @@ -535,7 +532,7 @@ raise_unsupported_algorithm_error(_hashlibstate *state, PyObject *digestmod) { raise_unsupported_algorithm_impl( state->unsupported_digestmod_error, - _Py_HASHLIB_UNSUPPORTED_ALGORITHM, + HASHLIB_UNSUPPORTED_ALGORITHM, digestmod ); } @@ -545,7 +542,7 @@ raise_unsupported_str_algorithm_error(_hashlibstate *state, const char *name) { raise_unsupported_algorithm_impl( state->unsupported_digestmod_error, - _Py_HASHLIB_UNSUPPORTED_STR_ALGORITHM, + HASHLIB_UNSUPPORTED_STR_ALGORITHM, name ); } diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index faf3b25b68c4eb..dc79cb088f94e5 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -20,12 +20,18 @@ #include "_interpreters_common.h" +#include "clinic/_interpretersmodule.c.h" #define MODULE_NAME _interpreters #define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME) #define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME) +/*[clinic input] +module _interpreters +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=bfd967980a0de892]*/ + static PyInterpreterState * _get_current_interp(void) { @@ -797,12 +803,12 @@ get_summary(PyInterpreterState *interp) } +// Not converted to Argument Clinic because the function uses ``**kwargs``. static PyObject * interp_new_config(PyObject *self, PyObject *args, PyObject *kwds) { const char *name = NULL; - if (!PyArg_ParseTuple(args, "|s:" MODULE_NAME_STR ".new_config", - &name)) + if (!PyArg_ParseTuple(args, "|s:" MODULE_NAME_STR ".new_config", &name)) { return NULL; } @@ -830,7 +836,8 @@ interp_new_config(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(new_config_doc, -"new_config(name='isolated', /, **overrides) -> type.SimpleNamespace\n\ +"new_config($module, name='isolated', /, **overrides)\n\ +--\n\ \n\ Return a representation of a new PyInterpreterConfig.\n\ \n\ @@ -841,17 +848,28 @@ Any keyword arguments are set on the corresponding config fields,\n\ overriding the initial values."); +/*[clinic input] +_interpreters.create + config as configobj: object(py_default="'isolated'") = NULL + * + reqrefs: bool = False + +Create a new interpreter and return a unique generated ID. + +The caller is responsible for destroying the interpreter before exiting, +typically by using _interpreters.destroy(). This can be managed +automatically by passing "reqrefs=True" and then using _incref() and +_decref() appropriately. + +"config" must be a valid interpreter config or the name of a +predefined config ('isolated' or 'legacy'). The default +is 'isolated'. +[clinic start generated code]*/ + static PyObject * -interp_create(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_create_impl(PyObject *module, PyObject *configobj, int reqrefs) +/*[clinic end generated code: output=c1cc6835b1277c16 input=235ce396a23624d5]*/ { - static char *kwlist[] = {"config", "reqrefs", NULL}; - PyObject *configobj = NULL; - int reqrefs = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O$p:create", kwlist, - &configobj, &reqrefs)) { - return NULL; - } - PyInterpreterConfig config; if (config_from_object(configobj, &config) < 0) { return NULL; @@ -885,34 +903,22 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) } -PyDoc_STRVAR(create_doc, -"create([config], *, reqrefs=False) -> ID\n\ -\n\ -Create a new interpreter and return a unique generated ID.\n\ -\n\ -The caller is responsible for destroying the interpreter before exiting,\n\ -typically by using _interpreters.destroy(). This can be managed \n\ -automatically by passing \"reqrefs=True\" and then using _incref() and\n\ -_decref() appropriately.\n\ -\n\ -\"config\" must be a valid interpreter config or the name of a\n\ -predefined config (\"isolated\" or \"legacy\"). The default\n\ -is \"isolated\"."); +/*[clinic input] +_interpreters.destroy + id: object + * + restrict as restricted: bool = False + +Destroy the identified interpreter. +Attempting to destroy the current interpreter raises InterpreterError. +So does an unrecognized ID. +[clinic start generated code]*/ static PyObject * -interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_destroy_impl(PyObject *module, PyObject *id, int restricted) +/*[clinic end generated code: output=0bc20da8700ab4dd input=561bdd6537639d40]*/ { - static char *kwlist[] = {"id", "restrict", NULL}; - PyObject *id; - int restricted = 0; - // XXX Use "L" for id? - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|$p:destroy", kwlist, &id, &restricted)) - { - return NULL; - } - // Look up the interpreter. int reqready = 0; PyInterpreterState *interp = \ @@ -946,27 +952,19 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) Py_RETURN_NONE; } -PyDoc_STRVAR(destroy_doc, -"destroy(id, *, restrict=False)\n\ -\n\ -Destroy the identified interpreter.\n\ -\n\ -Attempting to destroy the current interpreter raises InterpreterError.\n\ -So does an unrecognized ID."); +/*[clinic input] +_interpreters.list_all + * + require_ready as reqready: bool = False + +Return a list containing the ID of every existing interpreter. +[clinic start generated code]*/ static PyObject * -interp_list_all(PyObject *self, PyObject *args, PyObject *kwargs) +_interpreters_list_all_impl(PyObject *module, int reqready) +/*[clinic end generated code: output=3f21c1a7c78043c0 input=35bae91c381a2cf9]*/ { - static char *kwlist[] = {"require_ready", NULL}; - int reqready = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "|$p:" MODULE_NAME_STR ".list_all", - kwlist, &reqready)) - { - return NULL; - } - PyObject *ids = PyList_New(0); if (ids == NULL) { return NULL; @@ -995,14 +993,16 @@ interp_list_all(PyObject *self, PyObject *args, PyObject *kwargs) return ids; } -PyDoc_STRVAR(list_all_doc, -"list_all() -> [(ID, whence)]\n\ -\n\ -Return a list containing the ID of every existing interpreter."); +/*[clinic input] +_interpreters.get_current + +Return (ID, whence) of the current interpreter. +[clinic start generated code]*/ static PyObject * -interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) +_interpreters_get_current_impl(PyObject *module) +/*[clinic end generated code: output=03161c8fcc0136eb input=37fb2c067c14d543]*/ { PyInterpreterState *interp =_get_current_interp(); if (interp == NULL) { @@ -1012,39 +1012,38 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) return get_summary(interp); } -PyDoc_STRVAR(get_current_doc, -"get_current() -> (ID, whence)\n\ -\n\ -Return the ID of current interpreter."); +/*[clinic input] +_interpreters.get_main + +Return (ID, whence) of the main interpreter. +[clinic start generated code]*/ static PyObject * -interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored)) +_interpreters_get_main_impl(PyObject *module) +/*[clinic end generated code: output=9647288aff735557 input=b4ace23ca562146f]*/ { PyInterpreterState *interp = _PyInterpreterState_Main(); assert(_PyInterpreterState_IsReady(interp)); return get_summary(interp); } -PyDoc_STRVAR(get_main_doc, -"get_main() -> (ID, whence)\n\ -\n\ -Return the ID of main interpreter."); +/*[clinic input] +_interpreters.set___main___attrs + id: object + updates: object(subclass_of='&PyDict_Type') + * + restrict as restricted: bool = False + +Bind the given attributes in the interpreter's __main__ module. +[clinic start generated code]*/ static PyObject * -interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs) +_interpreters_set___main___attrs_impl(PyObject *module, PyObject *id, + PyObject *updates, int restricted) +/*[clinic end generated code: output=f3803010cb452bf0 input=d16ab8d81371f86a]*/ { - static char *kwlist[] = {"id", "updates", "restrict", NULL}; - PyObject *id, *updates; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "OO!|$p:" MODULE_NAME_STR ".set___main___attrs", - kwlist, &id, &PyDict_Type, &updates, &restricted)) - { - return NULL; - } - // Look up the interpreter. int reqready = 1; PyInterpreterState *interp = \ @@ -1091,11 +1090,6 @@ interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs) Py_RETURN_NONE; } -PyDoc_STRVAR(set___main___attrs_doc, -"set___main___attrs(id, ns, *, restrict=False)\n\ -\n\ -Bind the given attributes in the interpreter's __main__ module."); - static PyObject * _handle_script_error(struct run_result *runres) @@ -1109,23 +1103,36 @@ _handle_script_error(struct run_result *runres) return runres->excinfo; } +/*[clinic input] +_interpreters.exec + id: object + code: object + shared: object(subclass_of='&PyDict_Type', c_default='NULL') = {} + * + restrict as restricted: bool = False + +Execute the provided code in the identified interpreter. + +This is equivalent to running the builtin exec() under the target +interpreter, using the __dict__ of its __main__ module as both +globals and locals. + +"code" may be a string containing the text of a Python script. + +Functions (and code objects) are also supported, with some restrictions. +The code/function must not take any arguments or be a closure +(i.e. have cell vars). Methods and other callables are not supported. + +If a function is provided, its code object is used and all its state +is ignored, including its __globals__ dict. +[clinic start generated code]*/ + static PyObject * -interp_exec(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_exec_impl(PyObject *module, PyObject *id, PyObject *code, + PyObject *shared, int restricted) +/*[clinic end generated code: output=492057c4f10dc304 input=5a22c1ed0c5dbcf3]*/ { -#define FUNCNAME MODULE_NAME_STR ".exec" PyThreadState *tstate = _PyThreadState_GET(); - static char *kwlist[] = {"id", "code", "shared", "restrict", NULL}; - PyObject *id, *code; - PyObject *shared = NULL; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O!$p:" FUNCNAME, kwlist, - &id, &code, &PyDict_Type, &shared, - &restricted)) - { - return NULL; - } - int reqready = 1; PyInterpreterState *interp = \ resolve_interp(id, restricted, reqready, "exec code for"); @@ -1150,43 +1157,29 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) } assert(runres.result == NULL); Py_RETURN_NONE; -#undef FUNCNAME } -PyDoc_STRVAR(exec_doc, -"exec(id, code, shared=None, *, restrict=False)\n\ -\n\ -Execute the provided code in the identified interpreter.\n\ -This is equivalent to running the builtin exec() under the target\n\ -interpreter, using the __dict__ of its __main__ module as both\n\ -globals and locals.\n\ -\n\ -\"code\" may be a string containing the text of a Python script.\n\ -\n\ -Functions (and code objects) are also supported, with some restrictions.\n\ -The code/function must not take any arguments or be a closure\n\ -(i.e. have cell vars). Methods and other callables are not supported.\n\ -\n\ -If a function is provided, its code object is used and all its state\n\ -is ignored, including its __globals__ dict."); +/*[clinic input] +_interpreters.run_string + id: object + script: unicode + shared: object(subclass_of='&PyDict_Type', c_default='NULL') = {} + * + restrict as restricted: bool = False + +Execute the provided string in the identified interpreter. + +(See _interpreters.exec().) +[clinic start generated code]*/ static PyObject * -interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_run_string_impl(PyObject *module, PyObject *id, + PyObject *script, PyObject *shared, + int restricted) +/*[clinic end generated code: output=a30a64fb9ad396a2 input=51ce549b9a8dbe21]*/ { #define FUNCNAME MODULE_NAME_STR ".run_string" PyThreadState *tstate = _PyThreadState_GET(); - static char *kwlist[] = {"id", "script", "shared", "restrict", NULL}; - PyObject *id, *script; - PyObject *shared = NULL; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OU|O!$p:" FUNCNAME, kwlist, - &id, &script, &PyDict_Type, &shared, - &restricted)) - { - return NULL; - } - int reqready = 1; PyInterpreterState *interp = \ resolve_interp(id, restricted, reqready, "run a string in"); @@ -1217,30 +1210,29 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) #undef FUNCNAME } -PyDoc_STRVAR(run_string_doc, -"run_string(id, script, shared=None, *, restrict=False)\n\ -\n\ -Execute the provided string in the identified interpreter.\n\ -\n\ -(See " MODULE_NAME_STR ".exec()."); +/*[clinic input] +_interpreters.run_func + id: object + func: object + shared: object(subclass_of='&PyDict_Type', c_default='NULL') = {} + * + restrict as restricted: bool = False + +Execute the body of the provided function in the identified interpreter. + +Code objects are also supported. In both cases, closures and args +are not supported. Methods and other callables are not supported either. + +(See _interpreters.exec().) +[clinic start generated code]*/ static PyObject * -interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_run_func_impl(PyObject *module, PyObject *id, PyObject *func, + PyObject *shared, int restricted) +/*[clinic end generated code: output=131f7202ca4a0c5e input=2d62bb9b9eaf4948]*/ { #define FUNCNAME MODULE_NAME_STR ".run_func" PyThreadState *tstate = _PyThreadState_GET(); - static char *kwlist[] = {"id", "func", "shared", "restrict", NULL}; - PyObject *id, *func; - PyObject *shared = NULL; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O!$p:" FUNCNAME, kwlist, - &id, &func, &PyDict_Type, &shared, - &restricted)) - { - return NULL; - } - int reqready = 1; PyInterpreterState *interp = \ resolve_interp(id, restricted, reqready, "run a function in"); @@ -1280,37 +1272,28 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) #undef FUNCNAME } -PyDoc_STRVAR(run_func_doc, -"run_func(id, func, shared=None, *, restrict=False)\n\ -\n\ -Execute the body of the provided function in the identified interpreter.\n\ -Code objects are also supported. In both cases, closures and args\n\ -are not supported. Methods and other callables are not supported either.\n\ -\n\ -(See " MODULE_NAME_STR ".exec()."); +/*[clinic input] +_interpreters.call + id: object + callable: object + args: object(subclass_of='&PyTuple_Type', c_default='NULL') = () + kwargs: object(subclass_of='&PyDict_Type', c_default='NULL') = {} + * + preserve_exc: bool = False + restrict as restricted: bool = False + +Call the provided object in the identified interpreter. + +Pass the given args and kwargs, if possible. +[clinic start generated code]*/ static PyObject * -interp_call(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_call_impl(PyObject *module, PyObject *id, PyObject *callable, + PyObject *args, PyObject *kwargs, int preserve_exc, + int restricted) +/*[clinic end generated code: output=b7a4a27d72df3ebc input=b026d0b212a575e6]*/ { -#define FUNCNAME MODULE_NAME_STR ".call" PyThreadState *tstate = _PyThreadState_GET(); - static char *kwlist[] = {"id", "callable", "args", "kwargs", - "preserve_exc", "restrict", NULL}; - PyObject *id, *callable; - PyObject *args_obj = NULL; - PyObject *kwargs_obj = NULL; - int preserve_exc = 0; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O!O!$pp:" FUNCNAME, kwlist, - &id, &callable, - &PyTuple_Type, &args_obj, - &PyDict_Type, &kwargs_obj, - &preserve_exc, &restricted)) - { - return NULL; - } - int reqready = 1; PyInterpreterState *interp = \ resolve_interp(id, restricted, reqready, "make a call in"); @@ -1319,7 +1302,7 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) } struct interp_call call = {0}; - if (_interp_call_pack(tstate, &call, callable, args_obj, kwargs_obj) < 0) { + if (_interp_call_pack(tstate, &call, callable, args, kwargs) < 0) { return NULL; } @@ -1341,26 +1324,20 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) _interp_call_clear(&call); _run_result_clear(&runres); return res_and_exc; -#undef FUNCNAME } -PyDoc_STRVAR(call_doc, -"call(id, callable, args=None, kwargs=None, *, restrict=False)\n\ -\n\ -Call the provided object in the identified interpreter.\n\ -Pass the given args and kwargs, if possible."); +/*[clinic input] +_interpreters.is_shareable + obj: object + +Return True if the object's data may be shared between interpreters and False otherwise. +[clinic start generated code]*/ static PyObject * -object_is_shareable(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_is_shareable_impl(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=227856926a22940b input=72b9a36bdf1d2a53]*/ { - static char *kwlist[] = {"obj", NULL}; - PyObject *obj; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:is_shareable", kwlist, &obj)) { - return NULL; - } - PyThreadState *tstate = _PyThreadState_GET(); if (_PyObject_CheckXIData(tstate, obj) == 0) { Py_RETURN_TRUE; @@ -1369,26 +1346,20 @@ object_is_shareable(PyObject *self, PyObject *args, PyObject *kwds) Py_RETURN_FALSE; } -PyDoc_STRVAR(is_shareable_doc, -"is_shareable(obj) -> bool\n\ -\n\ -Return True if the object's data may be shared between interpreters and\n\ -False otherwise."); +/*[clinic input] +_interpreters.is_running + id: object + * + restrict as restricted: bool = False + +Return whether or not the identified interpreter is running. +[clinic start generated code]*/ static PyObject * -interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_is_running_impl(PyObject *module, PyObject *id, int restricted) +/*[clinic end generated code: output=32a6225d5ded9bdb input=3291578d04231125]*/ { - static char *kwlist[] = {"id", "restrict", NULL}; - PyObject *id; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|$p:is_running", kwlist, - &id, &restricted)) - { - return NULL; - } - int reqready = 1; PyInterpreterState *interp = \ resolve_interp(id, restricted, reqready, "check if running for"); @@ -1402,31 +1373,27 @@ interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) Py_RETURN_FALSE; } -PyDoc_STRVAR(is_running_doc, -"is_running(id, *, restrict=False) -> bool\n\ -\n\ -Return whether or not the identified interpreter is running."); +/*[clinic input] +_interpreters.get_config + id: object + * + restrict as restricted: bool = False + +Return a representation of the config used to initialize the interpreter. +[clinic start generated code]*/ static PyObject * -interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_get_config_impl(PyObject *module, PyObject *id, int restricted) +/*[clinic end generated code: output=56773353b9b7224a input=59519a01c22d96d1]*/ { - static char *kwlist[] = {"id", "restrict", NULL}; - PyObject *idobj = NULL; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|$p:get_config", kwlist, - &idobj, &restricted)) - { - return NULL; - } - if (idobj == Py_None) { - idobj = NULL; + if (id == Py_None) { + id = NULL; } int reqready = 0; PyInterpreterState *interp = \ - resolve_interp(idobj, restricted, reqready, "get the config of"); + resolve_interp(id, restricted, reqready, "get the config of"); if (interp == NULL) { return NULL; } @@ -1445,23 +1412,18 @@ interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) return configobj; } -PyDoc_STRVAR(get_config_doc, -"get_config(id, *, restrict=False) -> types.SimpleNamespace\n\ -\n\ -Return a representation of the config used to initialize the interpreter."); +/*[clinic input] +_interpreters.whence + id: object + +Return an identifier for where the interpreter was created. +[clinic start generated code]*/ static PyObject * -interp_whence(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_whence_impl(PyObject *module, PyObject *id) +/*[clinic end generated code: output=ef2c21ab106c2c20 input=eeede0a2fbfa2968]*/ { - static char *kwlist[] = {"id", NULL}; - PyObject *id; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:whence", kwlist, &id)) - { - return NULL; - } - PyInterpreterState *interp = look_up_interp(id); if (interp == NULL) { return NULL; @@ -1471,26 +1433,21 @@ interp_whence(PyObject *self, PyObject *args, PyObject *kwds) return PyLong_FromLong(whence); } -PyDoc_STRVAR(whence_doc, -"whence(id) -> int\n\ -\n\ -Return an identifier for where the interpreter was created."); +/*[clinic input] +_interpreters.incref + id: object + * + implieslink: bool = False + restrict as restricted: bool = False + +[clinic start generated code]*/ static PyObject * -interp_incref(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_incref_impl(PyObject *module, PyObject *id, int implieslink, + int restricted) +/*[clinic end generated code: output=eccaa4e03fbe8ee2 input=a0a614748f2e348c]*/ { - static char *kwlist[] = {"id", "implieslink", "restrict", NULL}; - PyObject *id; - int implieslink = 0; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|$pp:incref", kwlist, - &id, &implieslink, &restricted)) - { - return NULL; - } - int reqready = 1; PyInterpreterState *interp = \ resolve_interp(id, restricted, reqready, "incref"); @@ -1508,18 +1465,18 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) } +/*[clinic input] +_interpreters.decref + id: object + * + restrict as restricted: bool = False + +[clinic start generated code]*/ + static PyObject * -interp_decref(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_decref_impl(PyObject *module, PyObject *id, int restricted) +/*[clinic end generated code: output=5c54db4b22086171 input=c4aa34f09c44e62a]*/ { - static char *kwlist[] = {"id", "restrict", NULL}; - PyObject *id; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|$p:decref", kwlist, &id, &restricted)) - { - return NULL; - } - int reqready = 1; PyInterpreterState *interp = \ resolve_interp(id, restricted, reqready, "decref"); @@ -1533,18 +1490,20 @@ interp_decref(PyObject *self, PyObject *args, PyObject *kwds) } +/*[clinic input] +_interpreters.capture_exception + exc as exc_arg: object = None + +Return a snapshot of an exception. + +If "exc" is None then the current exception, if any, is used (but not cleared). +The returned snapshot is the same as what _interpreters.exec() returns. +[clinic start generated code]*/ + static PyObject * -capture_exception(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_capture_exception_impl(PyObject *module, PyObject *exc_arg) +/*[clinic end generated code: output=ef3f5393ef9c88a6 input=32045341e979bc9e]*/ { - static char *kwlist[] = {"exc", NULL}; - PyObject *exc_arg = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "|O:capture_exception", kwlist, - &exc_arg)) - { - return NULL; - } - PyObject *exc = exc_arg; if (exc == NULL || exc == Py_None) { exc = PyErr_GetRaisedException(); @@ -1592,58 +1551,33 @@ capture_exception(PyObject *self, PyObject *args, PyObject *kwds) return captured; } -PyDoc_STRVAR(capture_exception_doc, -"capture_exception(exc=None) -> types.SimpleNamespace\n\ -\n\ -Return a snapshot of an exception. If \"exc\" is None\n\ -then the current exception, if any, is used (but not cleared).\n\ -\n\ -The returned snapshot is the same as what _interpreters.exec() returns."); - static PyMethodDef module_functions[] = { {"new_config", _PyCFunction_CAST(interp_new_config), METH_VARARGS | METH_KEYWORDS, new_config_doc}, - {"create", _PyCFunction_CAST(interp_create), - METH_VARARGS | METH_KEYWORDS, create_doc}, - {"destroy", _PyCFunction_CAST(interp_destroy), - METH_VARARGS | METH_KEYWORDS, destroy_doc}, - {"list_all", _PyCFunction_CAST(interp_list_all), - METH_VARARGS | METH_KEYWORDS, list_all_doc}, - {"get_current", interp_get_current, - METH_NOARGS, get_current_doc}, - {"get_main", interp_get_main, - METH_NOARGS, get_main_doc}, - - {"is_running", _PyCFunction_CAST(interp_is_running), - METH_VARARGS | METH_KEYWORDS, is_running_doc}, - {"get_config", _PyCFunction_CAST(interp_get_config), - METH_VARARGS | METH_KEYWORDS, get_config_doc}, - {"whence", _PyCFunction_CAST(interp_whence), - METH_VARARGS | METH_KEYWORDS, whence_doc}, - {"exec", _PyCFunction_CAST(interp_exec), - METH_VARARGS | METH_KEYWORDS, exec_doc}, - {"call", _PyCFunction_CAST(interp_call), - METH_VARARGS | METH_KEYWORDS, call_doc}, - {"run_string", _PyCFunction_CAST(interp_run_string), - METH_VARARGS | METH_KEYWORDS, run_string_doc}, - {"run_func", _PyCFunction_CAST(interp_run_func), - METH_VARARGS | METH_KEYWORDS, run_func_doc}, - - {"set___main___attrs", _PyCFunction_CAST(interp_set___main___attrs), - METH_VARARGS | METH_KEYWORDS, set___main___attrs_doc}, - - {"incref", _PyCFunction_CAST(interp_incref), - METH_VARARGS | METH_KEYWORDS, NULL}, - {"decref", _PyCFunction_CAST(interp_decref), - METH_VARARGS | METH_KEYWORDS, NULL}, - - {"is_shareable", _PyCFunction_CAST(object_is_shareable), - METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, - - {"capture_exception", _PyCFunction_CAST(capture_exception), - METH_VARARGS | METH_KEYWORDS, capture_exception_doc}, + _INTERPRETERS_CREATE_METHODDEF + _INTERPRETERS_DESTROY_METHODDEF + _INTERPRETERS_LIST_ALL_METHODDEF + _INTERPRETERS_GET_CURRENT_METHODDEF + _INTERPRETERS_GET_MAIN_METHODDEF + + _INTERPRETERS_IS_RUNNING_METHODDEF + _INTERPRETERS_GET_CONFIG_METHODDEF + _INTERPRETERS_WHENCE_METHODDEF + _INTERPRETERS_EXEC_METHODDEF + _INTERPRETERS_CALL_METHODDEF + _INTERPRETERS_RUN_STRING_METHODDEF + _INTERPRETERS_RUN_FUNC_METHODDEF + + _INTERPRETERS_SET___MAIN___ATTRS_METHODDEF + + _INTERPRETERS_INCREF_METHODDEF + _INTERPRETERS_DECREF_METHODDEF + + _INTERPRETERS_IS_SHAREABLE_METHODDEF + + _INTERPRETERS_CAPTURE_EXCEPTION_METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_json.c b/Modules/_json.c index 7580b589e2d937..e1d6042cb78ab5 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -51,7 +51,7 @@ typedef struct _PyEncoderObject { char sort_keys; char skipkeys; int allow_nan; - PyCFunction fast_encode; + int (*fast_encode)(PyUnicodeWriter *, PyObject *); } PyEncoderObject; #define PyEncoderObject_CAST(op) ((PyEncoderObject *)(op)) @@ -102,8 +102,10 @@ static PyObject * _encoded_const(PyObject *obj); static void raise_errmsg(const char *msg, PyObject *s, Py_ssize_t end); -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj); +static int +_steal_accumulate(PyUnicodeWriter *writer, PyObject *stolen); +static int +encoder_write_string(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *obj); static PyObject * encoder_encode_float(PyEncoderObject *s, PyObject *obj); @@ -146,22 +148,11 @@ ascii_escape_unichar(Py_UCS4 c, unsigned char *output, Py_ssize_t chars) return chars; } -static PyObject * -ascii_escape_unicode(PyObject *pystr) +static Py_ssize_t +ascii_escape_size(const void *input, int kind, Py_ssize_t input_chars) { - /* Take a PyUnicode pystr and return a new ASCII-only escaped PyUnicode */ Py_ssize_t i; - Py_ssize_t input_chars; Py_ssize_t output_size; - Py_ssize_t chars; - PyObject *rval; - const void *input; - Py_UCS1 *output; - int kind; - - input_chars = PyUnicode_GET_LENGTH(pystr); - input = PyUnicode_DATA(pystr); - kind = PyUnicode_KIND(pystr); /* Compute the output size */ for (i = 0, output_size = 2; i < input_chars; i++) { @@ -181,11 +172,22 @@ ascii_escape_unicode(PyObject *pystr) } if (output_size > PY_SSIZE_T_MAX - d) { PyErr_SetString(PyExc_OverflowError, "string is too long to escape"); - return NULL; + return -1; } output_size += d; } + return output_size; +} + +static PyObject * +ascii_escape_unicode_and_size(const void *input, int kind, Py_ssize_t input_chars, Py_ssize_t output_size) +{ + Py_ssize_t i; + Py_ssize_t chars; + PyObject *rval; + Py_UCS1 *output; + rval = PyUnicode_New(output_size, 127); if (rval == NULL) { return NULL; @@ -210,23 +212,62 @@ ascii_escape_unicode(PyObject *pystr) } static PyObject * -escape_unicode(PyObject *pystr) +ascii_escape_unicode(PyObject *pystr) +{ + /* Take a PyUnicode pystr and return a new ASCII-only escaped PyUnicode */ + Py_ssize_t input_chars = PyUnicode_GET_LENGTH(pystr); + const void *input = PyUnicode_DATA(pystr); + int kind = PyUnicode_KIND(pystr); + + Py_ssize_t output_size = ascii_escape_size(input, kind, input_chars); + if (output_size < 0) { + return NULL; + } + + return ascii_escape_unicode_and_size(input, kind, input_chars, output_size); +} + +static int +write_escaped_ascii(PyUnicodeWriter *writer, PyObject *pystr) { - /* Take a PyUnicode pystr and return a new escaped PyUnicode */ - Py_ssize_t i; Py_ssize_t input_chars; - Py_ssize_t output_size; - Py_ssize_t chars; - PyObject *rval; const void *input; int kind; - Py_UCS4 maxchar; - maxchar = PyUnicode_MAX_CHAR_VALUE(pystr); input_chars = PyUnicode_GET_LENGTH(pystr); input = PyUnicode_DATA(pystr); kind = PyUnicode_KIND(pystr); + Py_ssize_t output_size = ascii_escape_size(input, kind, input_chars); + if (output_size < 0) { + return -1; + } + + if (output_size == input_chars + 2) { + /* No need to escape anything */ + if (PyUnicodeWriter_WriteChar(writer, '"') < 0) { + return -1; + } + if (PyUnicodeWriter_WriteStr(writer, pystr) < 0) { + return -1; + } + return PyUnicodeWriter_WriteChar(writer, '"'); + } + + PyObject *rval = ascii_escape_unicode_and_size(input, kind, input_chars, output_size); + if (rval == NULL) { + return -1; + } + + return _steal_accumulate(writer, rval); +} + +static Py_ssize_t +escape_size(const void *input, int kind, Py_ssize_t input_chars) +{ + Py_ssize_t i; + Py_ssize_t output_size; + /* Compute the output size */ for (i = 0, output_size = 2; i < input_chars; i++) { Py_UCS4 c = PyUnicode_READ(kind, input, i); @@ -244,11 +285,21 @@ escape_unicode(PyObject *pystr) } if (output_size > PY_SSIZE_T_MAX - d) { PyErr_SetString(PyExc_OverflowError, "string is too long to escape"); - return NULL; + return -1; } output_size += d; } + return output_size; +} + +static PyObject * +escape_unicode_and_size(const void *input, int kind, Py_UCS4 maxchar, Py_ssize_t input_chars, Py_ssize_t output_size) +{ + Py_ssize_t i; + Py_ssize_t chars; + PyObject *rval; + rval = PyUnicode_New(output_size, maxchar); if (rval == NULL) return NULL; @@ -303,6 +354,55 @@ escape_unicode(PyObject *pystr) return rval; } +static PyObject * +escape_unicode(PyObject *pystr) +{ + /* Take a PyUnicode pystr and return a new escaped PyUnicode */ + Py_ssize_t input_chars = PyUnicode_GET_LENGTH(pystr); + const void *input = PyUnicode_DATA(pystr); + int kind = PyUnicode_KIND(pystr); + Py_UCS4 maxchar = PyUnicode_MAX_CHAR_VALUE(pystr); + + Py_ssize_t output_size = escape_size(input, kind, input_chars); + if (output_size < 0) { + return NULL; + } + + return escape_unicode_and_size(input, kind, maxchar, input_chars, output_size); +} + +static int +write_escaped_unicode(PyUnicodeWriter *writer, PyObject *pystr) +{ + Py_ssize_t input_chars = PyUnicode_GET_LENGTH(pystr); + const void *input = PyUnicode_DATA(pystr); + int kind = PyUnicode_KIND(pystr); + Py_UCS4 maxchar = PyUnicode_MAX_CHAR_VALUE(pystr); + + Py_ssize_t output_size = escape_size(input, kind, input_chars); + if (output_size < 0) { + return -1; + } + + if (output_size == input_chars + 2) { + /* No need to escape anything */ + if (PyUnicodeWriter_WriteChar(writer, '"') < 0) { + return -1; + } + if (PyUnicodeWriter_WriteStr(writer, pystr) < 0) { + return -1; + } + return PyUnicodeWriter_WriteChar(writer, '"'); + } + + PyObject *rval = escape_unicode_and_size(input, kind, maxchar, input_chars, output_size); + if (rval == NULL) { + return -1; + } + + return _steal_accumulate(writer, rval); +} + static void raise_errmsg(const char *msg, PyObject *s, Py_ssize_t end) { @@ -1256,8 +1356,11 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (PyCFunction_Check(s->encoder)) { PyCFunction f = PyCFunction_GetFunction(s->encoder); - if (f == py_encode_basestring_ascii || f == py_encode_basestring) { - s->fast_encode = f; + if (f == py_encode_basestring_ascii) { + s->fast_encode = write_escaped_ascii; + } + else if (f == py_encode_basestring) { + s->fast_encode = write_escaped_unicode; } } @@ -1438,24 +1541,27 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj) return PyFloat_Type.tp_repr(obj); } -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj) +static int +encoder_write_string(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *obj) { /* Return the JSON representation of a string */ PyObject *encoded; if (s->fast_encode) { - return s->fast_encode(NULL, obj); + return s->fast_encode(writer, obj); } encoded = PyObject_CallOneArg(s->encoder, obj); - if (encoded != NULL && !PyUnicode_Check(encoded)) { + if (encoded == NULL) { + return -1; + } + if (!PyUnicode_Check(encoded)) { PyErr_Format(PyExc_TypeError, "encoder() must return a string, not %.80s", Py_TYPE(encoded)->tp_name); Py_DECREF(encoded); - return NULL; + return -1; } - return encoded; + return _steal_accumulate(writer, encoded); } static int @@ -1486,10 +1592,7 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, return PyUnicodeWriter_WriteASCII(writer, "false", 5); } else if (PyUnicode_Check(obj)) { - PyObject *encoded = encoder_encode_string(s, obj); - if (encoded == NULL) - return -1; - return _steal_accumulate(writer, encoded); + return encoder_write_string(s, writer, obj); } else if (PyLong_Check(obj)) { if (PyLong_CheckExact(obj)) { @@ -1578,7 +1681,7 @@ encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *firs PyObject *item_separator) { PyObject *keystr = NULL; - PyObject *encoded; + int rv; if (PyUnicode_Check(key)) { keystr = Py_NewRef(key); @@ -1624,13 +1727,10 @@ encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *firs } } - encoded = encoder_encode_string(s, keystr); + rv = encoder_write_string(s, writer, keystr); Py_DECREF(keystr); - if (encoded == NULL) { - return -1; - } - if (_steal_accumulate(writer, encoded) < 0) { + if (rv < 0) { return -1; } if (PyUnicodeWriter_WriteStr(writer, s->key_separator) < 0) { diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 41e6d48b1dbd9b..17b5220fd6f9e1 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -87,6 +87,41 @@ copy_grouping(const char* s) return result; } +#if defined(MS_WINDOWS) + +// 16 is the number of elements in the szCodePage field +// of the __crt_locale_strings structure. +#define MAX_CP_LEN 15 + +static int +check_locale_name(const char *locale, const char *end) +{ + size_t len = end ? (size_t)(end - locale) : strlen(locale); + const char *dot = memchr(locale, '.', len); + if (dot && locale + len - dot - 1 > MAX_CP_LEN) { + return -1; + } + return 0; +} + +static int +check_locale_name_all(const char *locale) +{ + const char *start = locale; + while (1) { + const char *end = strchr(start, ';'); + if (check_locale_name(start, end) < 0) { + return -1; + } + if (end == NULL) { + break; + } + start = end + 1; + } + return 0; +} +#endif + /*[clinic input] _locale.setlocale @@ -111,6 +146,18 @@ _locale_setlocale_impl(PyObject *module, int category, const char *locale) "invalid locale category"); return NULL; } + if (locale) { + if ((category == LC_ALL + ? check_locale_name_all(locale) + : check_locale_name(locale, NULL)) < 0) + { + /* Debug assertion failure on Windows. + * _Py_BEGIN_SUPPRESS_IPH/_Py_END_SUPPRESS_IPH do not help. */ + PyErr_SetString(get_locale_state(module)->Error, + "unsupported locale setting"); + return NULL; + } + } #endif if (locale) { diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index b50e5e403a1a19..a8542be19dc3dd 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -75,7 +75,7 @@ #endif #define INTERP_STATE_BUFFER_SIZE MAX(INTERP_STATE_MIN_SIZE, 256) - +#define MAX_TLBC_SIZE 2048 // Copied from Modules/_asynciomodule.c because it's not exported @@ -1568,15 +1568,34 @@ cache_tlbc_array(RemoteUnwinderObject *unwinder, uintptr_t code_addr, uintptr_t TLBCCacheEntry *entry = NULL; // Read the TLBC array pointer - if (read_ptr(unwinder, tlbc_array_addr, &tlbc_array_ptr) != 0 || tlbc_array_ptr == 0) { + if (read_ptr(unwinder, tlbc_array_addr, &tlbc_array_ptr) != 0) { + PyErr_SetString(PyExc_RuntimeError, "Failed to read TLBC array pointer"); set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read TLBC array pointer"); + return 0; // Read error + } + + // Validate TLBC array pointer + if (tlbc_array_ptr == 0) { + PyErr_SetString(PyExc_RuntimeError, "TLBC array pointer is NULL"); return 0; // No TLBC array } // Read the TLBC array size Py_ssize_t tlbc_size; - if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, tlbc_array_ptr, sizeof(tlbc_size), &tlbc_size) != 0 || tlbc_size <= 0) { + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, tlbc_array_ptr, sizeof(tlbc_size), &tlbc_size) != 0) { + PyErr_SetString(PyExc_RuntimeError, "Failed to read TLBC array size"); set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read TLBC array size"); + return 0; // Read error + } + + // Validate TLBC array size + if (tlbc_size <= 0) { + PyErr_SetString(PyExc_RuntimeError, "Invalid TLBC array size"); + return 0; // Invalid size + } + + if (tlbc_size > MAX_TLBC_SIZE) { + PyErr_SetString(PyExc_RuntimeError, "TLBC array size exceeds maximum limit"); return 0; // Invalid size } @@ -1584,6 +1603,7 @@ cache_tlbc_array(RemoteUnwinderObject *unwinder, uintptr_t code_addr, uintptr_t size_t array_data_size = tlbc_size * sizeof(void*); tlbc_array = PyMem_RawMalloc(sizeof(Py_ssize_t) + array_data_size); if (!tlbc_array) { + PyErr_NoMemory(); set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate TLBC array"); return 0; // Memory error } @@ -1597,6 +1617,7 @@ cache_tlbc_array(RemoteUnwinderObject *unwinder, uintptr_t code_addr, uintptr_t // Create cache entry entry = PyMem_RawMalloc(sizeof(TLBCCacheEntry)); if (!entry) { + PyErr_NoMemory(); PyMem_RawFree(tlbc_array); set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate TLBC cache entry"); return 0; // Memory error @@ -1777,6 +1798,7 @@ parse_code_object(RemoteUnwinderObject *unwinder, meta = PyMem_RawMalloc(sizeof(CachedCodeMetadata)); if (!meta) { + PyErr_NoMemory(); set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate cached code metadata"); goto error; } diff --git a/Modules/_ssl.c b/Modules/_ssl.c index ab30258faf3f62..a74654b7573f45 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -366,9 +366,6 @@ typedef struct { * and shutdown methods check for chained exceptions. */ PyObject *exc; - /* Lock to synchronize calls when the thread state is detached. - See also gh-134698. */ - PyMutex tstate_mutex; } PySSLSocket; #define PySSLSocket_CAST(op) ((PySSLSocket *)(op)) @@ -918,7 +915,6 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, self->server_hostname = NULL; self->err = err; self->exc = NULL; - self->tstate_mutex = (PyMutex){0}; /* Make sure the SSL error state is initialized */ ERR_clear_error(); @@ -994,12 +990,12 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, BIO_set_nbio(SSL_get_wbio(self->ssl), 1); } - PySSL_BEGIN_ALLOW_THREADS(self) + Py_BEGIN_ALLOW_THREADS; if (socket_type == PY_SSL_CLIENT) SSL_set_connect_state(self->ssl); else SSL_set_accept_state(self->ssl); - PySSL_END_ALLOW_THREADS(self) + Py_END_ALLOW_THREADS; self->socket_type = socket_type; if (sock != NULL) { @@ -1068,10 +1064,11 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) /* Actually negotiate SSL connection */ /* XXX If SSL_do_handshake() returns 0, it's also a failure. */ do { - PySSL_BEGIN_ALLOW_THREADS(self) + Py_BEGIN_ALLOW_THREADS ret = SSL_do_handshake(self->ssl); err = _PySSL_errno(ret < 1, self->ssl, ret); - PySSL_END_ALLOW_THREADS(self) + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; self->err = err; if (PyErr_CheckSignals()) @@ -2615,10 +2612,11 @@ _ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, } do { - PySSL_BEGIN_ALLOW_THREADS(self) + Py_BEGIN_ALLOW_THREADS retval = SSL_sendfile(self->ssl, fd, (off_t)offset, size, flags); err = _PySSL_errno(retval < 0, self->ssl, (int)retval); - PySSL_END_ALLOW_THREADS(self) + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; self->err = err; if (PyErr_CheckSignals()) { @@ -2746,10 +2744,11 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) } do { - PySSL_BEGIN_ALLOW_THREADS(self) + Py_BEGIN_ALLOW_THREADS; retval = SSL_write_ex(self->ssl, b->buf, (size_t)b->len, &count); err = _PySSL_errno(retval == 0, self->ssl, retval); - PySSL_END_ALLOW_THREADS(self) + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; self->err = err; if (PyErr_CheckSignals()) @@ -2807,10 +2806,11 @@ _ssl__SSLSocket_pending_impl(PySSLSocket *self) int count = 0; _PySSLError err; - PySSL_BEGIN_ALLOW_THREADS(self) + Py_BEGIN_ALLOW_THREADS; count = SSL_pending(self->ssl); err = _PySSL_errno(count < 0, self->ssl, count); - PySSL_END_ALLOW_THREADS(self) + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; self->err = err; if (count < 0) @@ -2901,10 +2901,11 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, deadline = _PyDeadline_Init(timeout); do { - PySSL_BEGIN_ALLOW_THREADS(self) + Py_BEGIN_ALLOW_THREADS; retval = SSL_read_ex(self->ssl, mem, (size_t)len, &count); err = _PySSL_errno(retval == 0, self->ssl, retval); - PySSL_END_ALLOW_THREADS(self) + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; self->err = err; if (PyErr_CheckSignals()) @@ -3003,7 +3004,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) } while (1) { - PySSL_BEGIN_ALLOW_THREADS(self) + Py_BEGIN_ALLOW_THREADS; /* Disable read-ahead so that unwrap can work correctly. * Otherwise OpenSSL might read in too much data, * eating clear text data that happens to be @@ -3016,7 +3017,8 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) SSL_set_read_ahead(self->ssl, 0); ret = SSL_shutdown(self->ssl); err = _PySSL_errno(ret < 0, self->ssl, ret); - PySSL_END_ALLOW_THREADS(self) + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; self->err = err; /* If err == 1, a secure shutdown with SSL_shutdown() is complete */ diff --git a/Modules/_ssl/debughelpers.c b/Modules/_ssl/debughelpers.c index 211fe15a439bfc..aee446d0ccb1b8 100644 --- a/Modules/_ssl/debughelpers.c +++ b/Modules/_ssl/debughelpers.c @@ -140,7 +140,6 @@ _PySSL_keylog_callback(const SSL *ssl, const char *line) * critical debug helper. */ - assert(PyMutex_IsLocked(&ssl_obj->tstate_mutex)); Py_BEGIN_ALLOW_THREADS PyThread_acquire_lock(lock, 1); res = BIO_printf(ssl_obj->ctx->keylog_bio, "%s\n", line); diff --git a/Modules/_zstd/clinic/zstddict.c.h b/Modules/_zstd/clinic/zstddict.c.h index 79db85405d6e6b..166d925a542352 100644 --- a/Modules/_zstd/clinic/zstddict.c.h +++ b/Modules/_zstd/clinic/zstddict.c.h @@ -198,7 +198,7 @@ PyDoc_STRVAR(_zstd_ZstdDict_as_prefix__doc__, "1. Prefix is compatible with long distance matching, while dictionary is not.\n" "2. It only works for the first frame, then the compressor/decompressor will\n" " return to no prefix state.\n" -"3. When decompressing, must use the same prefix as when compressing.\""); +"3. When decompressing, must use the same prefix as when compressing."); #if defined(_zstd_ZstdDict_as_prefix_DOCSTR) # undef _zstd_ZstdDict_as_prefix_DOCSTR #endif @@ -222,4 +222,4 @@ _zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context)) { return _zstd_ZstdDict_as_prefix_get_impl((ZstdDict *)self); } -/*[clinic end generated code: output=4696cbc722e5fdfc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f41d9e2e2cc2928f input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index b00ee05d2f51bf..026d39f68291da 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -157,10 +157,7 @@ _zstd_load_impl(ZstdDecompressor *self, ZstdDict *zd, zd->dict_len); } else { - /* Impossible code path */ - PyErr_SetString(PyExc_SystemError, - "load_d_dict() impossible code path"); - return -1; + Py_UNREACHABLE(); } /* Check error */ diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index 35d6ca8e55a265..76e966f5b1b52a 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -119,10 +119,10 @@ ZstdDict_dealloc(PyObject *ob) } PyDoc_STRVAR(ZstdDict_dictid_doc, -"the Zstandard dictionary, an int between 0 and 2**32.\n\n" -"A non-zero value represents an ordinary Zstandard dictionary, " +"The Zstandard dictionary, an int between 0 and 2**32.\n\n" +"A non-zero value represents an ordinary Zstandard dictionary,\n" "conforming to the standardised format.\n\n" -"The special value '0' means a 'raw content' dictionary," +"A value of zero indicates a 'raw content' dictionary,\n" "without any restrictions on format or content."); static PyObject * @@ -210,12 +210,12 @@ compress(dat, zstd_dict=zd.as_prefix) 1. Prefix is compatible with long distance matching, while dictionary is not. 2. It only works for the first frame, then the compressor/decompressor will return to no prefix state. -3. When decompressing, must use the same prefix as when compressing." +3. When decompressing, must use the same prefix as when compressing. [clinic start generated code]*/ static PyObject * _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self) -/*[clinic end generated code: output=6f7130c356595a16 input=d59757b0b5a9551a]*/ +/*[clinic end generated code: output=6f7130c356595a16 input=45b3b6110f36d127]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_PREFIX); } diff --git a/Modules/blake2module.c b/Modules/blake2module.c index 13c969056be354..163f238a4268d0 100644 --- a/Modules/blake2module.c +++ b/Modules/blake2module.c @@ -15,12 +15,10 @@ #endif #include "Python.h" -#include "pycore_moduleobject.h" -#include "pycore_strhex.h" // _Py_strhex() +#include "hashlib.h" +#include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" - -#include "_hashlib/hashlib_buffer.h" -#include "_hashlib/hashlib_mutex.h" +#include "pycore_moduleobject.h" // QUICK CPU AUTODETECTION // diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h index 18e6129fad8a89..7c4bd5503ed56b 100644 --- a/Modules/clinic/_datetimemodule.c.h +++ b/Modules/clinic/_datetimemodule.c.h @@ -8,6 +8,206 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +PyDoc_STRVAR(delta_new__doc__, +"timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0,\n" +" hours=0, weeks=0)\n" +"--\n" +"\n" +"Difference between two datetime values.\n" +"\n" +"All arguments are optional and default to 0.\n" +"Arguments may be integers or floats, and may be positive or negative."); + +static PyObject * +delta_new_impl(PyTypeObject *type, PyObject *days, PyObject *seconds, + PyObject *microseconds, PyObject *milliseconds, + PyObject *minutes, PyObject *hours, PyObject *weeks); + +static PyObject * +delta_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 7 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(days), &_Py_ID(seconds), &_Py_ID(microseconds), &_Py_ID(milliseconds), &_Py_ID(minutes), &_Py_ID(hours), &_Py_ID(weeks), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"days", "seconds", "microseconds", "milliseconds", "minutes", "hours", "weeks", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "timedelta", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[7]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *days = NULL; + PyObject *seconds = NULL; + PyObject *microseconds = NULL; + PyObject *milliseconds = NULL; + PyObject *minutes = NULL; + PyObject *hours = NULL; + PyObject *weeks = NULL; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 7, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + days = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[1]) { + seconds = fastargs[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[2]) { + microseconds = fastargs[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[3]) { + milliseconds = fastargs[3]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[4]) { + minutes = fastargs[4]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[5]) { + hours = fastargs[5]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + weeks = fastargs[6]; +skip_optional_pos: + return_value = delta_new_impl(type, days, seconds, microseconds, milliseconds, minutes, hours, weeks); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date__doc__, +"date(year, month, day)\n" +"--\n" +"\n" +"Concrete date type."); + +static PyObject * +datetime_date_impl(PyTypeObject *type, int year, int month, int day); + +static PyObject * +datetime_date(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "month", "day", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "date", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + int year; + int month; + int day; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + year = PyLong_AsInt(fastargs[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + month = PyLong_AsInt(fastargs[1]); + if (month == -1 && PyErr_Occurred()) { + goto exit; + } + day = PyLong_AsInt(fastargs[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = datetime_date_impl(type, year, month, day); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date_today__doc__, +"today($type, /)\n" +"--\n" +"\n" +"Current date or datetime.\n" +"\n" +"Equivalent to fromtimestamp(time.time())."); + +#define DATETIME_DATE_TODAY_METHODDEF \ + {"today", (PyCFunction)datetime_date_today, METH_NOARGS|METH_CLASS, datetime_date_today__doc__}, + +static PyObject * +datetime_date_today_impl(PyTypeObject *type); + +static PyObject * +datetime_date_today(PyObject *type, PyObject *Py_UNUSED(ignored)) +{ + return datetime_date_today_impl((PyTypeObject *)type); +} + PyDoc_STRVAR(datetime_date_fromtimestamp__doc__, "fromtimestamp($type, timestamp, /)\n" "--\n" @@ -33,12 +233,83 @@ datetime_date_fromtimestamp(PyObject *type, PyObject *timestamp) return return_value; } +PyDoc_STRVAR(datetime_date_fromordinal__doc__, +"fromordinal($type, ordinal, /)\n" +"--\n" +"\n" +"Construct a date from a proleptic Gregorian ordinal.\n" +"\n" +"January 1 of year 1 is day 1. Only the year, month and day are\n" +"non-zero in the result."); + +#define DATETIME_DATE_FROMORDINAL_METHODDEF \ + {"fromordinal", (PyCFunction)datetime_date_fromordinal, METH_O|METH_CLASS, datetime_date_fromordinal__doc__}, + static PyObject * -iso_calendar_date_new_impl(PyTypeObject *type, int year, int week, - int weekday); +datetime_date_fromordinal_impl(PyTypeObject *type, int ordinal); static PyObject * -iso_calendar_date_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +datetime_date_fromordinal(PyObject *type, PyObject *arg) +{ + PyObject *return_value = NULL; + int ordinal; + + ordinal = PyLong_AsInt(arg); + if (ordinal == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = datetime_date_fromordinal_impl((PyTypeObject *)type, ordinal); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date_fromisoformat__doc__, +"fromisoformat($type, string, /)\n" +"--\n" +"\n" +"Construct a date from a string in ISO 8601 format."); + +#define DATETIME_DATE_FROMISOFORMAT_METHODDEF \ + {"fromisoformat", (PyCFunction)datetime_date_fromisoformat, METH_O|METH_CLASS, datetime_date_fromisoformat__doc__}, + +static PyObject * +datetime_date_fromisoformat_impl(PyTypeObject *type, PyObject *string); + +static PyObject * +datetime_date_fromisoformat(PyObject *type, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *string; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("fromisoformat", "argument", "str", arg); + goto exit; + } + string = arg; + return_value = datetime_date_fromisoformat_impl((PyTypeObject *)type, string); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date_fromisocalendar__doc__, +"fromisocalendar($type, /, year, week, day)\n" +"--\n" +"\n" +"Construct a date from the ISO year, week number and weekday.\n" +"\n" +"This is the inverse of the date.isocalendar() function."); + +#define DATETIME_DATE_FROMISOCALENDAR_METHODDEF \ + {"fromisocalendar", _PyCFunction_CAST(datetime_date_fromisocalendar), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, datetime_date_fromisocalendar__doc__}, + +static PyObject * +datetime_date_fromisocalendar_impl(PyTypeObject *type, int year, int week, + int day); + +static PyObject * +datetime_date_fromisocalendar(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -52,7 +323,7 @@ iso_calendar_date_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(year), &_Py_ID(week), &_Py_ID(weekday), }, + .ob_item = { &_Py_ID(year), &_Py_ID(week), &_Py_ID(day), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -61,58 +332,1293 @@ iso_calendar_date_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"year", "week", "weekday", NULL}; + static const char * const _keywords[] = {"year", "week", "day", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "IsoCalendarDate", + .fname = "fromisocalendar", .kwtuple = KWTUPLE, }; #undef KWTUPLE PyObject *argsbuf[3]; - PyObject * const *fastargs; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); int year; int week; - int weekday; + int day; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!fastargs) { + if (!args) { goto exit; } - year = PyLong_AsInt(fastargs[0]); + year = PyLong_AsInt(args[0]); if (year == -1 && PyErr_Occurred()) { goto exit; } - week = PyLong_AsInt(fastargs[1]); + week = PyLong_AsInt(args[1]); if (week == -1 && PyErr_Occurred()) { goto exit; } - weekday = PyLong_AsInt(fastargs[2]); - if (weekday == -1 && PyErr_Occurred()) { + day = PyLong_AsInt(args[2]); + if (day == -1 && PyErr_Occurred()) { goto exit; } - return_value = iso_calendar_date_new_impl(type, year, week, weekday); + return_value = datetime_date_fromisocalendar_impl((PyTypeObject *)type, year, week, day); exit: return return_value; } -PyDoc_STRVAR(datetime_date_replace__doc__, -"replace($self, /, year=unchanged, month=unchanged, day=unchanged)\n" +PyDoc_STRVAR(datetime_date_strptime__doc__, +"strptime($type, string, format, /)\n" "--\n" "\n" -"Return date with new specified fields."); +"Parse string according to the given date format (like time.strptime())."); + +#define DATETIME_DATE_STRPTIME_METHODDEF \ + {"strptime", _PyCFunction_CAST(datetime_date_strptime), METH_FASTCALL|METH_CLASS, datetime_date_strptime__doc__}, + +static PyObject * +datetime_date_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format); + +static PyObject * +datetime_date_strptime(PyObject *type, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *string; + PyObject *format; + + if (!_PyArg_CheckPositional("strptime", nargs, 2, 2)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("strptime", "argument 1", "str", args[0]); + goto exit; + } + string = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("strptime", "argument 2", "str", args[1]); + goto exit; + } + format = args[1]; + return_value = datetime_date_strptime_impl((PyTypeObject *)type, string, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date_strftime__doc__, +"strftime($self, /, format)\n" +"--\n" +"\n" +"Format using strftime().\n" +"\n" +"Example: \"%d/%m/%Y, %H:%M:%S\"."); + +#define DATETIME_DATE_STRFTIME_METHODDEF \ + {"strftime", _PyCFunction_CAST(datetime_date_strftime), METH_FASTCALL|METH_KEYWORDS, datetime_date_strftime__doc__}, + +static PyObject * +datetime_date_strftime_impl(PyObject *self, PyObject *format); + +static PyObject * +datetime_date_strftime(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(format), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"format", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "strftime", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *format; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("strftime", "argument 'format'", "str", args[0]); + goto exit; + } + format = args[0]; + return_value = datetime_date_strftime_impl(self, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date___format____doc__, +"__format__($self, format, /)\n" +"--\n" +"\n" +"Formats self with strftime."); + +#define DATETIME_DATE___FORMAT___METHODDEF \ + {"__format__", (PyCFunction)datetime_date___format__, METH_O, datetime_date___format____doc__}, + +static PyObject * +datetime_date___format___impl(PyObject *self, PyObject *format); + +static PyObject * +datetime_date___format__(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *format; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("__format__", "argument", "str", arg); + goto exit; + } + format = arg; + return_value = datetime_date___format___impl(self, format); + +exit: + return return_value; +} + +static PyObject * +iso_calendar_date_new_impl(PyTypeObject *type, int year, int week, + int weekday); + +static PyObject * +iso_calendar_date_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(year), &_Py_ID(week), &_Py_ID(weekday), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "week", "weekday", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "IsoCalendarDate", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + int year; + int week; + int weekday; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + year = PyLong_AsInt(fastargs[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + week = PyLong_AsInt(fastargs[1]); + if (week == -1 && PyErr_Occurred()) { + goto exit; + } + weekday = PyLong_AsInt(fastargs[2]); + if (weekday == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = iso_calendar_date_new_impl(type, year, week, weekday); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date_replace__doc__, +"replace($self, /, year=unchanged, month=unchanged, day=unchanged)\n" +"--\n" +"\n" +"Return date with new specified fields."); + +#define DATETIME_DATE_REPLACE_METHODDEF \ + {"replace", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL|METH_KEYWORDS, datetime_date_replace__doc__}, + +static PyObject * +datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, + int day); + +static PyObject * +datetime_date_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "month", "day", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int year = GET_YEAR(self); + int month = GET_MONTH(self); + int day = GET_DAY(self); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + year = PyLong_AsInt(args[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + month = PyLong_AsInt(args[1]); + if (month == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + day = PyLong_AsInt(args[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: + return_value = datetime_date_replace_impl((PyDateTime_Date *)self, year, month, day); + +exit: + return return_value; +} + +PyDoc_STRVAR(timezone_new__doc__, +"timezone(offset, name=)\n" +"--\n" +"\n" +"Fixed offset from UTC implementation of tzinfo."); + +static PyObject * +timezone_new_impl(PyTypeObject *type, PyObject *offset, PyObject *name); + +static PyObject * +timezone_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(offset), &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"offset", "name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "timezone", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; + PyObject *offset; + PyObject *name = NULL; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!PyObject_TypeCheck(fastargs[0], DELTA_TYPE(NO_STATE))) { + _PyArg_BadArgument("timezone", "argument 'offset'", (DELTA_TYPE(NO_STATE))->tp_name, fastargs[0]); + goto exit; + } + offset = fastargs[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (!PyUnicode_Check(fastargs[1])) { + _PyArg_BadArgument("timezone", "argument 'name'", "str", fastargs[1]); + goto exit; + } + name = fastargs[1]; +skip_optional_pos: + return_value = timezone_new_impl(type, offset, name); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time__doc__, +"time(hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0)\n" +"--\n" +"\n" +"Time with time zone.\n" +"\n" +"All arguments are optional. tzinfo may be None, or an instance of\n" +"a tzinfo subclass. The remaining arguments may be ints."); + +static PyObject * +datetime_time_impl(PyTypeObject *type, int hour, int minute, int second, + int microsecond, PyObject *tzinfo, int fold); + +static PyObject * +datetime_time(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "time", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + int hour = 0; + int minute = 0; + int second = 0; + int microsecond = 0; + PyObject *tzinfo = Py_None; + int fold = 0; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + hour = PyLong_AsInt(fastargs[0]); + if (hour == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[1]) { + minute = PyLong_AsInt(fastargs[1]); + if (minute == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[2]) { + second = PyLong_AsInt(fastargs[2]); + if (second == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[3]) { + microsecond = PyLong_AsInt(fastargs[3]); + if (microsecond == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[4]) { + tzinfo = fastargs[4]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + fold = PyLong_AsInt(fastargs[5]); + if (fold == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = datetime_time_impl(type, hour, minute, second, microsecond, tzinfo, fold); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_strptime__doc__, +"strptime($type, string, format, /)\n" +"--\n" +"\n" +"Parse string according to the given time format (like time.strptime())."); + +#define DATETIME_TIME_STRPTIME_METHODDEF \ + {"strptime", _PyCFunction_CAST(datetime_time_strptime), METH_FASTCALL|METH_CLASS, datetime_time_strptime__doc__}, + +static PyObject * +datetime_time_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format); + +static PyObject * +datetime_time_strptime(PyObject *type, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *string; + PyObject *format; + + if (!_PyArg_CheckPositional("strptime", nargs, 2, 2)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("strptime", "argument 1", "str", args[0]); + goto exit; + } + string = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("strptime", "argument 2", "str", args[1]); + goto exit; + } + format = args[1]; + return_value = datetime_time_strptime_impl((PyTypeObject *)type, string, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_isoformat__doc__, +"isoformat($self, /, timespec=\'auto\')\n" +"--\n" +"\n" +"Return the time formatted according to ISO.\n" +"\n" +"The full format is \'HH:MM:SS.mmmmmm+zz:zz\'. By default, the fractional\n" +"part is omitted if self.microsecond == 0.\n" +"\n" +"The optional argument timespec specifies the number of additional\n" +"terms of the time to include. Valid options are \'auto\', \'hours\',\n" +"\'minutes\', \'seconds\', \'milliseconds\' and \'microseconds\'."); + +#define DATETIME_TIME_ISOFORMAT_METHODDEF \ + {"isoformat", _PyCFunction_CAST(datetime_time_isoformat), METH_FASTCALL|METH_KEYWORDS, datetime_time_isoformat__doc__}, + +static PyObject * +datetime_time_isoformat_impl(PyDateTime_Time *self, const char *timespec); + +static PyObject * +datetime_time_isoformat(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(timespec), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"timespec", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "isoformat", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + const char *timespec = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("isoformat", "argument 'timespec'", "str", args[0]); + goto exit; + } + Py_ssize_t timespec_length; + timespec = PyUnicode_AsUTF8AndSize(args[0], ×pec_length); + if (timespec == NULL) { + goto exit; + } + if (strlen(timespec) != (size_t)timespec_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } +skip_optional_pos: + return_value = datetime_time_isoformat_impl((PyDateTime_Time *)self, timespec); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_strftime__doc__, +"strftime($self, /, format)\n" +"--\n" +"\n" +"Format using strftime().\n" +"\n" +"The date part of the timestamp passed to underlying strftime should not be used."); + +#define DATETIME_TIME_STRFTIME_METHODDEF \ + {"strftime", _PyCFunction_CAST(datetime_time_strftime), METH_FASTCALL|METH_KEYWORDS, datetime_time_strftime__doc__}, + +static PyObject * +datetime_time_strftime_impl(PyDateTime_Time *self, PyObject *format); + +static PyObject * +datetime_time_strftime(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(format), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"format", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "strftime", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *format; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("strftime", "argument 'format'", "str", args[0]); + goto exit; + } + format = args[0]; + return_value = datetime_time_strftime_impl((PyDateTime_Time *)self, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time___format____doc__, +"__format__($self, format, /)\n" +"--\n" +"\n" +"Formats self with strftime."); + +#define DATETIME_TIME___FORMAT___METHODDEF \ + {"__format__", (PyCFunction)datetime_time___format__, METH_O, datetime_time___format____doc__}, + +static PyObject * +datetime_time___format___impl(PyObject *self, PyObject *format); + +static PyObject * +datetime_time___format__(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *format; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("__format__", "argument", "str", arg); + goto exit; + } + format = arg; + return_value = datetime_time___format___impl(self, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_replace__doc__, +"replace($self, /, hour=unchanged, minute=unchanged, second=unchanged,\n" +" microsecond=unchanged, tzinfo=unchanged, *, fold=unchanged)\n" +"--\n" +"\n" +"Return time with new specified fields."); + +#define DATETIME_TIME_REPLACE_METHODDEF \ + {"replace", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL|METH_KEYWORDS, datetime_time_replace__doc__}, + +static PyObject * +datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold); + +static PyObject * +datetime_time_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int hour = TIME_GET_HOUR(self); + int minute = TIME_GET_MINUTE(self); + int second = TIME_GET_SECOND(self); + int microsecond = TIME_GET_MICROSECOND(self); + PyObject *tzinfo = HASTZINFO(self) ? ((PyDateTime_Time *)self)->tzinfo : Py_None; + int fold = TIME_GET_FOLD(self); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + hour = PyLong_AsInt(args[0]); + if (hour == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + minute = PyLong_AsInt(args[1]); + if (minute == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[2]) { + second = PyLong_AsInt(args[2]); + if (second == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[3]) { + microsecond = PyLong_AsInt(args[3]); + if (microsecond == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[4]) { + tzinfo = args[4]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + fold = PyLong_AsInt(args[5]); + if (fold == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = datetime_time_replace_impl((PyDateTime_Time *)self, hour, minute, second, microsecond, tzinfo, fold); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_fromisoformat__doc__, +"fromisoformat($type, string, /)\n" +"--\n" +"\n" +"Construct a time from a string in ISO 8601 format."); + +#define DATETIME_TIME_FROMISOFORMAT_METHODDEF \ + {"fromisoformat", (PyCFunction)datetime_time_fromisoformat, METH_O|METH_CLASS, datetime_time_fromisoformat__doc__}, + +static PyObject * +datetime_time_fromisoformat_impl(PyTypeObject *type, PyObject *string); + +static PyObject * +datetime_time_fromisoformat(PyObject *type, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *string; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("fromisoformat", "argument", "str", arg); + goto exit; + } + string = arg; + return_value = datetime_time_fromisoformat_impl((PyTypeObject *)type, string); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time___reduce_ex____doc__, +"__reduce_ex__($self, proto, /)\n" +"--\n" +"\n"); + +#define DATETIME_TIME___REDUCE_EX___METHODDEF \ + {"__reduce_ex__", (PyCFunction)datetime_time___reduce_ex__, METH_O, datetime_time___reduce_ex____doc__}, + +static PyObject * +datetime_time___reduce_ex___impl(PyDateTime_Time *self, int proto); + +static PyObject * +datetime_time___reduce_ex__(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + int proto; + + proto = PyLong_AsInt(arg); + if (proto == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = datetime_time___reduce_ex___impl((PyDateTime_Time *)self, proto); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define DATETIME_TIME___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)datetime_time___reduce__, METH_NOARGS, datetime_time___reduce____doc__}, + +static PyObject * +datetime_time___reduce___impl(PyDateTime_Time *self); + +static PyObject * +datetime_time___reduce__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return datetime_time___reduce___impl((PyDateTime_Time *)self); +} + +PyDoc_STRVAR(datetime_datetime__doc__, +"datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0,\n" +" tzinfo=None, *, fold=0)\n" +"--\n" +"\n" +"A combination of a date and a time.\n" +"\n" +"The year, month and day arguments are required. tzinfo may be None, or an\n" +"instance of a tzinfo subclass. The remaining arguments may be ints."); + +static PyObject * +datetime_datetime_impl(PyTypeObject *type, int year, int month, int day, + int hour, int minute, int second, int microsecond, + PyObject *tzinfo, int fold); + +static PyObject * +datetime_datetime(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 9 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "month", "day", "hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "datetime", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[9]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 3; + int year; + int month; + int day; + int hour = 0; + int minute = 0; + int second = 0; + int microsecond = 0; + PyObject *tzinfo = Py_None; + int fold = 0; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 3, /*maxpos*/ 8, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + year = PyLong_AsInt(fastargs[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + month = PyLong_AsInt(fastargs[1]); + if (month == -1 && PyErr_Occurred()) { + goto exit; + } + day = PyLong_AsInt(fastargs[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[3]) { + hour = PyLong_AsInt(fastargs[3]); + if (hour == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[4]) { + minute = PyLong_AsInt(fastargs[4]); + if (minute == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[5]) { + second = PyLong_AsInt(fastargs[5]); + if (second == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[6]) { + microsecond = PyLong_AsInt(fastargs[6]); + if (microsecond == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[7]) { + tzinfo = fastargs[7]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + fold = PyLong_AsInt(fastargs[8]); + if (fold == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = datetime_datetime_impl(type, year, month, day, hour, minute, second, microsecond, tzinfo, fold); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime_now__doc__, +"now($type, /, tz=None)\n" +"--\n" +"\n" +"Returns new datetime object representing current time local to tz.\n" +"\n" +" tz\n" +" Timezone object.\n" +"\n" +"If no tz is specified, uses local timezone."); + +#define DATETIME_DATETIME_NOW_METHODDEF \ + {"now", _PyCFunction_CAST(datetime_datetime_now), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, datetime_datetime_now__doc__}, + +static PyObject * +datetime_datetime_now_impl(PyTypeObject *type, PyObject *tz); + +static PyObject * +datetime_datetime_now(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(tz), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"tz", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "now", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *tz = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + tz = args[0]; +skip_optional_pos: + return_value = datetime_datetime_now_impl((PyTypeObject *)type, tz); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime_utcnow__doc__, +"utcnow($type, /)\n" +"--\n" +"\n" +"Return a new datetime representing UTC day and time."); + +#define DATETIME_DATETIME_UTCNOW_METHODDEF \ + {"utcnow", (PyCFunction)datetime_datetime_utcnow, METH_NOARGS|METH_CLASS, datetime_datetime_utcnow__doc__}, + +static PyObject * +datetime_datetime_utcnow_impl(PyTypeObject *type); + +static PyObject * +datetime_datetime_utcnow(PyObject *type, PyObject *Py_UNUSED(ignored)) +{ + return datetime_datetime_utcnow_impl((PyTypeObject *)type); +} + +PyDoc_STRVAR(datetime_datetime_fromtimestamp__doc__, +"fromtimestamp($type, /, timestamp, tz=None)\n" +"--\n" +"\n" +"Create a datetime from a POSIX timestamp.\n" +"\n" +"The timestamp is a number, e.g. created via time.time(), that is interpreted\n" +"as local time."); + +#define DATETIME_DATETIME_FROMTIMESTAMP_METHODDEF \ + {"fromtimestamp", _PyCFunction_CAST(datetime_datetime_fromtimestamp), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, datetime_datetime_fromtimestamp__doc__}, + +static PyObject * +datetime_datetime_fromtimestamp_impl(PyTypeObject *type, PyObject *timestamp, + PyObject *tzinfo); + +static PyObject * +datetime_datetime_fromtimestamp(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(timestamp), &_Py_ID(tz), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"timestamp", "tz", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "fromtimestamp", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *timestamp; + PyObject *tzinfo = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + timestamp = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + tzinfo = args[1]; +skip_optional_pos: + return_value = datetime_datetime_fromtimestamp_impl((PyTypeObject *)type, timestamp, tzinfo); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime_utcfromtimestamp__doc__, +"utcfromtimestamp($type, timestamp, /)\n" +"--\n" +"\n" +"Create a naive UTC datetime from a POSIX timestamp."); + +#define DATETIME_DATETIME_UTCFROMTIMESTAMP_METHODDEF \ + {"utcfromtimestamp", (PyCFunction)datetime_datetime_utcfromtimestamp, METH_O|METH_CLASS, datetime_datetime_utcfromtimestamp__doc__}, + +static PyObject * +datetime_datetime_utcfromtimestamp_impl(PyTypeObject *type, + PyObject *timestamp); + +static PyObject * +datetime_datetime_utcfromtimestamp(PyObject *type, PyObject *timestamp) +{ + PyObject *return_value = NULL; + + return_value = datetime_datetime_utcfromtimestamp_impl((PyTypeObject *)type, timestamp); + + return return_value; +} + +PyDoc_STRVAR(datetime_datetime_strptime__doc__, +"strptime($type, string, format, /)\n" +"--\n" +"\n" +"Parse string according to the given date and time format (like time.strptime())."); + +#define DATETIME_DATETIME_STRPTIME_METHODDEF \ + {"strptime", _PyCFunction_CAST(datetime_datetime_strptime), METH_FASTCALL|METH_CLASS, datetime_datetime_strptime__doc__}, + +static PyObject * +datetime_datetime_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format); + +static PyObject * +datetime_datetime_strptime(PyObject *type, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *string; + PyObject *format; + + if (!_PyArg_CheckPositional("strptime", nargs, 2, 2)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("strptime", "argument 1", "str", args[0]); + goto exit; + } + string = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("strptime", "argument 2", "str", args[1]); + goto exit; + } + format = args[1]; + return_value = datetime_datetime_strptime_impl((PyTypeObject *)type, string, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime_combine__doc__, +"combine($type, /, date, time, tzinfo=)\n" +"--\n" +"\n" +"Construct a datetime from a given date and a given time."); -#define DATETIME_DATE_REPLACE_METHODDEF \ - {"replace", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL|METH_KEYWORDS, datetime_date_replace__doc__}, +#define DATETIME_DATETIME_COMBINE_METHODDEF \ + {"combine", _PyCFunction_CAST(datetime_datetime_combine), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, datetime_datetime_combine__doc__}, static PyObject * -datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, - int day); +datetime_datetime_combine_impl(PyTypeObject *type, PyObject *date, + PyObject *time, PyObject *tzinfo); static PyObject * -datetime_date_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +datetime_datetime_combine(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -126,7 +1632,7 @@ datetime_date_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, P } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), }, + .ob_item = { &_Py_ID(date), &_Py_ID(time), &_Py_ID(tzinfo), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -135,200 +1641,107 @@ datetime_date_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, P # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"year", "month", "day", NULL}; + static const char * const _keywords[] = {"date", "time", "tzinfo", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "replace", + .fname = "combine", .kwtuple = KWTUPLE, }; #undef KWTUPLE PyObject *argsbuf[3]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - int year = GET_YEAR(self); - int month = GET_MONTH(self); - int day = GET_DAY(self); + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *date; + PyObject *time; + PyObject *tzinfo = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - if (!noptargs) { - goto skip_optional_pos; - } - if (args[0]) { - year = PyLong_AsInt(args[0]); - if (year == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_pos; - } - } - if (args[1]) { - month = PyLong_AsInt(args[1]); - if (month == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_pos; - } + if (!PyObject_TypeCheck(args[0], DATE_TYPE(NO_STATE))) { + _PyArg_BadArgument("combine", "argument 'date'", (DATE_TYPE(NO_STATE))->tp_name, args[0]); + goto exit; } - day = PyLong_AsInt(args[2]); - if (day == -1 && PyErr_Occurred()) { + date = args[0]; + if (!PyObject_TypeCheck(args[1], TIME_TYPE(NO_STATE))) { + _PyArg_BadArgument("combine", "argument 'time'", (TIME_TYPE(NO_STATE))->tp_name, args[1]); goto exit; } + time = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + tzinfo = args[2]; skip_optional_pos: - return_value = datetime_date_replace_impl((PyDateTime_Date *)self, year, month, day); + return_value = datetime_datetime_combine_impl((PyTypeObject *)type, date, time, tzinfo); exit: return return_value; } -PyDoc_STRVAR(datetime_time_replace__doc__, -"replace($self, /, hour=unchanged, minute=unchanged, second=unchanged,\n" -" microsecond=unchanged, tzinfo=unchanged, *, fold=unchanged)\n" +PyDoc_STRVAR(datetime_datetime_fromisoformat__doc__, +"fromisoformat($type, string, /)\n" "--\n" "\n" -"Return time with new specified fields."); +"Construct a date from a string in ISO 8601 format."); -#define DATETIME_TIME_REPLACE_METHODDEF \ - {"replace", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL|METH_KEYWORDS, datetime_time_replace__doc__}, +#define DATETIME_DATETIME_FROMISOFORMAT_METHODDEF \ + {"fromisoformat", (PyCFunction)datetime_datetime_fromisoformat, METH_O|METH_CLASS, datetime_datetime_fromisoformat__doc__}, static PyObject * -datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, - int second, int microsecond, PyObject *tzinfo, - int fold); +datetime_datetime_fromisoformat_impl(PyTypeObject *type, PyObject *string); static PyObject * -datetime_time_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +datetime_datetime_fromisoformat(PyObject *type, PyObject *arg) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 6 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - Py_hash_t ob_hash; - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_hash = -1, - .ob_item = { &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "replace", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[6]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - int hour = TIME_GET_HOUR(self); - int minute = TIME_GET_MINUTE(self); - int second = TIME_GET_SECOND(self); - int microsecond = TIME_GET_MICROSECOND(self); - PyObject *tzinfo = HASTZINFO(self) ? ((PyDateTime_Time *)self)->tzinfo : Py_None; - int fold = TIME_GET_FOLD(self); + PyObject *string; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 0, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { - goto exit; - } - if (!noptargs) { - goto skip_optional_pos; - } - if (args[0]) { - hour = PyLong_AsInt(args[0]); - if (hour == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_pos; - } - } - if (args[1]) { - minute = PyLong_AsInt(args[1]); - if (minute == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_pos; - } - } - if (args[2]) { - second = PyLong_AsInt(args[2]); - if (second == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_pos; - } - } - if (args[3]) { - microsecond = PyLong_AsInt(args[3]); - if (microsecond == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_pos; - } - } - if (args[4]) { - tzinfo = args[4]; - if (!--noptargs) { - goto skip_optional_pos; - } - } -skip_optional_pos: - if (!noptargs) { - goto skip_optional_kwonly; - } - fold = PyLong_AsInt(args[5]); - if (fold == -1 && PyErr_Occurred()) { + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("fromisoformat", "argument", "str", arg); goto exit; } -skip_optional_kwonly: - return_value = datetime_time_replace_impl((PyDateTime_Time *)self, hour, minute, second, microsecond, tzinfo, fold); + string = arg; + return_value = datetime_datetime_fromisoformat_impl((PyTypeObject *)type, string); exit: return return_value; } -PyDoc_STRVAR(datetime_datetime_now__doc__, -"now($type, /, tz=None)\n" +PyDoc_STRVAR(datetime_datetime_isoformat__doc__, +"isoformat($self, /, sep=\'T\', timespec=\'auto\')\n" "--\n" "\n" -"Returns new datetime object representing current time local to tz.\n" +"Return the time formatted according to ISO.\n" "\n" -" tz\n" -" Timezone object.\n" +"The full format looks like \'YYYY-MM-DD HH:MM:SS.mmmmmm\'.\n" +"By default, the fractional part is omitted if self.microsecond == 0.\n" "\n" -"If no tz is specified, uses local timezone."); +"If self.tzinfo is not None, the UTC offset is also attached, giving\n" +"a full format of \'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM\'.\n" +"\n" +"Optional argument sep specifies the separator between date and\n" +"time, default \'T\'.\n" +"\n" +"The optional argument timespec specifies the number of additional\n" +"terms of the time to include. Valid options are \'auto\', \'hours\',\n" +"\'minutes\', \'seconds\', \'milliseconds\' and \'microseconds\'."); -#define DATETIME_DATETIME_NOW_METHODDEF \ - {"now", _PyCFunction_CAST(datetime_datetime_now), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, datetime_datetime_now__doc__}, +#define DATETIME_DATETIME_ISOFORMAT_METHODDEF \ + {"isoformat", _PyCFunction_CAST(datetime_datetime_isoformat), METH_FASTCALL|METH_KEYWORDS, datetime_datetime_isoformat__doc__}, static PyObject * -datetime_datetime_now_impl(PyTypeObject *type, PyObject *tz); +datetime_datetime_isoformat_impl(PyDateTime_DateTime *self, int sep, + const char *timespec); static PyObject * -datetime_datetime_now(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +datetime_datetime_isoformat(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -337,7 +1750,7 @@ datetime_datetime_now(PyObject *type, PyObject *const *args, Py_ssize_t nargs, P } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(tz), }, + .ob_item = { &_Py_ID(sep), &_Py_ID(timespec), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -346,28 +1759,58 @@ datetime_datetime_now(PyObject *type, PyObject *const *args, Py_ssize_t nargs, P # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"tz", NULL}; + static const char * const _keywords[] = {"sep", "timespec", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "now", + .fname = "isoformat", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[1]; + PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *tz = Py_None; + int sep = 'T'; + const char *timespec = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } if (!noptargs) { goto skip_optional_pos; } - tz = args[0]; + if (args[0]) { + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("isoformat", "argument 'sep'", "a unicode character", args[0]); + goto exit; + } + if (PyUnicode_GET_LENGTH(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "isoformat(): argument 'sep' must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); + goto exit; + } + sep = PyUnicode_READ_CHAR(args[0], 0); + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("isoformat", "argument 'timespec'", "str", args[1]); + goto exit; + } + Py_ssize_t timespec_length; + timespec = PyUnicode_AsUTF8AndSize(args[1], ×pec_length); + if (timespec == NULL) { + goto exit; + } + if (strlen(timespec) != (size_t)timespec_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } skip_optional_pos: - return_value = datetime_datetime_now_impl((PyTypeObject *)type, tz); + return_value = datetime_datetime_isoformat_impl((PyDateTime_DateTime *)self, sep, timespec); exit: return return_value; @@ -524,4 +1967,112 @@ datetime_datetime_replace(PyObject *self, PyObject *const *args, Py_ssize_t narg exit: return return_value; } -/*[clinic end generated code: output=809640e747529c72 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(datetime_datetime_astimezone__doc__, +"astimezone($self, /, tz=None)\n" +"--\n" +"\n" +"Convert to local time in new timezone tz."); + +#define DATETIME_DATETIME_ASTIMEZONE_METHODDEF \ + {"astimezone", _PyCFunction_CAST(datetime_datetime_astimezone), METH_FASTCALL|METH_KEYWORDS, datetime_datetime_astimezone__doc__}, + +static PyObject * +datetime_datetime_astimezone_impl(PyDateTime_DateTime *self, + PyObject *tzinfo); + +static PyObject * +datetime_datetime_astimezone(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(tz), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"tz", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "astimezone", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *tzinfo = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + tzinfo = args[0]; +skip_optional_pos: + return_value = datetime_datetime_astimezone_impl((PyDateTime_DateTime *)self, tzinfo); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime___reduce_ex____doc__, +"__reduce_ex__($self, proto, /)\n" +"--\n" +"\n"); + +#define DATETIME_DATETIME___REDUCE_EX___METHODDEF \ + {"__reduce_ex__", (PyCFunction)datetime_datetime___reduce_ex__, METH_O, datetime_datetime___reduce_ex____doc__}, + +static PyObject * +datetime_datetime___reduce_ex___impl(PyDateTime_DateTime *self, int proto); + +static PyObject * +datetime_datetime___reduce_ex__(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + int proto; + + proto = PyLong_AsInt(arg); + if (proto == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = datetime_datetime___reduce_ex___impl((PyDateTime_DateTime *)self, proto); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define DATETIME_DATETIME___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)datetime_datetime___reduce__, METH_NOARGS, datetime_datetime___reduce____doc__}, + +static PyObject * +datetime_datetime___reduce___impl(PyDateTime_DateTime *self); + +static PyObject * +datetime_datetime___reduce__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return datetime_datetime___reduce___impl((PyDateTime_DateTime *)self); +} +/*[clinic end generated code: output=0b8403bc58982e60 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_interpretersmodule.c.h b/Modules/clinic/_interpretersmodule.c.h new file mode 100644 index 00000000000000..d70ffcea527895 --- /dev/null +++ b/Modules/clinic/_interpretersmodule.c.h @@ -0,0 +1,1201 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(_interpreters_create__doc__, +"create($module, /, config=\'isolated\', *, reqrefs=False)\n" +"--\n" +"\n" +"Create a new interpreter and return a unique generated ID.\n" +"\n" +"The caller is responsible for destroying the interpreter before exiting,\n" +"typically by using _interpreters.destroy(). This can be managed\n" +"automatically by passing \"reqrefs=True\" and then using _incref() and\n" +"_decref() appropriately.\n" +"\n" +"\"config\" must be a valid interpreter config or the name of a\n" +"predefined config (\'isolated\' or \'legacy\'). The default\n" +"is \'isolated\'."); + +#define _INTERPRETERS_CREATE_METHODDEF \ + {"create", _PyCFunction_CAST(_interpreters_create), METH_FASTCALL|METH_KEYWORDS, _interpreters_create__doc__}, + +static PyObject * +_interpreters_create_impl(PyObject *module, PyObject *configobj, int reqrefs); + +static PyObject * +_interpreters_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(config), &_Py_ID(reqrefs), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"config", "reqrefs", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "create", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *configobj = NULL; + int reqrefs = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + configobj = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + reqrefs = PyObject_IsTrue(args[1]); + if (reqrefs < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_create_impl(module, configobj, reqrefs); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_destroy__doc__, +"destroy($module, /, id, *, restrict=False)\n" +"--\n" +"\n" +"Destroy the identified interpreter.\n" +"\n" +"Attempting to destroy the current interpreter raises InterpreterError.\n" +"So does an unrecognized ID."); + +#define _INTERPRETERS_DESTROY_METHODDEF \ + {"destroy", _PyCFunction_CAST(_interpreters_destroy), METH_FASTCALL|METH_KEYWORDS, _interpreters_destroy__doc__}, + +static PyObject * +_interpreters_destroy_impl(PyObject *module, PyObject *id, int restricted); + +static PyObject * +_interpreters_destroy(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "destroy", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *id; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + restricted = PyObject_IsTrue(args[1]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_destroy_impl(module, id, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_list_all__doc__, +"list_all($module, /, *, require_ready=False)\n" +"--\n" +"\n" +"Return a list containing the ID of every existing interpreter."); + +#define _INTERPRETERS_LIST_ALL_METHODDEF \ + {"list_all", _PyCFunction_CAST(_interpreters_list_all), METH_FASTCALL|METH_KEYWORDS, _interpreters_list_all__doc__}, + +static PyObject * +_interpreters_list_all_impl(PyObject *module, int reqready); + +static PyObject * +_interpreters_list_all(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(require_ready), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"require_ready", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "list_all", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int reqready = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + reqready = PyObject_IsTrue(args[0]); + if (reqready < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_list_all_impl(module, reqready); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_get_current__doc__, +"get_current($module, /)\n" +"--\n" +"\n" +"Return (ID, whence) of the current interpreter."); + +#define _INTERPRETERS_GET_CURRENT_METHODDEF \ + {"get_current", (PyCFunction)_interpreters_get_current, METH_NOARGS, _interpreters_get_current__doc__}, + +static PyObject * +_interpreters_get_current_impl(PyObject *module); + +static PyObject * +_interpreters_get_current(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _interpreters_get_current_impl(module); +} + +PyDoc_STRVAR(_interpreters_get_main__doc__, +"get_main($module, /)\n" +"--\n" +"\n" +"Return (ID, whence) of the main interpreter."); + +#define _INTERPRETERS_GET_MAIN_METHODDEF \ + {"get_main", (PyCFunction)_interpreters_get_main, METH_NOARGS, _interpreters_get_main__doc__}, + +static PyObject * +_interpreters_get_main_impl(PyObject *module); + +static PyObject * +_interpreters_get_main(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _interpreters_get_main_impl(module); +} + +PyDoc_STRVAR(_interpreters_set___main___attrs__doc__, +"set___main___attrs($module, /, id, updates, *, restrict=False)\n" +"--\n" +"\n" +"Bind the given attributes in the interpreter\'s __main__ module."); + +#define _INTERPRETERS_SET___MAIN___ATTRS_METHODDEF \ + {"set___main___attrs", _PyCFunction_CAST(_interpreters_set___main___attrs), METH_FASTCALL|METH_KEYWORDS, _interpreters_set___main___attrs__doc__}, + +static PyObject * +_interpreters_set___main___attrs_impl(PyObject *module, PyObject *id, + PyObject *updates, int restricted); + +static PyObject * +_interpreters_set___main___attrs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(updates), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "updates", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set___main___attrs", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *id; + PyObject *updates; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + if (!PyDict_Check(args[1])) { + _PyArg_BadArgument("set___main___attrs", "argument 'updates'", "dict", args[1]); + goto exit; + } + updates = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + restricted = PyObject_IsTrue(args[2]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_set___main___attrs_impl(module, id, updates, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_exec__doc__, +"exec($module, /, id, code, shared={}, *, restrict=False)\n" +"--\n" +"\n" +"Execute the provided code in the identified interpreter.\n" +"\n" +"This is equivalent to running the builtin exec() under the target\n" +"interpreter, using the __dict__ of its __main__ module as both\n" +"globals and locals.\n" +"\n" +"\"code\" may be a string containing the text of a Python script.\n" +"\n" +"Functions (and code objects) are also supported, with some restrictions.\n" +"The code/function must not take any arguments or be a closure\n" +"(i.e. have cell vars). Methods and other callables are not supported.\n" +"\n" +"If a function is provided, its code object is used and all its state\n" +"is ignored, including its __globals__ dict."); + +#define _INTERPRETERS_EXEC_METHODDEF \ + {"exec", _PyCFunction_CAST(_interpreters_exec), METH_FASTCALL|METH_KEYWORDS, _interpreters_exec__doc__}, + +static PyObject * +_interpreters_exec_impl(PyObject *module, PyObject *id, PyObject *code, + PyObject *shared, int restricted); + +static PyObject * +_interpreters_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(code), &_Py_ID(shared), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "code", "shared", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "exec", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *id; + PyObject *code; + PyObject *shared = NULL; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + code = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + if (!PyDict_Check(args[2])) { + _PyArg_BadArgument("exec", "argument 'shared'", "dict", args[2]); + goto exit; + } + shared = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + restricted = PyObject_IsTrue(args[3]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_exec_impl(module, id, code, shared, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_run_string__doc__, +"run_string($module, /, id, script, shared={}, *, restrict=False)\n" +"--\n" +"\n" +"Execute the provided string in the identified interpreter.\n" +"\n" +"(See _interpreters.exec().)"); + +#define _INTERPRETERS_RUN_STRING_METHODDEF \ + {"run_string", _PyCFunction_CAST(_interpreters_run_string), METH_FASTCALL|METH_KEYWORDS, _interpreters_run_string__doc__}, + +static PyObject * +_interpreters_run_string_impl(PyObject *module, PyObject *id, + PyObject *script, PyObject *shared, + int restricted); + +static PyObject * +_interpreters_run_string(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(script), &_Py_ID(shared), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "script", "shared", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "run_string", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *id; + PyObject *script; + PyObject *shared = NULL; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("run_string", "argument 'script'", "str", args[1]); + goto exit; + } + script = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + if (!PyDict_Check(args[2])) { + _PyArg_BadArgument("run_string", "argument 'shared'", "dict", args[2]); + goto exit; + } + shared = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + restricted = PyObject_IsTrue(args[3]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_run_string_impl(module, id, script, shared, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_run_func__doc__, +"run_func($module, /, id, func, shared={}, *, restrict=False)\n" +"--\n" +"\n" +"Execute the body of the provided function in the identified interpreter.\n" +"\n" +"Code objects are also supported. In both cases, closures and args\n" +"are not supported. Methods and other callables are not supported either.\n" +"\n" +"(See _interpreters.exec().)"); + +#define _INTERPRETERS_RUN_FUNC_METHODDEF \ + {"run_func", _PyCFunction_CAST(_interpreters_run_func), METH_FASTCALL|METH_KEYWORDS, _interpreters_run_func__doc__}, + +static PyObject * +_interpreters_run_func_impl(PyObject *module, PyObject *id, PyObject *func, + PyObject *shared, int restricted); + +static PyObject * +_interpreters_run_func(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(func), &_Py_ID(shared), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "func", "shared", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "run_func", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *id; + PyObject *func; + PyObject *shared = NULL; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + func = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + if (!PyDict_Check(args[2])) { + _PyArg_BadArgument("run_func", "argument 'shared'", "dict", args[2]); + goto exit; + } + shared = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + restricted = PyObject_IsTrue(args[3]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_run_func_impl(module, id, func, shared, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_call__doc__, +"call($module, /, id, callable, args=(), kwargs={}, *,\n" +" preserve_exc=False, restrict=False)\n" +"--\n" +"\n" +"Call the provided object in the identified interpreter.\n" +"\n" +"Pass the given args and kwargs, if possible."); + +#define _INTERPRETERS_CALL_METHODDEF \ + {"call", _PyCFunction_CAST(_interpreters_call), METH_FASTCALL|METH_KEYWORDS, _interpreters_call__doc__}, + +static PyObject * +_interpreters_call_impl(PyObject *module, PyObject *id, PyObject *callable, + PyObject *args, PyObject *kwargs, int preserve_exc, + int restricted); + +static PyObject * +_interpreters_call(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(callable), &_Py_ID(args), &_Py_ID(kwargs), &_Py_ID(preserve_exc), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "callable", "args", "kwargs", "preserve_exc", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "call", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *id; + PyObject *callable; + PyObject *__clinic_args = NULL; + PyObject *__clinic_kwargs = NULL; + int preserve_exc = 0; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + callable = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + if (!PyTuple_Check(args[2])) { + _PyArg_BadArgument("call", "argument 'args'", "tuple", args[2]); + goto exit; + } + __clinic_args = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[3]) { + if (!PyDict_Check(args[3])) { + _PyArg_BadArgument("call", "argument 'kwargs'", "dict", args[3]); + goto exit; + } + __clinic_kwargs = args[3]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[4]) { + preserve_exc = PyObject_IsTrue(args[4]); + if (preserve_exc < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + restricted = PyObject_IsTrue(args[5]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_call_impl(module, id, callable, __clinic_args, __clinic_kwargs, preserve_exc, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_is_shareable__doc__, +"is_shareable($module, /, obj)\n" +"--\n" +"\n" +"Return True if the object\'s data may be shared between interpreters and False otherwise."); + +#define _INTERPRETERS_IS_SHAREABLE_METHODDEF \ + {"is_shareable", _PyCFunction_CAST(_interpreters_is_shareable), METH_FASTCALL|METH_KEYWORDS, _interpreters_is_shareable__doc__}, + +static PyObject * +_interpreters_is_shareable_impl(PyObject *module, PyObject *obj); + +static PyObject * +_interpreters_is_shareable(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(obj), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"obj", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_shareable", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *obj; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + obj = args[0]; + return_value = _interpreters_is_shareable_impl(module, obj); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_is_running__doc__, +"is_running($module, /, id, *, restrict=False)\n" +"--\n" +"\n" +"Return whether or not the identified interpreter is running."); + +#define _INTERPRETERS_IS_RUNNING_METHODDEF \ + {"is_running", _PyCFunction_CAST(_interpreters_is_running), METH_FASTCALL|METH_KEYWORDS, _interpreters_is_running__doc__}, + +static PyObject * +_interpreters_is_running_impl(PyObject *module, PyObject *id, int restricted); + +static PyObject * +_interpreters_is_running(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_running", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *id; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + restricted = PyObject_IsTrue(args[1]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_is_running_impl(module, id, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_get_config__doc__, +"get_config($module, /, id, *, restrict=False)\n" +"--\n" +"\n" +"Return a representation of the config used to initialize the interpreter."); + +#define _INTERPRETERS_GET_CONFIG_METHODDEF \ + {"get_config", _PyCFunction_CAST(_interpreters_get_config), METH_FASTCALL|METH_KEYWORDS, _interpreters_get_config__doc__}, + +static PyObject * +_interpreters_get_config_impl(PyObject *module, PyObject *id, int restricted); + +static PyObject * +_interpreters_get_config(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "get_config", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *id; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + restricted = PyObject_IsTrue(args[1]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_get_config_impl(module, id, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_whence__doc__, +"whence($module, /, id)\n" +"--\n" +"\n" +"Return an identifier for where the interpreter was created."); + +#define _INTERPRETERS_WHENCE_METHODDEF \ + {"whence", _PyCFunction_CAST(_interpreters_whence), METH_FASTCALL|METH_KEYWORDS, _interpreters_whence__doc__}, + +static PyObject * +_interpreters_whence_impl(PyObject *module, PyObject *id); + +static PyObject * +_interpreters_whence(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "whence", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *id; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + return_value = _interpreters_whence_impl(module, id); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_incref__doc__, +"incref($module, /, id, *, implieslink=False, restrict=False)\n" +"--\n" +"\n"); + +#define _INTERPRETERS_INCREF_METHODDEF \ + {"incref", _PyCFunction_CAST(_interpreters_incref), METH_FASTCALL|METH_KEYWORDS, _interpreters_incref__doc__}, + +static PyObject * +_interpreters_incref_impl(PyObject *module, PyObject *id, int implieslink, + int restricted); + +static PyObject * +_interpreters_incref(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(implieslink), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "implieslink", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "incref", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *id; + int implieslink = 0; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[1]) { + implieslink = PyObject_IsTrue(args[1]); + if (implieslink < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + restricted = PyObject_IsTrue(args[2]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_incref_impl(module, id, implieslink, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_decref__doc__, +"decref($module, /, id, *, restrict=False)\n" +"--\n" +"\n"); + +#define _INTERPRETERS_DECREF_METHODDEF \ + {"decref", _PyCFunction_CAST(_interpreters_decref), METH_FASTCALL|METH_KEYWORDS, _interpreters_decref__doc__}, + +static PyObject * +_interpreters_decref_impl(PyObject *module, PyObject *id, int restricted); + +static PyObject * +_interpreters_decref(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "decref", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *id; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + restricted = PyObject_IsTrue(args[1]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_decref_impl(module, id, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_capture_exception__doc__, +"capture_exception($module, /, exc=None)\n" +"--\n" +"\n" +"Return a snapshot of an exception.\n" +"\n" +"If \"exc\" is None then the current exception, if any, is used (but not cleared).\n" +"The returned snapshot is the same as what _interpreters.exec() returns."); + +#define _INTERPRETERS_CAPTURE_EXCEPTION_METHODDEF \ + {"capture_exception", _PyCFunction_CAST(_interpreters_capture_exception), METH_FASTCALL|METH_KEYWORDS, _interpreters_capture_exception__doc__}, + +static PyObject * +_interpreters_capture_exception_impl(PyObject *module, PyObject *exc_arg); + +static PyObject * +_interpreters_capture_exception(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(exc), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"exc", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "capture_exception", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *exc_arg = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + exc_arg = args[0]; +skip_optional_pos: + return_value = _interpreters_capture_exception_impl(module, exc_arg); + +exit: + return return_value; +} +/*[clinic end generated code: output=c80f73761f860f6c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 8af9e1db781c8f..df4f802ff0bdc9 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -215,8 +215,8 @@ PyDoc_STRVAR(os_access__doc__, " NotImplementedError.\n" "\n" "Note that most operations will use the effective uid/gid, therefore this\n" -" routine can be used in a suid/sgid environment to test if the invoking user\n" -" has the specified access to the path."); +" routine can be used in a suid/sgid environment to test if the invoking\n" +" user has the specified access to the path."); #define OS_ACCESS_METHODDEF \ {"access", _PyCFunction_CAST(os_access), METH_FASTCALL|METH_KEYWORDS, os_access__doc__}, @@ -13419,4 +13419,4 @@ os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #ifndef OS__EMSCRIPTEN_LOG_METHODDEF #define OS__EMSCRIPTEN_LOG_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ -/*[clinic end generated code: output=b1e2615384347102 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=23de5d098e2dd73f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/resource.c.h b/Modules/clinic/resource.c.h index 9eda7de27532a1..e4ef93900d1797 100644 --- a/Modules/clinic/resource.c.h +++ b/Modules/clinic/resource.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + #if defined(HAVE_GETRUSAGE) PyDoc_STRVAR(resource_getrusage__doc__, @@ -66,7 +68,7 @@ PyDoc_STRVAR(resource_setrlimit__doc__, "\n"); #define RESOURCE_SETRLIMIT_METHODDEF \ - {"setrlimit", (PyCFunction)(void(*)(void))resource_setrlimit, METH_FASTCALL, resource_setrlimit__doc__}, + {"setrlimit", _PyCFunction_CAST(resource_setrlimit), METH_FASTCALL, resource_setrlimit__doc__}, static PyObject * resource_setrlimit_impl(PyObject *module, int resource, PyObject *limits); @@ -78,8 +80,7 @@ resource_setrlimit(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int resource; PyObject *limits; - if (nargs != 2) { - PyErr_Format(PyExc_TypeError, "setrlimit expected 2 arguments, got %zd", nargs); + if (!_PyArg_CheckPositional("setrlimit", nargs, 2, 2)) { goto exit; } resource = PyLong_AsInt(args[0]); @@ -101,7 +102,7 @@ PyDoc_STRVAR(resource_prlimit__doc__, "\n"); #define RESOURCE_PRLIMIT_METHODDEF \ - {"prlimit", (PyCFunction)(void(*)(void))resource_prlimit, METH_FASTCALL, resource_prlimit__doc__}, + {"prlimit", _PyCFunction_CAST(resource_prlimit), METH_FASTCALL, resource_prlimit__doc__}, static PyObject * resource_prlimit_impl(PyObject *module, pid_t pid, int resource, @@ -115,12 +116,7 @@ resource_prlimit(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int resource; PyObject *limits = Py_None; - if (nargs < 2) { - PyErr_Format(PyExc_TypeError, "prlimit expected at least 2 arguments, got %zd", nargs); - goto exit; - } - if (nargs > 3) { - PyErr_Format(PyExc_TypeError, "prlimit expected at most 3 arguments, got %zd", nargs); + if (!_PyArg_CheckPositional("prlimit", nargs, 2, 3)) { goto exit; } pid = PyLong_AsPid(args[0]); @@ -178,4 +174,4 @@ resource_getpagesize(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef RESOURCE_PRLIMIT_METHODDEF #define RESOURCE_PRLIMIT_METHODDEF #endif /* !defined(RESOURCE_PRLIMIT_METHODDEF) */ -/*[clinic end generated code: output=e45883ace510414a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8e905b2f5c35170e input=a9049054013a1b77]*/ diff --git a/Modules/gc_weakref.txt b/Modules/gc_weakref.txt index f53fb99dd6cdcb..c3b8cc743ccd21 100644 --- a/Modules/gc_weakref.txt +++ b/Modules/gc_weakref.txt @@ -1,6 +1,16 @@ Intro ===== +************************************************************************** +Note: this document was written long ago, before PEP 442 (safe object +finalization) was implemented. While that has changed some things, this +document is still mostly accurate. Just note that the rules being discussed +here apply to the unreachable set of objects *after* non-legacy finalizers +have been called. Also, the clearing of weakrefs has been changed to happen +later in the collection (after running finalizers but before tp_clear is +called). +************************************************************************** + The basic rule for dealing with weakref callbacks (and __del__ methods too, for that matter) during cyclic gc: diff --git a/Modules/hashlib.h b/Modules/hashlib.h new file mode 100644 index 00000000000000..5ada4ef4b863ec --- /dev/null +++ b/Modules/hashlib.h @@ -0,0 +1,173 @@ +/* Common code for use by all hashlib related modules. */ + +#include "pycore_lock.h" // PyMutex + +/* + * Internal error messages used for reporting an unsupported hash algorithm. + * The algorithm can be given by its name, a callable or a PEP-247 module. + * The same message is raised by Lib/hashlib.py::__get_builtin_constructor() + * and _hmacmodule.c::find_hash_info(). + */ +#define HASHLIB_UNSUPPORTED_ALGORITHM "unsupported hash algorithm %S" +#define HASHLIB_UNSUPPORTED_STR_ALGORITHM "unsupported hash algorithm %s" + +/* + * Obtain a buffer view from a buffer-like object 'obj'. + * + * On success, store the result in 'view' and return 0. + * On error, set an exception and return -1. + */ +static inline int +_Py_hashlib_get_buffer_view(PyObject *obj, Py_buffer *view) +{ + if (PyUnicode_Check(obj)) { + PyErr_SetString(PyExc_TypeError, + "Strings must be encoded before hashing"); + return -1; + } + if (!PyObject_CheckBuffer(obj)) { + PyErr_SetString(PyExc_TypeError, + "object supporting the buffer API required"); + return -1; + } + if (PyObject_GetBuffer(obj, view, PyBUF_SIMPLE) == -1) { + return -1; + } + if (view->ndim > 1) { + PyErr_SetString(PyExc_BufferError, + "Buffer must be single dimension"); + PyBuffer_Release(view); + return -1; + } + return 0; +} + +/* + * Call _Py_hashlib_get_buffer_view() and check if it succeeded. + * + * On error, set an exception and execute the ERRACTION statements. + */ +#define GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, ERRACTION) \ + do { \ + if (_Py_hashlib_get_buffer_view(OBJ, VIEW) < 0) { \ + assert(PyErr_Occurred()); \ + ERRACTION; \ + } \ + } while (0) + +#define GET_BUFFER_VIEW_OR_ERROUT(OBJ, VIEW) \ + GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, return NULL) + +/* + * Helper code to synchronize access to the hash object when the GIL is + * released around a CPU consuming hashlib operation. + * + * Code accessing a mutable part of the hash object must be enclosed in + * an HASHLIB_{ACQUIRE,RELEASE}_LOCK block or explicitly acquire and release + * the mutex inside a Py_BEGIN_ALLOW_THREADS -- Py_END_ALLOW_THREADS block if + * they wish to release the GIL for an operation. + */ + +#define HASHLIB_OBJECT_HEAD \ + PyObject_HEAD \ + /* Guard against race conditions during incremental update(). */ \ + PyMutex mutex; + +#define HASHLIB_INIT_MUTEX(OBJ) \ + do { \ + (OBJ)->mutex = (PyMutex){0}; \ + } while (0) + +#define HASHLIB_ACQUIRE_LOCK(OBJ) PyMutex_Lock(&(OBJ)->mutex) +#define HASHLIB_RELEASE_LOCK(OBJ) PyMutex_Unlock(&(OBJ)->mutex) + +/* + * Message length above which the GIL is to be released + * when performing hashing operations. + */ +#define HASHLIB_GIL_MINSIZE 2048 + +// Macros for executing code while conditionally holding the GIL. +// +// These only drop the GIL if the lock acquisition itself is likely to +// block. Thus the non-blocking acquire gating the GIL release for a +// blocking lock acquisition. The intent of these macros is to surround +// the assumed always "fast" operations that you aren't releasing the +// GIL around. + +/* + * Execute a suite of C statements 'STATEMENTS'. + * + * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold. + */ +#define HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED(SIZE, STATEMENTS) \ + do { \ + if ((SIZE) > HASHLIB_GIL_MINSIZE) { \ + Py_BEGIN_ALLOW_THREADS \ + STATEMENTS; \ + Py_END_ALLOW_THREADS \ + } \ + else { \ + STATEMENTS; \ + } \ + } while (0) + +/* + * Lock 'OBJ' and execute a suite of C statements 'STATEMENTS'. + * + * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold. + */ +#define HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED(OBJ, SIZE, STATEMENTS) \ + do { \ + if ((SIZE) > HASHLIB_GIL_MINSIZE) { \ + Py_BEGIN_ALLOW_THREADS \ + HASHLIB_ACQUIRE_LOCK(OBJ); \ + STATEMENTS; \ + HASHLIB_RELEASE_LOCK(OBJ); \ + Py_END_ALLOW_THREADS \ + } \ + else { \ + HASHLIB_ACQUIRE_LOCK(OBJ); \ + STATEMENTS; \ + HASHLIB_RELEASE_LOCK(OBJ); \ + } \ + } while (0) + +static inline int +_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string) +{ + if (data != NULL && string == NULL) { + // called as H(data) or H(data=...) + *res = data; + return 1; + } + else if (data == NULL && string != NULL) { + // called as H(string=...) + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "the 'string' keyword parameter is deprecated since " + "Python 3.15 and slated for removal in Python 3.19; " + "use the 'data' keyword parameter or pass the data " + "to hash as a positional argument instead", 1) < 0) + { + *res = NULL; + return -1; + } + *res = string; + return 1; + } + else if (data == NULL && string == NULL) { + // fast path when no data is given + assert(!PyErr_Occurred()); + *res = NULL; + return 0; + } + else { + // called as H(data=..., string) + *res = NULL; + PyErr_SetString(PyExc_TypeError, + "'data' and 'string' are mutually exclusive " + "and support for 'string' keyword parameter " + "is slated for removal in a future version."); + return -1; + } +} diff --git a/Modules/hmacmodule.c b/Modules/hmacmodule.c index 92be49c5a879f3..694e2a095744ff 100644 --- a/Modules/hmacmodule.c +++ b/Modules/hmacmodule.c @@ -20,10 +20,6 @@ #include "pycore_hashtable.h" #include "pycore_strhex.h" // _Py_strhex() -#include "_hashlib/hashlib_buffer.h" -#include "_hashlib/hashlib_fetch.h" -#include "_hashlib/hashlib_mutex.h" - /* * Taken from blake2module.c. In the future, detection of SIMD support * should be delegated to https://github.com/python/cpython/pull/125011. @@ -51,6 +47,8 @@ #include +#include "hashlib.h" + // --- Reusable error messages ------------------------------------------------ static inline void @@ -658,7 +656,7 @@ find_hash_info(hmacmodule_state *state, PyObject *hash_info_ref) } if (rc == 0) { PyErr_Format(state->unknown_hash_error, - _Py_HASHLIB_UNSUPPORTED_ALGORITHM, hash_info_ref); + HASHLIB_UNSUPPORTED_ALGORITHM, hash_info_ref); return NULL; } assert(info != NULL); diff --git a/Modules/md5module.c b/Modules/md5module.c index d5dc4f60a575d4..8b6dd4a8195dfb 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -22,10 +22,9 @@ #endif #include "Python.h" -#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_strhex.h" // _Py_strhex() -#include "_hashlib/hashlib_buffer.h" -#include "_hashlib/hashlib_mutex.h" +#include "hashlib.h" #include "_hacl/Hacl_Hash_MD5.h" diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index b1a80788bd8115..76dbb84691db1f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3295,15 +3295,15 @@ dir_fd, effective_ids, and follow_symlinks may not be implemented NotImplementedError. Note that most operations will use the effective uid/gid, therefore this - routine can be used in a suid/sgid environment to test if the invoking user - has the specified access to the path. + routine can be used in a suid/sgid environment to test if the invoking + user has the specified access to the path. [clinic start generated code]*/ static int os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, int effective_ids, int follow_symlinks) -/*[clinic end generated code: output=cf84158bc90b1a77 input=3ffe4e650ee3bf20]*/ +/*[clinic end generated code: output=cf84158bc90b1a77 input=c33565f7584b99e4]*/ { int return_value; diff --git a/Modules/resource.c b/Modules/resource.c index 3fe18e7c98e3d8..2353bc6653abd8 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -1,7 +1,5 @@ -// Need limited C API version 3.13 for PySys_Audit() -#include "pyconfig.h" // Py_GIL_DISABLED -#ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030d0000 +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 #endif #include "Python.h" @@ -150,6 +148,35 @@ resource_getrusage_impl(PyObject *module, int who) } #endif +static int +py2rlim(PyObject *obj, rlim_t *out) +{ + obj = PyNumber_Index(obj); + if (obj == NULL) { + return -1; + } + int neg = PyLong_IsNegative(obj); + assert(neg >= 0); + Py_ssize_t bytes = PyLong_AsNativeBytes(obj, out, sizeof(*out), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + Py_DECREF(obj); + if (bytes < 0) { + return -1; + } + else if (neg && (*out != RLIM_INFINITY || bytes > (Py_ssize_t)sizeof(*out))) { + PyErr_SetString(PyExc_ValueError, + "Cannot convert negative int"); + return -1; + } + else if (bytes > (Py_ssize_t)sizeof(*out)) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C rlim_t"); + return -1; + } + return 0; +} + static int py2rlimit(PyObject *limits, struct rlimit *rl_out) { @@ -166,26 +193,13 @@ py2rlimit(PyObject *limits, struct rlimit *rl_out) } curobj = PyTuple_GetItem(limits, 0); // borrowed maxobj = PyTuple_GetItem(limits, 1); // borrowed -#if !defined(HAVE_LARGEFILE_SUPPORT) - rl_out->rlim_cur = PyLong_AsLong(curobj); - if (rl_out->rlim_cur == (rlim_t)-1 && PyErr_Occurred()) - goto error; - rl_out->rlim_max = PyLong_AsLong(maxobj); - if (rl_out->rlim_max == (rlim_t)-1 && PyErr_Occurred()) - goto error; -#else - /* The limits are probably bigger than a long */ - rl_out->rlim_cur = PyLong_AsLongLong(curobj); - if (rl_out->rlim_cur == (rlim_t)-1 && PyErr_Occurred()) - goto error; - rl_out->rlim_max = PyLong_AsLongLong(maxobj); - if (rl_out->rlim_max == (rlim_t)-1 && PyErr_Occurred()) + if (py2rlim(curobj, &rl_out->rlim_cur) < 0 || + py2rlim(maxobj, &rl_out->rlim_max) < 0) + { goto error; -#endif + } Py_DECREF(limits); - rl_out->rlim_cur = rl_out->rlim_cur & RLIM_INFINITY; - rl_out->rlim_max = rl_out->rlim_max & RLIM_INFINITY; return 0; error: @@ -193,15 +207,24 @@ py2rlimit(PyObject *limits, struct rlimit *rl_out) return -1; } +static PyObject* +rlim2py(rlim_t value) +{ + if (value == RLIM_INFINITY) { + return PyLong_FromNativeBytes(&value, sizeof(value), -1); + } + return PyLong_FromUnsignedNativeBytes(&value, sizeof(value), -1); +} + static PyObject* rlimit2py(struct rlimit rl) { - if (sizeof(rl.rlim_cur) > sizeof(long)) { - return Py_BuildValue("LL", - (long long) rl.rlim_cur, - (long long) rl.rlim_max); + PyObject *cur = rlim2py(rl.rlim_cur); + if (cur == NULL) { + return NULL; } - return Py_BuildValue("ll", (long) rl.rlim_cur, (long) rl.rlim_max); + PyObject *max = rlim2py(rl.rlim_max); + return Py_BuildValue("NN", cur, max); } /*[clinic input] @@ -495,14 +518,7 @@ resource_exec(PyObject *module) ADD_INT(module, RLIMIT_KQUEUES); #endif - PyObject *v; - if (sizeof(RLIM_INFINITY) > sizeof(long)) { - v = PyLong_FromLongLong((long long) RLIM_INFINITY); - } else - { - v = PyLong_FromLong((long) RLIM_INFINITY); - } - if (PyModule_Add(module, "RLIM_INFINITY", v) < 0) { + if (PyModule_Add(module, "RLIM_INFINITY", rlim2py(RLIM_INFINITY)) < 0) { return -1; } return 0; diff --git a/Modules/sha1module.c b/Modules/sha1module.c index 86e5691e8463e4..faa9dcccc5755b 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -20,11 +20,9 @@ #endif #include "Python.h" -#include "pycore_strhex.h" // _Py_strhex() -#include "pycore_typeobject.h" // _PyType_GetModuleState() - -#include "_hashlib/hashlib_buffer.h" -#include "_hashlib/hashlib_mutex.h" +#include "hashlib.h" +#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" // _PyType_GetModuleState() #include "_hacl/Hacl_Hash_SHA1.h" diff --git a/Modules/sha2module.c b/Modules/sha2module.c index dbf6dde1b8c121..36300ba899fd44 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -21,12 +21,11 @@ #endif #include "Python.h" -#include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_strhex.h" // _Py_strhex() -#include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_strhex.h" // _Py_strhex() -#include "_hashlib/hashlib_buffer.h" -#include "_hashlib/hashlib_mutex.h" +#include "hashlib.h" #include "_hacl/Hacl_Hash_SHA2.h" diff --git a/Modules/sha3module.c b/Modules/sha3module.c index c67bfadbe4664a..5764556bb680f3 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -21,11 +21,9 @@ #endif #include "Python.h" -#include "pycore_strhex.h" // _Py_strhex() -#include "pycore_typeobject.h" // _PyType_GetModuleState() - -#include "_hashlib/hashlib_buffer.h" -#include "_hashlib/hashlib_mutex.h" +#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "hashlib.h" #include "_hacl/Hacl_Hash_SHA3.h" diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index f3ad01854de93b..bca9e7bb712e38 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -3332,32 +3332,59 @@ sock_setsockopt(PyObject *self, PyObject *args) { PySocketSockObject *s = _PySocketSockObject_CAST(self); + Py_ssize_t arglen; int level; int optname; int res; - Py_buffer optval; + Py_buffer buffer; int flag; unsigned int optlen; - PyObject *none; + PyObject *optval; + + if (!PyArg_ParseTuple(args, "iiO|I:setsockopt", + &level, &optname, &optval, &optlen)) + { + return NULL; + } + + arglen = PyTuple_Size(args); + if (arglen == 3 && optval == Py_None) { + PyErr_Format(PyExc_TypeError, + "setsockopt() requires 4 arguments when the third argument is None", + arglen); + return NULL; + } + if (arglen == 4 && optval != Py_None) { + PyErr_Format(PyExc_TypeError, + "setsockopt() only takes 4 arguments when the third argument is None (got %T)", + optval); + return NULL; + } #ifdef AF_VSOCK if (s->sock_family == AF_VSOCK) { + if (!PyIndex_Check(optval)) { + PyErr_Format(PyExc_TypeError, + "setsockopt() argument 3 for AF_VSOCK must be an int (got %T)", + optval); + } uint64_t vflag; // Must be set width of 64 bits /* setsockopt(level, opt, flag) */ - if (PyArg_ParseTuple(args, "iiK:setsockopt", - &level, &optname, &vflag)) { - // level should always be set to AF_VSOCK - res = setsockopt(get_sock_fd(s), level, optname, - (void*)&vflag, sizeof vflag); - goto done; + if (!PyArg_Parse(optval, "K", &vflag)) { + return NULL; } - return NULL; + // level should always be set to AF_VSOCK + res = setsockopt(get_sock_fd(s), level, optname, + (void*)&vflag, sizeof vflag); + goto done; } #endif /* setsockopt(level, opt, flag) */ - if (PyArg_ParseTuple(args, "iii:setsockopt", - &level, &optname, &flag)) { + if (PyIndex_Check(optval)) { + if (!PyArg_Parse(optval, "i", &flag)) { + return NULL; + } #ifdef MS_WINDOWS if (optname == SIO_TCP_SET_ACK_FREQUENCY) { DWORD dummy; @@ -3374,36 +3401,40 @@ sock_setsockopt(PyObject *self, PyObject *args) goto done; } - PyErr_Clear(); - /* setsockopt(level, opt, None, flag) */ - if (PyArg_ParseTuple(args, "iiO!I:setsockopt", - &level, &optname, Py_TYPE(Py_None), &none, &optlen)) { + /* setsockopt(level, opt, None, optlen) */ + if (optval == Py_None) { assert(sizeof(socklen_t) >= sizeof(unsigned int)); res = setsockopt(get_sock_fd(s), level, optname, NULL, (socklen_t)optlen); goto done; } - PyErr_Clear(); /* setsockopt(level, opt, buffer) */ - if (!PyArg_ParseTuple(args, "iiy*:setsockopt", - &level, &optname, &optval)) - return NULL; - + if (PyObject_CheckBuffer(optval)) { + if (!PyArg_Parse(optval, "y*", &buffer)) { + return NULL; + } #ifdef MS_WINDOWS - if (optval.len > INT_MAX) { - PyBuffer_Release(&optval); - PyErr_Format(PyExc_OverflowError, - "socket option is larger than %i bytes", - INT_MAX); - return NULL; - } - res = setsockopt(get_sock_fd(s), level, optname, - optval.buf, (int)optval.len); + if (buffer.len > INT_MAX) { + PyBuffer_Release(&buffer); + PyErr_Format(PyExc_OverflowError, + "socket option is larger than %i bytes", + INT_MAX); + return NULL; + } + res = setsockopt(get_sock_fd(s), level, optname, + buffer.buf, (int)buffer.len); #else - res = setsockopt(get_sock_fd(s), level, optname, optval.buf, optval.len); + res = setsockopt(get_sock_fd(s), level, optname, buffer.buf, buffer.len); #endif - PyBuffer_Release(&optval); + PyBuffer_Release(&buffer); + goto done; + } + + PyErr_Format(PyExc_TypeError, + "socket option should be int, bytes-like object or None (got %T)", + optval); + return NULL; done: if (res < 0) { diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 42e021679b583f..478c571345cd03 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -550,16 +550,12 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_framesize = nlocalsplus + con->stacksize + FRAME_SPECIALS_SIZE; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; -#ifdef Py_GIL_DISABLED - PyMutex_Lock(&interp->func_state.mutex); -#endif + FT_MUTEX_LOCK(&interp->func_state.mutex); co->co_version = interp->func_state.next_version; if (interp->func_state.next_version != 0) { interp->func_state.next_version++; } -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&interp->func_state.mutex); -#endif + FT_MUTEX_UNLOCK(&interp->func_state.mutex); co->_co_monitoring = NULL; co->_co_instrumentation_version = 0; /* not set */ @@ -689,7 +685,7 @@ intern_code_constants(struct _PyCodeConstructor *con) #ifdef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); struct _py_code_state *state = &interp->code_state; - PyMutex_Lock(&state->mutex); + FT_MUTEX_LOCK(&state->mutex); #endif if (intern_strings(con->names) < 0) { goto error; @@ -700,15 +696,11 @@ intern_code_constants(struct _PyCodeConstructor *con) if (intern_strings(con->localsplusnames) < 0) { goto error; } -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&state->mutex); -#endif + FT_MUTEX_UNLOCK(&state->mutex); return 0; error: -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&state->mutex); -#endif + FT_MUTEX_UNLOCK(&state->mutex); return -1; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index d952a58d94af55..14bc5a4bc49f84 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1535,9 +1535,13 @@ type_set_name(PyObject *tp, PyObject *value, void *Py_UNUSED(closure)) return -1; } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); type->tp_name = tp_name; - Py_SETREF(((PyHeapTypeObject*)type)->ht_name, Py_NewRef(value)); - + PyObject *old_name = ((PyHeapTypeObject*)type)->ht_name; + ((PyHeapTypeObject*)type)->ht_name = Py_NewRef(value); + _PyEval_StartTheWorld(interp); + Py_DECREF(old_name); return 0; } @@ -10581,7 +10585,10 @@ _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name) getattr = _PyType_LookupRef(tp, &_Py_ID(__getattr__)); if (getattr == NULL) { /* No __getattr__ hook: use a simpler dispatcher */ +#ifndef Py_GIL_DISABLED + // Replacing the slot is only thread-safe if there is a GIL. tp->tp_getattro = _Py_slot_tp_getattro; +#endif return _Py_slot_tp_getattro(self, name); } /* speed hack: we could use lookup_maybe, but that would resolve the @@ -10706,9 +10713,11 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) get = _PyType_LookupRef(tp, &_Py_ID(__get__)); if (get == NULL) { +#ifndef Py_GIL_DISABLED /* Avoid further slowdowns */ if (tp->tp_descr_get == slot_tp_descr_get) tp->tp_descr_get = NULL; +#endif return Py_NewRef(self); } if (obj == NULL) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 8df7a48284dccd..47802c5e915be2 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -114,14 +114,6 @@ NOTE: In the interpreter's initialization phase, some globals are currently # define _PyUnicode_CHECK(op) PyUnicode_Check(op) #endif -#ifdef Py_GIL_DISABLED -# define LOCK_INTERNED(interp) PyMutex_Lock(&_Py_INTERP_CACHED_OBJECT(interp, interned_mutex)) -# define UNLOCK_INTERNED(interp) PyMutex_Unlock(&_Py_INTERP_CACHED_OBJECT(interp, interned_mutex)) -#else -# define LOCK_INTERNED(interp) -# define UNLOCK_INTERNED(interp) -#endif - static inline char* _PyUnicode_UTF8(PyObject *op) { return FT_ATOMIC_LOAD_PTR_ACQUIRE(_PyCompactUnicodeObject_CAST(op)->utf8); @@ -7684,10 +7676,6 @@ code_page_name(UINT code_page, PyObject **obj) *obj = NULL; if (code_page == CP_ACP) return "mbcs"; - if (code_page == CP_UTF7) - return "CP_UTF7"; - if (code_page == CP_UTF8) - return "CP_UTF8"; *obj = PyBytes_FromFormat("cp%u", code_page); if (*obj == NULL) @@ -15992,14 +15980,16 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, /* Do a setdefault on the per-interpreter cache. */ PyObject *interned = get_interned_dict(interp); assert(interned != NULL); - - LOCK_INTERNED(interp); +#ifdef Py_GIL_DISABLED +# define INTERN_MUTEX &_Py_INTERP_CACHED_OBJECT(interp, interned_mutex) +#endif + FT_MUTEX_LOCK(INTERN_MUTEX); PyObject *t; { int res = PyDict_SetDefaultRef(interned, s, s, &t); if (res < 0) { PyErr_Clear(); - UNLOCK_INTERNED(interp); + FT_MUTEX_UNLOCK(INTERN_MUTEX); return s; } else if (res == 1) { @@ -16009,7 +15999,7 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, PyUnicode_CHECK_INTERNED(t) == SSTATE_INTERNED_MORTAL) { immortalize_interned(t); } - UNLOCK_INTERNED(interp); + FT_MUTEX_UNLOCK(INTERN_MUTEX); return t; } else { @@ -16042,7 +16032,7 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, immortalize_interned(s); } - UNLOCK_INTERNED(interp); + FT_MUTEX_UNLOCK(INTERN_MUTEX); return s; } diff --git a/PCbuild/_hashlib.vcxproj b/PCbuild/_hashlib.vcxproj index cfb43cee935b86..2cd205224bc089 100644 --- a/PCbuild/_hashlib.vcxproj +++ b/PCbuild/_hashlib.vcxproj @@ -100,11 +100,6 @@ - - - - - diff --git a/PCbuild/_hashlib.vcxproj.filters b/PCbuild/_hashlib.vcxproj.filters index d465d92a956eda..7a0700c007f644 100644 --- a/PCbuild/_hashlib.vcxproj.filters +++ b/PCbuild/_hashlib.vcxproj.filters @@ -18,4 +18,4 @@ Resource Files - + \ No newline at end of file diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index e29054f5734d49..eff8d1ccd7f146 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -56,7 +56,7 @@ set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.16 set libraries=%libraries% mpdecimal-4.0.0 -set libraries=%libraries% sqlite-3.49.1.0 +set libraries=%libraries% sqlite-3.50.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.15.0 set libraries=%libraries% xz-5.2.5 diff --git a/PCbuild/python.props b/PCbuild/python.props index ddc7696d2762fe..e1c2ff3fe3cc11 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -74,7 +74,7 @@ - $(ExternalsDir)sqlite-3.49.1.0\ + $(ExternalsDir)sqlite-3.50.4.0\ $(ExternalsDir)bzip2-1.0.8\ $(ExternalsDir)xz-5.2.5\ $(ExternalsDir)libffi-3.4.4\ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index c59b380d814ed9..517103acea8d8e 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -447,10 +447,6 @@ HACL_CAN_COMPILE_VEC128;%(PreprocessorDefinitions) /arch:AVX %(AdditionalOptions) - - - - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 1410cbbef6c849..e9eedfd1312fae 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -255,15 +255,6 @@ Include - - Modules\_hashlib - - - Modules\_hashlib - - - Modules\_hashlib - Modules @@ -980,9 +971,6 @@ Modules - - Modules\_hashlib - Modules diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 3ae3255d933967..27c0d382281bdb 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -237,7 +237,7 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.49.1, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.50.4, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index ebc94715b6f361..0763866576733f 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1404,7 +1404,15 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) { if (PyBytes_AsStringAndSize(tok->bytes, &bstr, &bsize) == -1) { return NULL; } - PyObject* str = _PyPegen_decode_string(p, 0, bstr, bsize, tok); + + // Check if we're inside a raw f-string for format spec decoding + int is_raw = 0; + if (INSIDE_FSTRING(p->tok)) { + tokenizer_mode *mode = TOK_GET_MODE(p->tok); + is_raw = mode->raw; + } + + PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok); if (str == NULL) { return NULL; } diff --git a/Parser/pegen.c b/Parser/pegen.c index 50641de27d37fd..70493031656028 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -5,6 +5,7 @@ #include "pycore_pyerrors.h" // PyExc_IncompleteInputError #include "pycore_runtime.h" // _PyRuntime #include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal +#include "pycore_pyatomic_ft_wrappers.h" #include #include "lexer/lexer.h" @@ -299,22 +300,14 @@ _PyPegen_fill_token(Parser *p) #define NSTATISTICS _PYPEGEN_NSTATISTICS #define memo_statistics _PyRuntime.parser.memo_statistics -#ifdef Py_GIL_DISABLED -#define MUTEX_LOCK() PyMutex_Lock(&_PyRuntime.parser.mutex) -#define MUTEX_UNLOCK() PyMutex_Unlock(&_PyRuntime.parser.mutex) -#else -#define MUTEX_LOCK() -#define MUTEX_UNLOCK() -#endif - void _PyPegen_clear_memo_statistics(void) { - MUTEX_LOCK(); + FT_MUTEX_LOCK(&_PyRuntime.parser.mutex); for (int i = 0; i < NSTATISTICS; i++) { memo_statistics[i] = 0; } - MUTEX_UNLOCK(); + FT_MUTEX_UNLOCK(&_PyRuntime.parser.mutex); } PyObject * @@ -325,22 +318,22 @@ _PyPegen_get_memo_statistics(void) return NULL; } - MUTEX_LOCK(); + FT_MUTEX_LOCK(&_PyRuntime.parser.mutex); for (int i = 0; i < NSTATISTICS; i++) { PyObject *value = PyLong_FromLong(memo_statistics[i]); if (value == NULL) { - MUTEX_UNLOCK(); + FT_MUTEX_UNLOCK(&_PyRuntime.parser.mutex); Py_DECREF(ret); return NULL; } // PyList_SetItem borrows a reference to value. if (PyList_SetItem(ret, i, value) < 0) { - MUTEX_UNLOCK(); + FT_MUTEX_UNLOCK(&_PyRuntime.parser.mutex); Py_DECREF(ret); return NULL; } } - MUTEX_UNLOCK(); + FT_MUTEX_UNLOCK(&_PyRuntime.parser.mutex); return ret; } #endif @@ -366,9 +359,9 @@ _PyPegen_is_memoized(Parser *p, int type, void *pres) if (count <= 0) { count = 1; } - MUTEX_LOCK(); + FT_MUTEX_LOCK(&_PyRuntime.parser.mutex); memo_statistics[type] += count; - MUTEX_UNLOCK(); + FT_MUTEX_UNLOCK(&_PyRuntime.parser.mutex); } #endif p->mark = m->mark; diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 06d1ee016dc2a8..a5809b37b6b493 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -45,27 +45,40 @@ static const char header[] = static void runtime_init(void) { - PyConfig config; - PyConfig_InitIsolatedConfig(&config); - - config.site_import = 0; + PyInitConfig *config = PyInitConfig_Create(); + if (config == NULL) { + printf("memory allocation failed\n"); + exit(1); + } - PyStatus status; - status = PyConfig_SetString(&config, &config.program_name, - L"./_freeze_module"); - if (PyStatus_Exception(status)) { - PyConfig_Clear(&config); - Py_ExitStatusException(status); + if (PyInitConfig_SetInt(config, "site_import", 0) < 0) { + goto error; + } + if (PyInitConfig_SetStr(config, "program_name", "./_freeze_module") < 0) { + goto error; } /* Don't install importlib, since it could execute outdated bytecode. */ - config._install_importlib = 0; - config._init_main = 0; + if (PyInitConfig_SetInt(config, "_install_importlib", 0) < 0) { + goto error; + } + if (PyInitConfig_SetInt(config, "_init_main", 0) < 0) { + goto error; + } + + if (Py_InitializeFromInitConfig(config) < 0) { + goto error; + } + PyInitConfig_Free(config); + return; - status = Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); +error: + { + const char *err_msg; + (void)PyInitConfig_GetError(config, &err_msg); + printf("Python init error: %s\n", err_msg); + PyInitConfig_Free(config); + exit(1); } } diff --git a/Python/ast_preprocess.c b/Python/ast_preprocess.c index bafd67ed790b20..44d3075098be75 100644 --- a/Python/ast_preprocess.c +++ b/Python/ast_preprocess.c @@ -435,13 +435,38 @@ stmt_seq_remove_item(asdl_stmt_seq *stmts, Py_ssize_t idx) return 1; } +static int +remove_docstring(asdl_stmt_seq *stmts, Py_ssize_t idx, PyArena *ctx_) +{ + assert(_PyAST_GetDocString(stmts) != NULL); + // In case there's just the docstring in the body, replace it with `pass` + // keyword, so body won't be empty. + if (asdl_seq_LEN(stmts) == 1) { + stmt_ty docstring = (stmt_ty)asdl_seq_GET(stmts, 0); + stmt_ty pass = _PyAST_Pass( + docstring->lineno, docstring->col_offset, + // we know that `pass` always takes 4 chars and a single line, + // while docstring can span on multiple lines + docstring->lineno, docstring->col_offset + 4, + ctx_ + ); + if (pass == NULL) { + return 0; + } + asdl_seq_SET(stmts, 0, pass); + return 1; + } + // In case there are more than 1 body items, just remove the docstring. + return stmt_seq_remove_item(stmts, idx); +} + static int astfold_body(asdl_stmt_seq *stmts, PyArena *ctx_, _PyASTPreprocessState *state) { int docstring = _PyAST_GetDocString(stmts) != NULL; if (docstring && (state->optimize >= 2)) { /* remove the docstring */ - if (!stmt_seq_remove_item(stmts, 0)) { + if (!remove_docstring(stmts, 0, ctx_)) { return 0; } docstring = 0; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 51d7297ec243cc..2c4bf46d85b0ba 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2118,15 +2118,21 @@ builtin_oct(PyObject *module, PyObject *number) /*[clinic input] ord as builtin_ord - c: object + character as c: object / -Return the Unicode code point for a one-character string. +Return the ordinal value of a character. + +If the argument is a one-character string, return the Unicode code +point of that character. + +If the argument is a bytes or bytearray object of length 1, return its +single byte value. [clinic start generated code]*/ static PyObject * builtin_ord(PyObject *module, PyObject *c) -/*[clinic end generated code: output=4fa5e87a323bae71 input=3064e5d6203ad012]*/ +/*[clinic end generated code: output=4fa5e87a323bae71 input=98d38480432e1177]*/ { long ord; Py_ssize_t size; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d9abc4c53d1f50..8a60e48cd465b5 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -153,22 +153,19 @@ dummy_func( macro(NOT_TAKEN) = NOP; op(_CHECK_PERIODIC, (--)) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - int err = _Py_HandlePending(tstate); - ERROR_IF(err != 0); - } + int err = check_periodics(tstate); + ERROR_IF(err != 0); + } + + replaced op(_CHECK_PERIODIC_AT_END, (--)) { + int err = check_periodics(tstate); + ERROR_IF(err != 0); } op(_CHECK_PERIODIC_IF_NOT_YIELD_FROM, (--)) { if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - int err = _Py_HandlePending(tstate); - ERROR_IF(err != 0); - } + int err = check_periodics(tstate); + ERROR_IF(err != 0); } } @@ -181,7 +178,15 @@ dummy_func( } tier1 op(_MAYBE_INSTRUMENT, (--)) { - if (tstate->tracing == 0) { + #ifdef Py_GIL_DISABLED + // For thread-safety, we need to check instrumentation version + // even when tracing. Otherwise, another thread may concurrently + // re-write the bytecode while we are executing this function. + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { @@ -2964,6 +2969,8 @@ dummy_func( else { this_instr[1].counter = initial_jump_backoff_counter(); assert(tstate->current_executor == NULL); + assert(executor != tstate->interp->cold_executor); + tstate->jit_exit = NULL; GOTO_TIER_TWO(executor); } } @@ -3028,6 +3035,8 @@ dummy_func( } DISPATCH_GOTO(); } + assert(executor != tstate->interp->cold_executor); + tstate->jit_exit = NULL; GOTO_TIER_TWO(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); @@ -3790,8 +3799,8 @@ dummy_func( ERROR_IF(err); } - macro(CALL) = _SPECIALIZE_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC; - macro(INSTRUMENTED_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC; + macro(CALL) = _SPECIALIZE_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC_AT_END; + macro(INSTRUMENTED_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC_AT_END; op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -3912,7 +3921,7 @@ dummy_func( unused/2 + _CHECK_IS_NOT_PY_CALLABLE + _CALL_NON_PY_GENERAL + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { EXIT_IF(!PyStackRef_IsNull(null)); @@ -4069,7 +4078,7 @@ dummy_func( _GUARD_NOS_NULL + _GUARD_CALLABLE_STR_1 + _CALL_STR_1 + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_GUARD_CALLABLE_TUPLE_1, (callable, unused, unused -- callable, unused, unused)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4097,7 +4106,7 @@ dummy_func( _GUARD_NOS_NULL + _GUARD_CALLABLE_TUPLE_1 + _CALL_TUPLE_1 + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4193,7 +4202,7 @@ dummy_func( unused/1 + unused/2 + _CALL_BUILTIN_CLASS + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_O functions */ @@ -4228,7 +4237,7 @@ dummy_func( unused/1 + unused/2 + _CALL_BUILTIN_O + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL functions, without keywords */ @@ -4265,7 +4274,7 @@ dummy_func( unused/1 + unused/2 + _CALL_BUILTIN_FAST + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ @@ -4301,7 +4310,7 @@ dummy_func( unused/1 + unused/2 + _CALL_BUILTIN_FAST_WITH_KEYWORDS + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; macro(CALL_LEN) = unused/1 + @@ -4439,7 +4448,7 @@ dummy_func( unused/1 + unused/2 + _CALL_METHOD_DESCRIPTOR_O + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4481,7 +4490,7 @@ dummy_func( unused/1 + unused/2 + _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res)) { assert(oparg == 0 || oparg == 1); @@ -4519,7 +4528,7 @@ dummy_func( unused/1 + unused/2 + _CALL_METHOD_DESCRIPTOR_NOARGS + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4560,7 +4569,7 @@ dummy_func( unused/1 + unused/2 + _CALL_METHOD_DESCRIPTOR_FAST + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; // Cache layout: counter/1, func_version/2 family(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW) = { @@ -4816,7 +4825,7 @@ dummy_func( unused/2 + _CHECK_IS_NOT_PY_CALLABLE_KW + _CALL_KW_NON_PY + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs -- func, unused, callargs, kwargs)) { PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); @@ -4917,12 +4926,12 @@ dummy_func( macro(CALL_FUNCTION_EX) = _MAKE_CALLARGS_A_TUPLE + _DO_CALL_FUNCTION_EX + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; macro(INSTRUMENTED_CALL_FUNCTION_EX) = _MAKE_CALLARGS_A_TUPLE + _DO_CALL_FUNCTION_EX + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; inst(MAKE_FUNCTION, (codeobj_st -- func)) { PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); @@ -5185,23 +5194,20 @@ dummy_func( op (_GUARD_IS_TRUE_POP, (flag -- )) { int is_true = PyStackRef_IsTrue(flag); DEAD(flag); - SYNC_SP(); - EXIT_IF(!is_true); + AT_END_EXIT_IF(!is_true); } op (_GUARD_IS_FALSE_POP, (flag -- )) { int is_false = PyStackRef_IsFalse(flag); DEAD(flag); - SYNC_SP(); - EXIT_IF(!is_false); + AT_END_EXIT_IF(!is_false); } op (_GUARD_IS_NONE_POP, (val -- )) { int is_none = PyStackRef_IsNone(val); if (!is_none) { PyStackRef_CLOSE(val); - SYNC_SP(); - EXIT_IF(1); + AT_END_EXIT_IF(1); } DEAD(val); } @@ -5209,8 +5215,7 @@ dummy_func( op (_GUARD_IS_NOT_NONE_POP, (val -- )) { int is_none = PyStackRef_IsNone(val); PyStackRef_CLOSE(val); - SYNC_SP(); - EXIT_IF(is_none); + AT_END_EXIT_IF(is_none); } op(_JUMP_TO_TOP, (--)) { @@ -5238,9 +5243,8 @@ dummy_func( tier2 op(_EXIT_TRACE, (exit_p/4 --)) { _PyExitData *exit = (_PyExitData *)exit_p; - PyCodeObject *code = _PyFrame_GetCode(frame); - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; #if defined(Py_DEBUG) && !defined(_Py_JIT) + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { printf("SIDE EXIT: [UOp "); @@ -5251,32 +5255,7 @@ dummy_func( _PyOpcode_OpName[target->op.code]); } #endif - if (exit->executor && !exit->executor->vm_data.valid) { - exit->temperature = initial_temperature_backoff_counter(); - Py_CLEAR(exit->executor); - } - if (exit->executor == NULL) { - _Py_BackoffCounter temperature = exit->temperature; - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); - } - _PyExecutorObject *executor; - if (target->op.code == ENTER_EXECUTOR) { - executor = code->co_executors->executors[target->op.arg]; - Py_INCREF(executor); - } - else { - int chain_depth = current_executor->vm_data.chain_depth + 1; - int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); - if (optimized <= 0) { - exit->temperature = restart_backoff_counter(temperature); - GOTO_TIER_ONE(optimized < 0 ? NULL : target); - } - exit->temperature = initial_temperature_backoff_counter(); - } - exit->executor = executor; - } + tstate->jit_exit = exit; GOTO_TIER_TWO(exit->executor); } @@ -5375,7 +5354,14 @@ dummy_func( #ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; #endif - assert(((_PyExecutorObject *)executor)->vm_data.valid); + assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor); + tstate->current_executor = (PyObject *)executor; + if (!current_executor->vm_data.valid) { + assert(tstate->jit_exit->executor == current_executor); + assert(tstate->current_executor == executor); + _PyExecutor_ClearExit(tstate->jit_exit); + DEOPT_IF(true); + } } tier2 op(_MAKE_WARM, (--)) { @@ -5395,6 +5381,11 @@ dummy_func( GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } + tier2 op(_HANDLE_PENDING_AND_DEOPT, (--)) { + int err = _Py_HandlePending(tstate); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + } + tier2 op(_ERROR_POP_N, (target/2 --)) { assert(oparg == 0); frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; @@ -5406,14 +5397,45 @@ dummy_func( * ENTER_EXECUTOR will not re-enter tier 2 with the eval breaker set. */ tier2 op(_TIER2_RESUME_CHECK, (--)) { #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0); + HANDLE_PENDING_AND_DEOPT_IF(_Py_emscripten_signal_clock == 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - DEOPT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK); + HANDLE_PENDING_AND_DEOPT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK); assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); } + tier2 op(_COLD_EXIT, ( -- )) { + _PyExitData *exit = tstate->jit_exit; + assert(exit != NULL); + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + _Py_BackoffCounter temperature = exit->temperature; + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target); + } + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } + else { + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + assert(tstate->current_executor == (PyObject *)previous_executor); + int chain_depth = previous_executor->vm_data.chain_depth + 1; + int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); + if (optimized <= 0) { + exit->temperature = restart_backoff_counter(temperature); + GOTO_TIER_ONE(optimized < 0 ? NULL : target); + } + exit->temperature = initial_temperature_backoff_counter(); + } + assert(tstate->jit_exit == exit); + exit->executor = executor; + GOTO_TIER_TWO(exit->executor); + } + label(pop_2_error) { stack_pointer -= 2; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/ceval.c b/Python/ceval.c index 9ccd42bdf0a55c..b8c1dd3e3bf74b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1158,7 +1158,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int uint64_t trace_uop_execution_counter = 0; #endif - assert(next_uop->opcode == _START_EXECUTOR); + assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); tier2_dispatch: for (;;) { uopcode = next_uop->opcode; @@ -1211,6 +1211,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int printf(" @ %d -> %s]\n", (int)(next_uop - current_executor->trace - 1), _PyOpcode_OpName[frame->instr_ptr->op.code]); + fflush(stdout); } #endif assert(next_uop[-1].format == UOP_FORMAT_JUMP); @@ -2509,21 +2510,10 @@ PyEval_SetProfile(Py_tracefunc func, PyObject *arg) void PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *arg) { - PyThreadState *this_tstate = _PyThreadState_GET(); - PyInterpreterState* interp = this_tstate->interp; - - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); - - while (ts) { - if (_PyEval_SetProfile(ts, func, arg) < 0) { - PyErr_FormatUnraisable("Exception ignored in PyEval_SetProfileAllThreads"); - } - HEAD_LOCK(runtime); - ts = PyThreadState_Next(ts); - HEAD_UNLOCK(runtime); + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyEval_SetProfileAllThreads(interp, func, arg) < 0) { + /* Log _PySys_Audit() error */ + PyErr_FormatUnraisable("Exception ignored in PyEval_SetProfileAllThreads"); } } @@ -2540,21 +2530,10 @@ PyEval_SetTrace(Py_tracefunc func, PyObject *arg) void PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *arg) { - PyThreadState *this_tstate = _PyThreadState_GET(); - PyInterpreterState* interp = this_tstate->interp; - - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); - - while (ts) { - if (_PyEval_SetTrace(ts, func, arg) < 0) { - PyErr_FormatUnraisable("Exception ignored in PyEval_SetTraceAllThreads"); - } - HEAD_LOCK(runtime); - ts = PyThreadState_Next(ts); - HEAD_UNLOCK(runtime); + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyEval_SetTraceAllThreads(interp, func, arg) < 0) { + /* Log _PySys_Audit() error */ + PyErr_FormatUnraisable("Exception ignored in PyEval_SetTraceAllThreads"); } } diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index aa68371ac8febf..6bf64868cbb2d3 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -907,13 +907,9 @@ unsignal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp) static void clear_pending_handling_thread(struct _pending_calls *pending) { -#ifdef Py_GIL_DISABLED - PyMutex_Lock(&pending->mutex); - pending->handling_thread = NULL; - PyMutex_Unlock(&pending->mutex); -#else + FT_MUTEX_LOCK(&pending->mutex); pending->handling_thread = NULL; -#endif + FT_MUTEX_UNLOCK(&pending->mutex); } static int diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 187ec8fdd26584..ddbcf2aa1bab02 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -359,7 +359,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer) do { \ OPT_STAT_INC(traces_executed); \ _PyExecutorObject *_executor = (EXECUTOR); \ - tstate->current_executor = (PyObject *)_executor; \ jit_func jitted = _executor->jit_code; \ /* Keep the shim frame alive via the executor: */ \ Py_INCREF(_executor); \ @@ -378,9 +377,8 @@ do { \ do { \ OPT_STAT_INC(traces_executed); \ _PyExecutorObject *_executor = (EXECUTOR); \ - tstate->current_executor = (PyObject *)_executor; \ next_uop = _executor->trace; \ - assert(next_uop->opcode == _START_EXECUTOR); \ + assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); \ goto enter_tier_two; \ } while (0) #endif @@ -390,7 +388,6 @@ do { \ { \ tstate->current_executor = NULL; \ next_instr = (TARGET); \ - assert(tstate->current_executor == NULL); \ OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ stack_pointer = _PyFrame_GetStackPointer(frame); \ @@ -425,3 +422,14 @@ do { \ _PyObjectArray_Free(NAME - 1, NAME##_temp); #define CONVERSION_FAILED(NAME) ((NAME) == NULL) + +static inline int +check_periodics(PyThreadState *tstate) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + return _Py_HandlePending(tstate); + } + return 0; +} + diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index c826a5724f769c..c1fcbbba8e2f57 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -831,10 +831,16 @@ PyDoc_STRVAR(builtin_oct__doc__, {"oct", (PyCFunction)builtin_oct, METH_O, builtin_oct__doc__}, PyDoc_STRVAR(builtin_ord__doc__, -"ord($module, c, /)\n" +"ord($module, character, /)\n" "--\n" "\n" -"Return the Unicode code point for a one-character string."); +"Return the ordinal value of a character.\n" +"\n" +"If the argument is a one-character string, return the Unicode code\n" +"point of that character.\n" +"\n" +"If the argument is a bytes or bytearray object of length 1, return its\n" +"single byte value."); #define BUILTIN_ORD_METHODDEF \ {"ord", (PyCFunction)builtin_ord, METH_O, builtin_ord__doc__}, @@ -1268,4 +1274,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=e7a5d0851d7f2cfb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cd5f80e3dc3082d5 input=a9049054013a1b77]*/ diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index a47e4d11b54441..09ce77fd12608f 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1793,6 +1793,37 @@ sys__baserepl(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys__baserepl_impl(module); } +PyDoc_STRVAR(sys__clear_type_descriptors__doc__, +"_clear_type_descriptors($module, type, /)\n" +"--\n" +"\n" +"Private function for clearing certain descriptors from a type\'s dictionary.\n" +"\n" +"See gh-135228 for context."); + +#define SYS__CLEAR_TYPE_DESCRIPTORS_METHODDEF \ + {"_clear_type_descriptors", (PyCFunction)sys__clear_type_descriptors, METH_O, sys__clear_type_descriptors__doc__}, + +static PyObject * +sys__clear_type_descriptors_impl(PyObject *module, PyObject *type); + +static PyObject * +sys__clear_type_descriptors(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *type; + + if (!PyObject_TypeCheck(arg, &PyType_Type)) { + _PyArg_BadArgument("_clear_type_descriptors", "argument", (&PyType_Type)->tp_name, arg); + goto exit; + } + type = arg; + return_value = sys__clear_type_descriptors_impl(module, type); + +exit: + return return_value; +} + PyDoc_STRVAR(sys__is_gil_enabled__doc__, "_is_gil_enabled($module, /)\n" "--\n" @@ -1948,4 +1979,4 @@ _jit_is_active(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=449d16326e69dcf6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9052f399f40ca32d input=a9049054013a1b77]*/ diff --git a/Python/codecs.c b/Python/codecs.c index caf8d9d5f3c188..8eb9f2db41359e 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -16,7 +16,7 @@ Copyright (c) Corporation for National Research Initiatives. #include "pycore_runtime.h" // _Py_ID() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI #include "pycore_unicodeobject.h" // _PyUnicode_InternMortal() - +#include "pycore_pyatomic_ft_wrappers.h" static const char *codecs_builtin_error_handlers[] = { "strict", "ignore", "replace", @@ -40,13 +40,10 @@ int PyCodec_Register(PyObject *search_function) PyErr_SetString(PyExc_TypeError, "argument must be callable"); goto onError; } -#ifdef Py_GIL_DISABLED - PyMutex_Lock(&interp->codecs.search_path_mutex); -#endif + FT_MUTEX_LOCK(&interp->codecs.search_path_mutex); int ret = PyList_Append(interp->codecs.search_path, search_function); -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&interp->codecs.search_path_mutex); -#endif + FT_MUTEX_UNLOCK(&interp->codecs.search_path_mutex); + return ret; onError: @@ -66,9 +63,7 @@ PyCodec_Unregister(PyObject *search_function) PyObject *codec_search_path = interp->codecs.search_path; assert(PyList_CheckExact(codec_search_path)); for (Py_ssize_t i = 0; i < PyList_GET_SIZE(codec_search_path); i++) { -#ifdef Py_GIL_DISABLED - PyMutex_Lock(&interp->codecs.search_path_mutex); -#endif + FT_MUTEX_LOCK(&interp->codecs.search_path_mutex); PyObject *item = PyList_GetItemRef(codec_search_path, i); int ret = 1; if (item == search_function) { @@ -76,9 +71,7 @@ PyCodec_Unregister(PyObject *search_function) // while we hold search_path_mutex. ret = PyList_SetSlice(codec_search_path, i, i+1, NULL); } -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&interp->codecs.search_path_mutex); -#endif + FT_MUTEX_UNLOCK(&interp->codecs.search_path_mutex); Py_DECREF(item); if (ret != 1) { assert(interp->codecs.search_cache != NULL); @@ -1204,7 +1197,7 @@ get_standard_encoding_impl(const char *encoding, int *bytelength) } } } - else if (strcmp(encoding, "CP_UTF8") == 0) { + else if (strcmp(encoding, "cp65001") == 0) { *bytelength = 3; return ENC_UTF8; } diff --git a/Python/errors.c b/Python/errors.c index a3122f76bdd87d..2688396004e98b 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1444,12 +1444,16 @@ make_unraisable_hook_args(PyThreadState *tstate, PyObject *exc_type, It can be called to log the exception of a custom sys.unraisablehook. - Do nothing if sys.stderr attribute doesn't exist or is set to None. */ + This assumes 'file' is neither NULL nor None. + */ static int write_unraisable_exc_file(PyThreadState *tstate, PyObject *exc_type, PyObject *exc_value, PyObject *exc_tb, PyObject *err_msg, PyObject *obj, PyObject *file) { + assert(file != NULL); + assert(!Py_IsNone(file)); + if (obj != NULL && obj != Py_None) { if (err_msg != NULL && err_msg != Py_None) { if (PyFile_WriteObject(err_msg, file, Py_PRINT_RAW) < 0) { @@ -1484,6 +1488,27 @@ write_unraisable_exc_file(PyThreadState *tstate, PyObject *exc_type, } } + // Try printing the exception using the stdlib module. + // If this fails, then we have to use the C implementation. + PyObject *print_exception_fn = PyImport_ImportModuleAttrString("traceback", + "_print_exception_bltin"); + if (print_exception_fn != NULL && PyCallable_Check(print_exception_fn)) { + PyObject *args[2] = {exc_value, file}; + PyObject *result = PyObject_Vectorcall(print_exception_fn, args, 2, NULL); + int ok = (result != NULL); + Py_DECREF(print_exception_fn); + Py_XDECREF(result); + if (ok) { + // Nothing else to do + return 0; + } + } + else { + Py_XDECREF(print_exception_fn); + } + // traceback module failed, fall back to pure C + _PyErr_Clear(tstate); + if (exc_tb != NULL && exc_tb != Py_None) { if (PyTraceBack_Print(exc_tb, file) < 0) { /* continue even if writing the traceback failed */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e152865e4ec9e8..182289922c7637 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -13,31 +13,25 @@ } case _CHECK_PERIODIC: { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_ERROR(); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_ERROR(); } break; } + /* _CHECK_PERIODIC_AT_END is not a viable micro-op for tier 2 because it is replaced */ + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { oparg = CURRENT_OPARG(); if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_ERROR(); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_ERROR(); } } break; @@ -7113,9 +7107,8 @@ case _EXIT_TRACE: { PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _PyExitData *exit = (_PyExitData *)exit_p; - PyCodeObject *code = _PyFrame_GetCode(frame); - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; #if defined(Py_DEBUG) && !defined(_Py_JIT) + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7128,36 +7121,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } #endif - if (exit->executor && !exit->executor->vm_data.valid) { - exit->temperature = initial_temperature_backoff_counter(); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(exit->executor); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (exit->executor == NULL) { - _Py_BackoffCounter temperature = exit->temperature; - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); - } - _PyExecutorObject *executor; - if (target->op.code == ENTER_EXECUTOR) { - executor = code->co_executors->executors[target->op.arg]; - Py_INCREF(executor); - } - else { - int chain_depth = current_executor->vm_data.chain_depth + 1; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - exit->temperature = restart_backoff_counter(temperature); - GOTO_TIER_ONE(optimized < 0 ? NULL : target); - } - exit->temperature = initial_temperature_backoff_counter(); - } - exit->executor = executor; - } + tstate->jit_exit = exit; GOTO_TIER_TWO(exit->executor); break; } @@ -7438,7 +7402,19 @@ #ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; #endif - assert(((_PyExecutorObject *)executor)->vm_data.valid); + assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor); + tstate->current_executor = (PyObject *)executor; + if (!current_executor->vm_data.valid) { + assert(tstate->jit_exit->executor == current_executor); + assert(tstate->current_executor == executor); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyExecutor_ClearExit(tstate->jit_exit); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } break; } @@ -7461,6 +7437,14 @@ break; } + case _HANDLE_PENDING_AND_DEOPT: { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + break; + } + case _ERROR_POP_N: { oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND0(); @@ -7487,4 +7471,40 @@ break; } + case _COLD_EXIT: { + _PyExitData *exit = tstate->jit_exit; + assert(exit != NULL); + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + _Py_BackoffCounter temperature = exit->temperature; + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target); + } + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(tstate->current_executor == (PyObject *)previous_executor); + int chain_depth = previous_executor->vm_data.chain_depth + 1; + _PyFrame_SetStackPointer(frame, stack_pointer); + int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (optimized <= 0) { + exit->temperature = restart_backoff_counter(temperature); + GOTO_TIER_ONE(optimized < 0 ? NULL : target); + } + exit->temperature = initial_temperature_backoff_counter(); + } + assert(tstate->jit_exit == exit); + exit->executor = executor; + GOTO_TIER_TWO(exit->executor); + break; + } + #undef TIER_TWO diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 1cb6f03169e3b5..f8a4fa60f223df 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -199,8 +199,10 @@ basicblock_addop(basicblock *b, int opcode, int oparg, location loc) cfg_instr *i = &b->b_instr[off]; i->i_opcode = opcode; i->i_oparg = oparg; - i->i_target = NULL; i->i_loc = loc; + // memory is already zero initialized + assert(i->i_target == NULL); + assert(i->i_except == NULL); return SUCCESS; } @@ -1104,6 +1106,7 @@ basicblock_remove_redundant_nops(basicblock *bb) { assert(dest <= bb->b_iused); int num_removed = bb->b_iused - dest; bb->b_iused = dest; + memset(&bb->b_instr[dest], 0, sizeof(cfg_instr) * num_removed); return num_removed; } @@ -3472,11 +3475,13 @@ convert_pseudo_conditional_jumps(cfg_builder *g) instr->i_opcode = instr->i_opcode == JUMP_IF_FALSE ? POP_JUMP_IF_FALSE : POP_JUMP_IF_TRUE; location loc = instr->i_loc; + basicblock *except = instr->i_except; cfg_instr copy = { .i_opcode = COPY, .i_oparg = 1, .i_loc = loc, .i_target = NULL, + .i_except = except, }; RETURN_IF_ERROR(basicblock_insert_instruction(b, i++, ©)); cfg_instr to_bool = { @@ -3484,6 +3489,7 @@ convert_pseudo_conditional_jumps(cfg_builder *g) .i_oparg = 0, .i_loc = loc, .i_target = NULL, + .i_except = except, }; RETURN_IF_ERROR(basicblock_insert_instruction(b, i++, &to_bool)); } @@ -3726,6 +3732,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl .i_oparg = 0, .i_loc = loc, .i_target = NULL, + .i_except = NULL, }; RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 0, &make_gen)); cfg_instr pop_top = { @@ -3733,6 +3740,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl .i_oparg = 0, .i_loc = loc, .i_target = NULL, + .i_except = NULL, }; RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 1, &pop_top)); } @@ -3763,6 +3771,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl .i_oparg = oldindex, .i_loc = NO_LOCATION, .i_target = NULL, + .i_except = NULL, }; if (basicblock_insert_instruction(entryblock, ncellsused, &make_cell) < 0) { PyMem_RawFree(sorted); @@ -3779,6 +3788,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl .i_oparg = nfreevars, .i_loc = NO_LOCATION, .i_target = NULL, + .i_except = NULL, }; RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 0, ©_frees)); } diff --git a/Python/frozenmain.c b/Python/frozenmain.c index ec4566bd4f84bc..3de587c0423226 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -1,8 +1,7 @@ /* Python interpreter main program for frozen scripts */ #include "Python.h" -#include "pycore_pystate.h" // _Py_GetConfig() -#include "pycore_runtime.h" // _PyRuntime_Initialize() +#include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() #ifdef HAVE_UNISTD_H # include // isatty() @@ -20,11 +19,6 @@ extern int PyInitFrozenExtensions(void); int Py_FrozenMain(int argc, char **argv) { - PyStatus status = _PyRuntime_Initialize(); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - PyConfig config; PyConfig_InitPythonConfig(&config); // Suppress errors from getpath.c @@ -32,7 +26,7 @@ Py_FrozenMain(int argc, char **argv) // Don't parse command line options like -E config.parse_argv = 0; - status = PyConfig_SetBytesArgv(&config, argc, argv); + PyStatus status = PyConfig_SetBytesArgv(&config, argc, argv); if (PyStatus_Exception(status)) { PyConfig_Clear(&config); Py_ExitStatusException(status); @@ -64,7 +58,12 @@ Py_FrozenMain(int argc, char **argv) PyWinFreeze_ExeInit(); #endif - if (_Py_GetConfig()->verbose) { + int verbose; + if (PyConfig_GetInt("verbose", &verbose) < 0) { + verbose = 0; + PyErr_Clear(); + } + if (verbose) { fprintf(stderr, "Python %s\n%s\n", Py_GetVersion(), Py_GetCopyright()); } diff --git a/Python/gc.c b/Python/gc.c index 2e697faac032b5..79c7476f4a9a74 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -570,8 +570,7 @@ _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) } /* Subtract internal references from gc_refs. After this, gc_refs is >= 0 - * for all objects in containers, and is GC_REACHABLE for all tracked gc - * objects not in containers. The ones with gc_refs > 0 are directly + * for all objects in containers. The ones with gc_refs > 0 are directly * reachable from outside containers, and so can't be collected. */ static void @@ -858,60 +857,65 @@ move_legacy_finalizer_reachable(PyGC_Head *finalizers) } } -/* Clear all weakrefs to unreachable objects, and if such a weakref has a - * callback, invoke it if necessary. Note that it's possible for such - * weakrefs to be outside the unreachable set -- indeed, those are precisely - * the weakrefs whose callbacks must be invoked. See gc_weakref.txt for - * overview & some details. Some weakrefs with callbacks may be reclaimed - * directly by this routine; the number reclaimed is the return value. Other - * weakrefs with callbacks may be moved into the `old` generation. Objects - * moved into `old` have gc_refs set to GC_REACHABLE; the objects remaining in - * unreachable are left at GC_TENTATIVELY_UNREACHABLE. When this returns, - * no object in `unreachable` is weakly referenced anymore. +/* Handle weakref callbacks. Note that it's possible for such weakrefs to be + * outside the unreachable set -- indeed, those are precisely the weakrefs + * whose callbacks must be invoked. See gc_weakref.txt for overview & some + * details. + * + * The clearing of weakrefs is suble and must be done carefully, as there was + * previous bugs related to this. First, weakrefs to the unreachable set of + * objects must be cleared before we start calling `tp_clear`. If we don't, + * those weakrefs can reveal unreachable objects to Python-level code and that + * is not safe. Many objects are not usable after `tp_clear` has been called + * and could even cause crashes if accessed (see bpo-38006 and gh-91636 as + * example bugs). + * + * Weakrefs with callbacks always need to be cleared before executing the + * callback. That's because sometimes a callback will call the ref object, + * to check if the reference is actually dead (KeyedRef does this, for + * example). We want to indicate that it is dead, even though it is possible + * a finalizer might resurrect it. Clearing also prevents the callback from + * executing more than once. + * + * Since Python 2.3, all weakrefs to cyclic garbage have been cleared *before* + * calling finalizers. However, since tp_subclasses started being necessary + * to invalidate caches (e.g. by PyType_Modified), that clearing has created + * a bug. If the weakref to the subclass is cleared before a finalizer is + * called, the cache may not be correctly invalidated. That can lead to + * segfaults since the caches can refer to deallocated objects (GH-135552 + * is an example). Now, we delay the clear of weakrefs without callbacks + * until *after* finalizers have been executed. That means weakrefs without + * callbacks are still usable while finalizers are being executed. + * + * The weakrefs that are inside the unreachable set must also be cleared. + * The reason this is required is not immediately obvious. If the weakref + * refers to an object outside of the unreachable set, that object might go + * away when we start clearing objects. Normally, the object should also be + * part of the unreachable set but that's not true in the case of incomplete + * or missing `tp_traverse` methods. When that object goes away, the callback + * for weakref can be executed and that could reveal unreachable objects to + * Python-level code. See bpo-38006 as an example bug. */ static int -handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old, bool allow_callbacks) +handle_weakref_callbacks(PyGC_Head *unreachable, PyGC_Head *old) { PyGC_Head *gc; - PyObject *op; /* generally FROM_GC(gc) */ - PyWeakReference *wr; /* generally a cast of op */ PyGC_Head wrcb_to_call; /* weakrefs with callbacks to call */ PyGC_Head *next; int num_freed = 0; - if (allow_callbacks) { - gc_list_init(&wrcb_to_call); - } + gc_list_init(&wrcb_to_call); - /* Clear all weakrefs to the objects in unreachable. If such a weakref - * also has a callback, move it into `wrcb_to_call` if the callback - * needs to be invoked. Note that we cannot invoke any callbacks until - * all weakrefs to unreachable objects are cleared, lest the callback - * resurrect an unreachable object via a still-active weakref. We - * make another pass over wrcb_to_call, invoking callbacks, after this - * pass completes. + /* Find all weakrefs with callbacks and move into `wrcb_to_call` if the + * callback needs to be invoked. We make another pass over wrcb_to_call, + * invoking callbacks, after this pass completes. */ for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { PyWeakReference **wrlist; - op = FROM_GC(gc); + PyObject *op = FROM_GC(gc); next = GC_NEXT(gc); - if (PyWeakref_Check(op)) { - /* A weakref inside the unreachable set must be cleared. If we - * allow its callback to execute inside delete_garbage(), it - * could expose objects that have tp_clear already called on - * them. Or, it could resurrect unreachable objects. One way - * this can happen is if some container objects do not implement - * tp_traverse. Then, wr_object can be outside the unreachable - * set but can be deallocated as a result of breaking the - * reference cycle. If we don't clear the weakref, the callback - * will run and potentially cause a crash. See bpo-38006 for - * one example. - */ - _PyWeakref_ClearRef((PyWeakReference *)op); - } - if (! _PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { continue; } @@ -923,30 +927,29 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old, bool allow_callbacks) */ wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op); - /* `op` may have some weakrefs. March over the list, clear - * all the weakrefs, and move the weakrefs with callbacks - * that must be called into wrcb_to_call. + /* `op` may have some weakrefs. March over the list and move the + * weakrefs with callbacks that must be called into wrcb_to_call. */ - for (wr = *wrlist; wr != NULL; wr = *wrlist) { - PyGC_Head *wrasgc; /* AS_GC(wr) */ - - /* _PyWeakref_ClearRef clears the weakref but leaves - * the callback pointer intact. Obscure: it also - * changes *wrlist. - */ - _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); - _PyWeakref_ClearRef(wr); - _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); - - if (!allow_callbacks) { - continue; - } + PyWeakReference *next_wr; + for (PyWeakReference *wr = *wrlist; wr != NULL; wr = next_wr) { + // Get the next list element to get iterator progress if we omit + // clearing of the weakref (because _PyWeakref_ClearRef changes + // next pointer in the wrlist). + next_wr = wr->wr_next; if (wr->wr_callback == NULL) { /* no callback */ continue; } + // Clear the weakref. See the comments above this function for + // reasons why we need to clear weakrefs that have callbacks. + // Note that _PyWeakref_ClearRef clears the weakref but leaves the + // callback pointer intact. Obscure: it also changes *wrlist. + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + /* Headache time. `op` is going away, and is weakly referenced by * `wr`, which has a callback. Should the callback be invoked? If wr * is also trash, no: @@ -962,10 +965,10 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old, bool allow_callbacks) * outside the current generation, CT may be reachable from the * callback. Then the callback could resurrect insane objects. * - * Since the callback is never needed and may be unsafe in this case, - * wr is simply left in the unreachable set. Note that because we - * already called _PyWeakref_ClearRef(wr), its callback will never - * trigger. + * Since the callback is never needed and may be unsafe in this + * case, wr is simply left in the unreachable set. Note that + * clear_weakrefs() will ensure its callback will not trigger + * inside delete_garbage(). * * OTOH, if wr isn't part of CT, we should invoke the callback: the * weakref outlived the trash. Note that since wr isn't CT in this @@ -976,8 +979,6 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old, bool allow_callbacks) * is moved to wrcb_to_call in this case. */ if (gc_is_collecting(AS_GC((PyObject *)wr))) { - /* it should already have been cleared above */ - _PyObject_ASSERT((PyObject*)wr, wr->wr_object == Py_None); continue; } @@ -987,17 +988,13 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old, bool allow_callbacks) Py_INCREF(wr); /* Move wr to wrcb_to_call, for the next pass. */ - wrasgc = AS_GC((PyObject *)wr); + PyGC_Head *wrasgc = AS_GC((PyObject *)wr); // wrasgc is reachable, but next isn't, so they can't be the same _PyObject_ASSERT((PyObject *)wr, wrasgc != next); gc_list_move(wrasgc, &wrcb_to_call); } } - if (!allow_callbacks) { - return 0; - } - /* Invoke the callbacks we decided to honor. It's safe to invoke them * because they can't reference unreachable objects. */ @@ -1007,9 +1004,9 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old, bool allow_callbacks) PyObject *callback; gc = (PyGC_Head*)wrcb_to_call._gc_next; - op = FROM_GC(gc); + PyObject *op = FROM_GC(gc); _PyObject_ASSERT(op, PyWeakref_Check(op)); - wr = (PyWeakReference *)op; + PyWeakReference *wr = (PyWeakReference *)op; callback = wr->wr_callback; _PyObject_ASSERT(op, callback != NULL); @@ -1048,6 +1045,56 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old, bool allow_callbacks) return num_freed; } +/* Clear all weakrefs to unreachable objects. When this returns, no object in + * `unreachable` is weakly referenced anymore. See the comments above + * handle_weakref_callbacks() for why these weakrefs need to be cleared. + */ +static void +clear_weakrefs(PyGC_Head *unreachable) +{ + PyGC_Head *gc; + PyGC_Head *next; + + for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { + PyWeakReference **wrlist; + + PyObject *op = FROM_GC(gc); + next = GC_NEXT(gc); + + if (PyWeakref_Check(op)) { + /* A weakref inside the unreachable set is always cleared. See + * the comments above handle_weakref_callbacks() for why these + * must be cleared. + */ + _PyWeakref_ClearRef((PyWeakReference *)op); + } + + if (! _PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { + continue; + } + + /* It supports weakrefs. Does it have any? + * + * This is never triggered for static types so we can avoid the + * (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). + */ + wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op); + + /* `op` may have some weakrefs. March over the list, clear + * all the weakrefs. + */ + for (PyWeakReference *wr = *wrlist; wr != NULL; wr = *wrlist) { + /* _PyWeakref_ClearRef clears the weakref but leaves + * the callback pointer intact. Obscure: it also + * changes *wrlist. + */ + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + } + } +} + static void debug_cycle(const char *msg, PyObject *op) { @@ -1710,12 +1757,13 @@ gc_collect_region(PyThreadState *tstate, deduce_unreachable(from, &unreachable); validate_consistent_old_space(from); untrack_tuples(from); + + /* Move reachable objects to next generation. */ validate_consistent_old_space(to); if (from != to) { gc_list_merge(from, to); } validate_consistent_old_space(to); - /* Move reachable objects to next generation. */ /* All objects in unreachable are trash, but objects reachable from * legacy finalizers (e.g. tp_del) can't safely be deleted. @@ -1738,8 +1786,8 @@ gc_collect_region(PyThreadState *tstate, } } - /* Clear weakrefs and invoke callbacks as necessary. */ - stats->collected += handle_weakrefs(&unreachable, to, true); + /* Invoke weakref callbacks as necessary. */ + stats->collected += handle_weakref_callbacks(&unreachable, to); gc_list_validate_space(to, gcstate->visited_space); validate_list(to, collecting_clear_unreachable_clear); validate_list(&unreachable, collecting_set_unreachable_clear); @@ -1753,13 +1801,10 @@ gc_collect_region(PyThreadState *tstate, gc_list_init(&final_unreachable); handle_resurrected_objects(&unreachable, &final_unreachable, to); - /* Clear weakrefs to objects in the unreachable set. No Python-level - * code must be allowed to access those unreachable objects. During - * delete_garbage(), finalizers outside the unreachable set might run - * and create new weakrefs. If those weakrefs were not cleared, they - * could reveal unreachable objects. Callbacks are not executed. + /* Clear weakrefs to objects in the unreachable set. See the comments + * above handle_weakref_callbacks() for details. */ - handle_weakrefs(&final_unreachable, NULL, false); + clear_weakrefs(&final_unreachable); /* Call tp_clear on objects in the final_unreachable set. This will cause * the reference cycles to be broken. It may also cause some objects diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 0b0ddf227e4952..842aa3401548c9 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1492,22 +1492,49 @@ move_legacy_finalizer_reachable(struct collection_state *state) return 0; } -// Clear all weakrefs to unreachable objects. Weakrefs with callbacks are -// optionally enqueued in `wrcb_to_call`, but not invoked yet. +// Weakrefs with callbacks are enqueued in `wrcb_to_call`, but not invoked +// yet. Note that it's possible for such weakrefs to be outside the +// unreachable set -- indeed, those are precisely the weakrefs whose callbacks +// must be invoked. See gc_weakref.txt for overview & some details. +// +// The clearing of weakrefs is suble and must be done carefully, as there was +// previous bugs related to this. First, weakrefs to the unreachable set of +// objects must be cleared before we start calling `tp_clear`. If we don't, +// those weakrefs can reveal unreachable objects to Python-level code and that +// is not safe. Many objects are not usable after `tp_clear` has been called +// and could even cause crashes if accessed (see bpo-38006 and gh-91636 as +// example bugs). +// +// Weakrefs with callbacks always need to be cleared before executing the +// callback. That's because sometimes a callback will call the ref object, +// to check if the reference is actually dead (KeyedRef does this, for +// example). We want to indicate that it is dead, even though it is possible +// a finalizer might resurrect it. Clearing also prevents the callback from +// executing more than once. +// +// Since Python 2.3, all weakrefs to cyclic garbage have been cleared *before* +// calling finalizers. However, since tp_subclasses started being necessary +// to invalidate caches (e.g. by PyType_Modified), that clearing has created +// a bug. If the weakref to the subclass is cleared before a finalizer is +// called, the cache may not be correctly invalidated. That can lead to +// segfaults since the caches can refer to deallocated objects (GH-91636 +// is an example). Now, we delay the clear of weakrefs without callbacks +// until *after* finalizers have been executed. That means weakrefs without +// callbacks are still usable while finalizers are being executed. +// +// The weakrefs that are inside the unreachable set must also be cleared. +// The reason this is required is not immediately obvious. If the weakref +// refers to an object outside of the unreachable set, that object might go +// away when we start clearing objects. Normally, the object should also be +// part of the unreachable set but that's not true in the case of incomplete +// or missing `tp_traverse` methods. When that object goes away, the callback +// for weakref can be executed and that could reveal unreachable objects to +// Python-level code. See bpo-38006 as an example bug. static void -clear_weakrefs(struct collection_state *state, bool enqueue_callbacks) +find_weakref_callbacks(struct collection_state *state) { PyObject *op; WORKSTACK_FOR_EACH(&state->unreachable, op) { - if (PyWeakref_Check(op)) { - // Clear weakrefs that are themselves unreachable to ensure their - // callbacks will not be executed later from a `tp_clear()` - // inside delete_garbage(). That would be unsafe: it could - // resurrect a dead object or access a an already cleared object. - // See bpo-38006 for one example. - _PyWeakref_ClearRef((PyWeakReference *)op); - } - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { continue; } @@ -1519,16 +1546,21 @@ clear_weakrefs(struct collection_state *state, bool enqueue_callbacks) // `op` may have some weakrefs. March over the list, clear // all the weakrefs, and enqueue the weakrefs with callbacks // that must be called into wrcb_to_call. - for (PyWeakReference *wr = *wrlist; wr != NULL; wr = *wrlist) { - // _PyWeakref_ClearRef clears the weakref but leaves - // the callback pointer intact. Obscure: it also - // changes *wrlist. - _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); - _PyWeakref_ClearRef(wr); - _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); - - if (!enqueue_callbacks) { - continue; + PyWeakReference *next_wr; + for (PyWeakReference *wr = *wrlist; wr != NULL; wr = next_wr) { + // Get the next list element to get iterator progress if we omit + // clearing of the weakref (because _PyWeakref_ClearRef changes + // next pointer in the wrlist). + next_wr = wr->wr_next; + + // Weakrefs with callbacks always need to be cleared before + // executing the callback. + if (wr->wr_callback != NULL) { + // _PyWeakref_ClearRef clears the weakref but leaves the + // callback pointer intact. Obscure: it also changes *wrlist. + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); } // We do not invoke callbacks for weakrefs that are themselves @@ -1550,6 +1582,39 @@ clear_weakrefs(struct collection_state *state, bool enqueue_callbacks) } } +// Clear weakrefs to objects in the unreachable set. See comments +// above find_weakref_callbacks() for why this clearing is required. +static void +clear_weakrefs(struct collection_state *state) +{ + PyObject *op; + WORKSTACK_FOR_EACH(&state->unreachable, op) { + if (PyWeakref_Check(op)) { + // Clear weakrefs that are themselves unreachable. + _PyWeakref_ClearRef((PyWeakReference *)op); + } + + if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { + continue; + } + + // NOTE: This is never triggered for static types so we can avoid the + // (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). + PyWeakReference **wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op); + + // `op` may have some weakrefs. March over the list, clear + // all the weakrefs. + for (PyWeakReference *wr = *wrlist; wr != NULL; wr = *wrlist) { + // _PyWeakref_ClearRef clears the weakref but leaves + // the callback pointer intact. Obscure: it also + // changes *wrlist. + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + } + } +} + static void call_weakref_callbacks(struct collection_state *state) { @@ -2222,8 +2287,8 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, // Record the number of live GC objects interp->gc.long_lived_total = state->long_lived_total; - // Clear weakrefs and enqueue callbacks (but do not call them). - clear_weakrefs(state, true); + // Find weakref callbacks we will honor (but do not call them). + find_weakref_callbacks(state); _PyEval_StartTheWorld(interp); // Deallocate any object from the refcount merge step @@ -2240,12 +2305,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, // Clear free lists in all threads _PyGC_ClearAllFreeLists(interp); if (err == 0) { - // Clear weakrefs to objects in the unreachable set. No Python-level - // code must be allowed to access those unreachable objects. During - // delete_garbage(), finalizers outside the unreachable set might - // run and create new weakrefs. If those weakrefs were not cleared, - // they could reveal unreachable objects. - clear_weakrefs(state, false); + clear_weakrefs(state); } _PyEval_StartTheWorld(interp); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index aa1eb373b7ba4b..d63225bb0cd3cd 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1670,26 +1670,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2175,26 +2167,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2292,26 +2276,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2409,26 +2385,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2501,26 +2469,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2676,26 +2636,18 @@ } result = PyStackRef_FromPyObjectSteal(result_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3268,26 +3220,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3659,26 +3603,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3790,26 +3726,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3891,26 +3819,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4005,26 +3925,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4128,26 +4040,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4421,26 +4325,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4501,26 +4397,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5595,6 +5483,8 @@ } DISPATCH_GOTO(); } + assert(executor != tstate->interp->cold_executor); + tstate->jit_exit = NULL; GOTO_TIER_TWO(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); @@ -6515,26 +6405,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6690,26 +6572,18 @@ } result = PyStackRef_FromPyObjectSteal(result_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -7091,15 +6965,11 @@ /* Skip 1 cache entry */ // _CHECK_PERIODIC { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } // _MONITOR_JUMP_BACKWARD @@ -7464,7 +7334,13 @@ } // _MAYBE_INSTRUMENT { - if (tstate->tracing == 0) { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { @@ -7482,15 +7358,11 @@ // _CHECK_PERIODIC_IF_NOT_YIELD_FROM { if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } } @@ -7719,15 +7591,11 @@ } // _CHECK_PERIODIC { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -7752,15 +7620,11 @@ /* Skip 1 cache entry */ // _CHECK_PERIODIC { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -7793,6 +7657,8 @@ this_instr[1].counter = initial_jump_backoff_counter(); stack_pointer = _PyFrame_GetStackPointer(frame); assert(tstate->current_executor == NULL); + assert(executor != tstate->interp->cold_executor); + tstate->jit_exit = NULL; GOTO_TIER_TWO(executor); } } @@ -7829,15 +7695,11 @@ /* Skip 1 cache entry */ // _CHECK_PERIODIC { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -10389,7 +10251,13 @@ } // _MAYBE_INSTRUMENT { - if (tstate->tracing == 0) { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { @@ -10415,15 +10283,11 @@ // _CHECK_PERIODIC_IF_NOT_YIELD_FROM { if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } } diff --git a/Python/hamt.c b/Python/hamt.c index 906149cc6cdbdc..e372b1a1b4c18b 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -256,9 +256,9 @@ Debug ===== The HAMT datatype is accessible for testing purposes under the -`_testcapi` module: +`_testinternalcapi` module: - >>> from _testcapi import hamt + >>> from _testinternalcapi import hamt >>> h = hamt() >>> h2 = h.set('a', 2) >>> h3 = h2.set('b', 3) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index f9913cff402ed1..b4b2bc5dc69f9d 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1040,6 +1040,8 @@ set_version_raw(uintptr_t *ptr, uint32_t version) static void set_global_version(PyThreadState *tstate, uint32_t version) { + ASSERT_WORLD_STOPPED(); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); PyInterpreterState *interp = tstate->interp; set_version_raw(&interp->ceval.instrumentation_version, version); @@ -1939,28 +1941,26 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) static int -instrument_all_executing_code_objects(PyInterpreterState *interp) { +instrument_all_executing_code_objects(PyInterpreterState *interp) +{ ASSERT_WORLD_STOPPED(); - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); - while (ts) { + int err = 0; + _Py_FOR_EACH_TSTATE_BEGIN(interp, ts) { _PyInterpreterFrame *frame = ts->current_frame; while (frame) { if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { - if (instrument_lock_held(_PyFrame_GetCode(frame), interp)) { - return -1; + err = instrument_lock_held(_PyFrame_GetCode(frame), interp); + if (err) { + goto done; } } frame = frame->previous; } - HEAD_LOCK(runtime); - ts = PyThreadState_Next(ts); - HEAD_UNLOCK(runtime); } - return 0; +done: + _Py_FOR_EACH_TSTATE_END(interp); + return err; } static void @@ -2006,6 +2006,7 @@ check_tool(PyInterpreterState *interp, int tool_id) int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { + ASSERT_WORLD_STOPPED(); assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyThreadState *tstate = _PyThreadState_GET(); PyInterpreterState *interp = tstate->interp; @@ -2014,33 +2015,28 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) return -1; } - int res; - _PyEval_StopTheWorld(interp); uint32_t existing_events = get_events(&interp->monitors, tool_id); if (existing_events == events) { - res = 0; - goto done; + return 0; } set_events(&interp->monitors, tool_id, events); uint32_t new_version = global_version(interp) + MONITORING_VERSION_INCREMENT; if (new_version == 0) { PyErr_Format(PyExc_OverflowError, "events set too many times"); - res = -1; - goto done; + return -1; } set_global_version(tstate, new_version); #ifdef _Py_TIER2 _Py_Executors_InvalidateAll(interp, 1); #endif - res = instrument_all_executing_code_objects(interp); -done: - _PyEval_StartTheWorld(interp); - return res; + return instrument_all_executing_code_objects(interp); } int _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet events) { + ASSERT_WORLD_STOPPED(); + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_GET(); assert(events < (1 << _PY_MONITORING_LOCAL_EVENTS)); @@ -2052,11 +2048,8 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent return -1; } - int res; - _PyEval_StopTheWorld(interp); if (allocate_instrumentation_data(code)) { - res = -1; - goto done; + return -1; } code->_co_monitoring->tool_versions[tool_id] = interp->monitoring_tool_versions[tool_id]; @@ -2064,16 +2057,11 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent _Py_LocalMonitors *local = &code->_co_monitoring->local_monitors; uint32_t existing_events = get_local_events(local, tool_id); if (existing_events == events) { - res = 0; - goto done; + return 0; } set_local_events(local, tool_id, events); - res = force_instrument_lock_held(code, interp); - -done: - _PyEval_StartTheWorld(interp); - return res; + return force_instrument_lock_held(code, interp); } int @@ -2105,11 +2093,12 @@ int _PyMonitoring_ClearToolId(int tool_id) } } + _PyEval_StopTheWorld(interp); if (_PyMonitoring_SetEvents(tool_id, 0) < 0) { + _PyEval_StartTheWorld(interp); return -1; } - _PyEval_StopTheWorld(interp); uint32_t version = global_version(interp) + MONITORING_VERSION_INCREMENT; if (version == 0) { PyErr_Format(PyExc_OverflowError, "events set too many times"); @@ -2346,7 +2335,11 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) event_set &= ~(1 << PY_MONITORING_EVENT_BRANCH); event_set |= (1 << PY_MONITORING_EVENT_BRANCH_RIGHT) | (1 << PY_MONITORING_EVENT_BRANCH_LEFT); } - if (_PyMonitoring_SetEvents(tool_id, event_set)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + int err = _PyMonitoring_SetEvents(tool_id, event_set); + _PyEval_StartTheWorld(interp); + if (err) { return NULL; } Py_RETURN_NONE; @@ -2427,7 +2420,11 @@ monitoring_set_local_events_impl(PyObject *module, int tool_id, return NULL; } - if (_PyMonitoring_SetLocalEvents((PyCodeObject*)code, tool_id, event_set)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + int err = _PyMonitoring_SetLocalEvents((PyCodeObject*)code, tool_id, event_set); + _PyEval_StartTheWorld(interp); + if (err) { return NULL; } Py_RETURN_NONE; diff --git a/Python/jit.c b/Python/jit.c index e4927d9ff04071..bd6a5e17a4164f 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -546,7 +546,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz group->emit(code, data, executor, NULL, &state); code += group->code_size; data += group->data_size; - assert(trace[0].opcode == _START_EXECUTOR); + assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT); for (size_t i = 0; i < length; i++) { const _PyUOpInstruction *instruction = &trace[i]; group = &stencil_groups[instruction->opcode]; diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index dbd19d7755c237..594d5c5ead5021 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -20,13 +20,6 @@ typedef struct _PyLegacyEventHandler { #define _PyLegacyEventHandler_CAST(op) ((_PyLegacyEventHandler *)(op)) -#ifdef Py_GIL_DISABLED -#define LOCK_SETUP() PyMutex_Lock(&_PyRuntime.ceval.sys_trace_profile_mutex); -#define UNLOCK_SETUP() PyMutex_Unlock(&_PyRuntime.ceval.sys_trace_profile_mutex); -#else -#define LOCK_SETUP() -#define UNLOCK_SETUP() -#endif /* The Py_tracefunc function expects the following arguments: * obj: the trace object (PyObject *) * frame: the current frame (PyFrameObject *) @@ -133,16 +126,10 @@ sys_profile_call_or_return( Py_RETURN_NONE; } -int -_PyEval_SetOpcodeTrace( - PyFrameObject *frame, - bool enable -) { - assert(frame != NULL); - - PyCodeObject *code = _PyFrame_GetCode(frame->f_frame); +static int +set_opcode_trace_world_stopped(PyCodeObject *code, bool enable) +{ _PyMonitoringEventSet events = 0; - if (_PyMonitoring_GetLocalEvents(code, PY_MONITORING_SYS_TRACE_ID, &events) < 0) { return -1; } @@ -161,6 +148,32 @@ _PyEval_SetOpcodeTrace( return _PyMonitoring_SetLocalEvents(code, PY_MONITORING_SYS_TRACE_ID, events); } +int +_PyEval_SetOpcodeTrace(PyFrameObject *frame, bool enable) +{ + assert(frame != NULL); + + PyCodeObject *code = _PyFrame_GetCode(frame->f_frame); + +#ifdef Py_GIL_DISABLED + // First check if a change is necessary outside of the stop-the-world pause + _PyMonitoringEventSet events = 0; + if (_PyMonitoring_GetLocalEvents(code, PY_MONITORING_SYS_TRACE_ID, &events) < 0) { + return -1; + } + int is_enabled = (events & (1 << PY_MONITORING_EVENT_INSTRUCTION)) != 0; + if (is_enabled == enable) { + return 0; // No change needed + } +#endif + + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + int res = set_opcode_trace_world_stopped(code, enable); + _PyEval_StartTheWorld(interp); + return res; +} + static PyObject * call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) { @@ -438,59 +451,74 @@ is_tstate_valid(PyThreadState *tstate) } #endif -static Py_ssize_t -setup_profile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg, PyObject **old_profileobj) +static int +setup_profile_callbacks(void *Py_UNUSED(arg)) { - *old_profileobj = NULL; /* Setup PEP 669 monitoring callbacks and events. */ - if (!tstate->interp->sys_profile_initialized) { - tstate->interp->sys_profile_initialized = true; - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_start, PyTrace_CALL, - PY_MONITORING_EVENT_PY_START, - PY_MONITORING_EVENT_PY_RESUME)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_throw, PyTrace_CALL, - PY_MONITORING_EVENT_PY_THROW, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_return, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_RETURN, - PY_MONITORING_EVENT_PY_YIELD)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_unwind, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_UNWIND, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_call_or_return, PyTrace_C_CALL, - PY_MONITORING_EVENT_CALL, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_call_or_return, PyTrace_C_RETURN, - PY_MONITORING_EVENT_C_RETURN, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_call_or_return, PyTrace_C_EXCEPTION, - PY_MONITORING_EVENT_C_RAISE, -1)) { - return -1; - } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_start, PyTrace_CALL, + PY_MONITORING_EVENT_PY_START, + PY_MONITORING_EVENT_PY_RESUME)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_throw, PyTrace_CALL, + PY_MONITORING_EVENT_PY_THROW, -1)) { + return -1; } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_return, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_RETURN, + PY_MONITORING_EVENT_PY_YIELD)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_unwind, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_UNWIND, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_call_or_return, PyTrace_C_CALL, + PY_MONITORING_EVENT_CALL, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_call_or_return, PyTrace_C_RETURN, + PY_MONITORING_EVENT_C_RETURN, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_call_or_return, PyTrace_C_EXCEPTION, + PY_MONITORING_EVENT_C_RAISE, -1)) { + return -1; + } + return 0; +} +static PyObject * +swap_profile_func_arg(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ int delta = (func != NULL) - (tstate->c_profilefunc != NULL); tstate->c_profilefunc = func; - *old_profileobj = tstate->c_profileobj; + PyObject *old_profileobj = tstate->c_profileobj; tstate->c_profileobj = Py_XNewRef(arg); tstate->interp->sys_profiling_threads += delta; assert(tstate->interp->sys_profiling_threads >= 0); - return tstate->interp->sys_profiling_threads; + return old_profileobj; +} + +static int +set_monitoring_profile_events(PyInterpreterState *interp) +{ + uint32_t events = 0; + if (interp->sys_profiling_threads) { + events = + (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | + (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | + (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | + (1 << PY_MONITORING_EVENT_PY_THROW); + } + return _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); } int @@ -507,87 +535,155 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } - // needs to be decref'd outside of the lock - PyObject *old_profileobj; - LOCK_SETUP(); - Py_ssize_t profiling_threads = setup_profile(tstate, func, arg, &old_profileobj); - UNLOCK_SETUP(); - Py_XDECREF(old_profileobj); + PyInterpreterState *interp = tstate->interp; + if (_PyOnceFlag_CallOnce(&interp->sys_profile_once_flag, + setup_profile_callbacks, NULL) < 0) { + return -1; + } - uint32_t events = 0; - if (profiling_threads) { - events = - (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | - (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | - (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | - (1 << PY_MONITORING_EVENT_PY_THROW); + _PyEval_StopTheWorld(interp); + PyObject *old_profileobj = swap_profile_func_arg(tstate, func, arg); + int ret = set_monitoring_profile_events(interp); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_profileobj); // needs to be decref'd outside of stop-the-world + return ret; +} + +int +_PyEval_SetProfileAllThreads(PyInterpreterState *interp, Py_tracefunc func, PyObject *arg) +{ + PyThreadState *current_tstate = _PyThreadState_GET(); + assert(is_tstate_valid(current_tstate)); + assert(current_tstate->interp == interp); + + if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { + return -1; } - return _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); + + if (_PyOnceFlag_CallOnce(&interp->sys_profile_once_flag, + setup_profile_callbacks, NULL) < 0) { + return -1; + } + + PyObject *old_profileobjs = NULL; + _PyEval_StopTheWorld(interp); + HEAD_LOCK(&_PyRuntime); + Py_ssize_t num_thread_states = 0; + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, p) { + num_thread_states++; + } + old_profileobjs = PyTuple_New(num_thread_states); + if (old_profileobjs == NULL) { + HEAD_UNLOCK(&_PyRuntime); + _PyEval_StartTheWorld(interp); + return -1; + } + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + PyObject *old = swap_profile_func_arg(tstate, func, arg); + PyTuple_SET_ITEM(old_profileobjs, --num_thread_states, old); + } + HEAD_UNLOCK(&_PyRuntime); + int ret = set_monitoring_profile_events(interp); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_profileobjs); // needs to be decref'd outside of stop-the-world + return ret; } -static Py_ssize_t -setup_tracing(PyThreadState *tstate, Py_tracefunc func, PyObject *arg, PyObject **old_traceobj) +static int +setup_trace_callbacks(void *Py_UNUSED(arg)) { - *old_traceobj = NULL; /* Setup PEP 669 monitoring callbacks and events. */ - if (!tstate->interp->sys_trace_initialized) { - tstate->interp->sys_trace_initialized = true; - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_start, PyTrace_CALL, - PY_MONITORING_EVENT_PY_START, - PY_MONITORING_EVENT_PY_RESUME)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_throw, PyTrace_CALL, - PY_MONITORING_EVENT_PY_THROW, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_return, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_RETURN, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_yield, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_YIELD, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_exception_func, PyTrace_EXCEPTION, - PY_MONITORING_EVENT_RAISE, - PY_MONITORING_EVENT_STOP_ITERATION)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_line_func, PyTrace_LINE, - PY_MONITORING_EVENT_LINE, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_unwind, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_UNWIND, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_jump_func, PyTrace_LINE, - PY_MONITORING_EVENT_JUMP, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_instruction_func, PyTrace_OPCODE, - PY_MONITORING_EVENT_INSTRUCTION, -1)) { - return -1; - } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_start, PyTrace_CALL, + PY_MONITORING_EVENT_PY_START, + PY_MONITORING_EVENT_PY_RESUME)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_throw, PyTrace_CALL, + PY_MONITORING_EVENT_PY_THROW, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_return, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_RETURN, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_yield, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_YIELD, -1)) { + return -1; } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_exception_func, PyTrace_EXCEPTION, + PY_MONITORING_EVENT_RAISE, + PY_MONITORING_EVENT_STOP_ITERATION)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_line_func, PyTrace_LINE, + PY_MONITORING_EVENT_LINE, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_unwind, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_UNWIND, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_jump_func, PyTrace_LINE, + PY_MONITORING_EVENT_JUMP, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_instruction_func, PyTrace_OPCODE, + PY_MONITORING_EVENT_INSTRUCTION, -1)) { + return -1; + } + return 0; +} +static PyObject * +swap_trace_func_arg(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ int delta = (func != NULL) - (tstate->c_tracefunc != NULL); tstate->c_tracefunc = func; - *old_traceobj = tstate->c_traceobj; + PyObject *old_traceobj = tstate->c_traceobj; tstate->c_traceobj = Py_XNewRef(arg); tstate->interp->sys_tracing_threads += delta; assert(tstate->interp->sys_tracing_threads >= 0); - return tstate->interp->sys_tracing_threads; + return old_traceobj; +} + +static int +set_monitoring_trace_events(PyInterpreterState *interp) +{ + uint32_t events = 0; + if (interp->sys_tracing_threads) { + events = + (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | + (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | + (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | + (1 << PY_MONITORING_EVENT_JUMP) | + (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW) | + (1 << PY_MONITORING_EVENT_STOP_ITERATION); + } + return _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, events); +} + +// Enable opcode tracing for the thread's current frame if needed. +static int +maybe_set_opcode_trace(PyThreadState *tstate) +{ + _PyInterpreterFrame *iframe = tstate->current_frame; + if (iframe == NULL) { + return 0; + } + PyFrameObject *frame = iframe->frame_obj; + if (frame == NULL || !frame->f_trace_opcodes) { + return 0; + } + return set_opcode_trace_world_stopped(_PyFrame_GetCode(iframe), true); } int @@ -603,35 +699,76 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { return -1; } - // needs to be decref'd outside of the lock - PyObject *old_traceobj; - LOCK_SETUP(); - assert(tstate->interp->sys_tracing_threads >= 0); - Py_ssize_t tracing_threads = setup_tracing(tstate, func, arg, &old_traceobj); - UNLOCK_SETUP(); - Py_XDECREF(old_traceobj); - if (tracing_threads < 0) { + + PyInterpreterState *interp = tstate->interp; + if (_PyOnceFlag_CallOnce(&interp->sys_trace_once_flag, + setup_trace_callbacks, NULL) < 0) { return -1; } - uint32_t events = 0; - if (tracing_threads) { - events = - (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | - (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | - (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | - (1 << PY_MONITORING_EVENT_JUMP) | - (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW) | - (1 << PY_MONITORING_EVENT_STOP_ITERATION); + int err = 0; + _PyEval_StopTheWorld(interp); + PyObject *old_traceobj = swap_trace_func_arg(tstate, func, arg); + err = set_monitoring_trace_events(interp); + if (err != 0) { + goto done; + } + if (interp->sys_tracing_threads) { + err = maybe_set_opcode_trace(tstate); + } +done: + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_traceobj); // needs to be decref'd outside stop-the-world + return err; +} - PyFrameObject* frame = PyEval_GetFrame(); - if (frame && frame->f_trace_opcodes) { - int ret = _PyEval_SetOpcodeTrace(frame, true); - if (ret != 0) { - return ret; +int +_PyEval_SetTraceAllThreads(PyInterpreterState *interp, Py_tracefunc func, PyObject *arg) +{ + PyThreadState *current_tstate = _PyThreadState_GET(); + assert(is_tstate_valid(current_tstate)); + assert(current_tstate->interp == interp); + + if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { + return -1; + } + + if (_PyOnceFlag_CallOnce(&interp->sys_trace_once_flag, + setup_trace_callbacks, NULL) < 0) { + return -1; + } + + PyObject *old_trace_objs = NULL; + _PyEval_StopTheWorld(interp); + HEAD_LOCK(&_PyRuntime); + Py_ssize_t num_thread_states = 0; + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, p) { + num_thread_states++; + } + old_trace_objs = PyTuple_New(num_thread_states); + if (old_trace_objs == NULL) { + HEAD_UNLOCK(&_PyRuntime); + _PyEval_StartTheWorld(interp); + return -1; + } + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + PyObject *old = swap_trace_func_arg(tstate, func, arg); + PyTuple_SET_ITEM(old_trace_objs, --num_thread_states, old); + } + if (interp->sys_tracing_threads) { + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + int err = maybe_set_opcode_trace(tstate); + if (err != 0) { + HEAD_UNLOCK(&_PyRuntime); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_trace_objs); + return -1; } } } - - return _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, events); + HEAD_UNLOCK(&_PyRuntime); + int err = set_monitoring_trace_events(interp); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_trace_objs); // needs to be decref'd outside of stop-the-world + return err; } diff --git a/Python/optimizer.c b/Python/optimizer.c index 8d01d605ef4a2a..f06df9644a1614 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -205,8 +205,8 @@ static int executor_clear(PyObject *executor); static void unlink_executor(_PyExecutorObject *executor); -static void -free_executor(_PyExecutorObject *self) +void +_PyExecutor_Free(_PyExecutorObject *self) { #ifdef _Py_JIT _PyJIT_Free(self); @@ -242,7 +242,7 @@ _Py_ClearExecutorDeletionList(PyInterpreterState *interp) } else { *prev_to_next_ptr = exec->vm_data.links.next; - free_executor(exec); + _PyExecutor_Free(exec); } exec = *prev_to_next_ptr; } @@ -432,6 +432,7 @@ _PyUOp_Replacements[MAX_UOP_ID + 1] = { [_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE, [_FOR_ITER] = _FOR_ITER_TIER_TWO, [_ITER_NEXT_LIST] = _ITER_NEXT_LIST_TIER_TWO, + [_CHECK_PERIODIC_AT_END] = _TIER2_RESUME_CHECK, }; static const uint8_t @@ -776,9 +777,12 @@ translate_bytecode_to_trace( case OPARG_REPLACED: uop = _PyUOp_Replacements[uop]; assert(uop != 0); + uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255); + if (uop == _TIER2_RESUME_CHECK) { + target = next_inst; + } #ifdef Py_DEBUG - { - uint32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + (oparg > 255); + else { uint32_t jump_target = next_inst + oparg; assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR); assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER); @@ -1043,9 +1047,15 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) _PyUOpInstruction *inst = &buffer[i]; int opcode = inst->opcode; int32_t target = (int32_t)uop_get_target(inst); - if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { - uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ? - _EXIT_TRACE : _DEOPT; + uint16_t exit_flags = _PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG | HAS_PERIODIC_FLAG); + if (exit_flags) { + uint16_t exit_op = _EXIT_TRACE; + if (exit_flags & HAS_DEOPT_FLAG) { + exit_op = _DEOPT; + } + else if (exit_flags & HAS_PERIODIC_FLAG) { + exit_op = _HANDLE_PENDING_AND_DEOPT; + } int32_t jump_target = target; if (is_for_iter_test[opcode]) { /* Target the POP_TOP immediately after the END_FOR, @@ -1129,7 +1139,7 @@ sanity_check(_PyExecutorObject *executor) } bool ended = false; uint32_t i = 0; - CHECK(executor->trace[0].opcode == _START_EXECUTOR); + CHECK(executor->trace[0].opcode == _START_EXECUTOR || executor->trace[0].opcode == _COLD_EXIT); for (; i < executor->code_size; i++) { const _PyUOpInstruction *inst = &executor->trace[i]; uint16_t opcode = inst->opcode; @@ -1159,6 +1169,7 @@ sanity_check(_PyExecutorObject *executor) uint16_t opcode = inst->opcode; CHECK( opcode == _DEOPT || + opcode == _HANDLE_PENDING_AND_DEOPT || opcode == _EXIT_TRACE || opcode == _ERROR_POP_N); } @@ -1182,9 +1193,11 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil } /* Initialize exits */ + _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); for (int i = 0; i < exit_count; i++) { - executor->exits[i].executor = NULL; + executor->exits[i].index = i; executor->exits[i].temperature = initial_temperature_backoff_counter(); + executor->exits[i].executor = cold; } int next_exit = exit_count-1; _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; @@ -1462,6 +1475,46 @@ _Py_ExecutorInit(_PyExecutorObject *executor, const _PyBloomFilter *dependency_s link_executor(executor); } +_PyExecutorObject * +_PyExecutor_GetColdExecutor(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->cold_executor != NULL) { + return interp->cold_executor; + } + _PyExecutorObject *cold = allocate_executor(0, 1); + if (cold == NULL) { + Py_FatalError("Cannot allocate core JIT code"); + } + ((_PyUOpInstruction *)cold->trace)->opcode = _COLD_EXIT; +#ifdef _Py_JIT + cold->jit_code = NULL; + cold->jit_side_entry = NULL; + cold->jit_size = 0; + // This is initialized to true so we can prevent the executor + // from being immediately detected as cold and invalidated. + cold->vm_data.warm = true; + if (_PyJIT_Compile(cold, cold->trace, 1)) { + Py_DECREF(cold); + Py_FatalError("Cannot allocate core JIT code"); + } +#endif + _Py_SetImmortal((PyObject *)cold); + interp->cold_executor = cold; + return cold; +} + +void +_PyExecutor_ClearExit(_PyExitData *exit) +{ + if (exit == NULL) { + return; + } + _PyExecutorObject *old = exit->executor; + exit->executor = _PyExecutor_GetColdExecutor(); + Py_DECREF(old); +} + /* Detaches the executor from the code object (if any) that * holds a reference to it */ void @@ -1492,14 +1545,18 @@ executor_clear(PyObject *op) assert(executor->vm_data.valid == 1); unlink_executor(executor); executor->vm_data.valid = 0; + /* It is possible for an executor to form a reference * cycle with itself, so decref'ing a side exit could * free the executor unless we hold a strong reference to it */ + _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); Py_INCREF(executor); for (uint32_t i = 0; i < executor->exit_count; i++) { executor->exits[i].temperature = initial_unreachable_backoff_counter(); - Py_CLEAR(executor->exits[i].executor); + _PyExecutorObject *e = executor->exits[i].executor; + executor->exits[i].executor = cold; + Py_DECREF(e); } _Py_ExecutorDetach(executor); Py_DECREF(executor); @@ -1741,4 +1798,11 @@ _PyDumpExecutors(FILE *out) return -1; } +void +_PyExecutor_Free(struct _PyExecutorObject *self) +{ + /* This should never be called */ + Py_UNREACHABLE(); +} + #endif /* _Py_TIER2 */ diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 99206f01618d79..2477ede3e68017 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -11,6 +11,8 @@ break; } + /* _CHECK_PERIODIC_AT_END is not a viable micro-op for tier 2 */ + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { break; } @@ -3198,6 +3200,10 @@ break; } + case _HANDLE_PENDING_AND_DEOPT: { + break; + } + case _ERROR_POP_N: { break; } @@ -3206,3 +3212,7 @@ break; } + case _COLD_EXIT: { + break; + } + diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index a2da3c7d56df50..987e8d2a11a659 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -202,6 +202,7 @@ enum perf_trampoline_type { #define perf_map_file _PyRuntime.ceval.perf.map_file #define persist_after_fork _PyRuntime.ceval.perf.persist_after_fork #define perf_trampoline_type _PyRuntime.ceval.perf.perf_trampoline_type +#define prev_eval_frame _PyRuntime.ceval.perf.prev_eval_frame static void perf_map_write_entry(void *state, const void *code_addr, @@ -407,9 +408,12 @@ py_trampoline_evaluator(PyThreadState *ts, _PyInterpreterFrame *frame, f = new_trampoline; } assert(f != NULL); - return f(ts, frame, throw, _PyEval_EvalFrameDefault); + return f(ts, frame, throw, prev_eval_frame != NULL ? prev_eval_frame : _PyEval_EvalFrameDefault); default_eval: // Something failed, fall back to the default evaluator. + if (prev_eval_frame) { + return prev_eval_frame(ts, frame, throw); + } return _PyEval_EvalFrameDefault(ts, frame, throw); } #endif // PY_HAVE_PERF_TRAMPOLINE @@ -481,18 +485,12 @@ _PyPerfTrampoline_Init(int activate) { #ifdef PY_HAVE_PERF_TRAMPOLINE PyThreadState *tstate = _PyThreadState_GET(); - if (tstate->interp->eval_frame && - tstate->interp->eval_frame != py_trampoline_evaluator) { - PyErr_SetString(PyExc_RuntimeError, - "Trampoline cannot be initialized as a custom eval " - "frame is already present"); - return -1; - } if (!activate) { - _PyInterpreterState_SetEvalFrameFunc(tstate->interp, NULL); + _PyInterpreterState_SetEvalFrameFunc(tstate->interp, prev_eval_frame); perf_status = PERF_STATUS_NO_INIT; } - else { + else if (tstate->interp->eval_frame != py_trampoline_evaluator) { + prev_eval_frame = _PyInterpreterState_GetEvalFrameFunc(tstate->interp); _PyInterpreterState_SetEvalFrameFunc(tstate->interp, py_trampoline_evaluator); extra_code_index = _PyEval_RequestCodeExtraIndex(NULL); if (extra_code_index == -1) { diff --git a/Python/pystate.c b/Python/pystate.c index 04ca6edb4aaa0e..a2914b3718eea2 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -324,7 +324,6 @@ _Py_COMP_DIAG_POP &(runtime)->unicode_state.ids.mutex, \ &(runtime)->imports.extensions.mutex, \ &(runtime)->ceval.pending_mainthread.mutex, \ - &(runtime)->ceval.sys_trace_profile_mutex, \ &(runtime)->atexit.mutex, \ &(runtime)->audit_hooks.mutex, \ &(runtime)->allocators.mutex, \ @@ -565,8 +564,6 @@ init_interpreter(PyInterpreterState *interp, } interp->monitoring_tool_versions[t] = 0; } - interp->sys_profile_initialized = false; - interp->sys_trace_initialized = false; interp->_code_object_generation = 0; interp->jit = false; interp->executor_list_head = NULL; @@ -773,8 +770,6 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->monitoring_callables[t][e]); } } - interp->sys_profile_initialized = false; - interp->sys_trace_initialized = false; for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { Py_CLEAR(interp->monitoring_tool_names[t]); } @@ -815,7 +810,13 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) /* Last garbage collection on this interpreter */ _PyGC_CollectNoFail(tstate); _PyGC_Fini(interp); - + struct _PyExecutorObject *cold = interp->cold_executor; + if (cold != NULL) { + interp->cold_executor = NULL; + assert(cold->vm_data.valid); + assert(cold->vm_data.warm); + _PyExecutor_Free(cold); + } /* We don't clear sysdict and builtins until the end of this function. Because clearing other attributes can execute arbitrary Python code which requires sysdict and builtins. */ @@ -1469,6 +1470,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->datastack_limit = NULL; tstate->what_event = -1; tstate->current_executor = NULL; + tstate->jit_exit = NULL; tstate->dict_global_version = 0; _tstate->c_stack_soft_limit = UINTPTR_MAX; @@ -1682,23 +1684,15 @@ PyThreadState_Clear(PyThreadState *tstate) "PyThreadState_Clear: warning: thread still has a generator\n"); } -#ifdef Py_GIL_DISABLED - PyMutex_Lock(&_PyRuntime.ceval.sys_trace_profile_mutex); -#endif - if (tstate->c_profilefunc != NULL) { - tstate->interp->sys_profiling_threads--; + FT_ATOMIC_ADD_SSIZE(tstate->interp->sys_profiling_threads, -1); tstate->c_profilefunc = NULL; } if (tstate->c_tracefunc != NULL) { - tstate->interp->sys_tracing_threads--; + FT_ATOMIC_ADD_SSIZE(tstate->interp->sys_tracing_threads, -1); tstate->c_tracefunc = NULL; } -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&_PyRuntime.ceval.sys_trace_profile_mutex); -#endif - Py_CLEAR(tstate->c_profileobj); Py_CLEAR(tstate->c_traceobj); diff --git a/Python/remote_debug.h b/Python/remote_debug.h index 5324a7aaa6f5e5..d920d9e5b5ff2c 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -50,9 +50,11 @@ extern "C" { # include # include # include +# include # include # include # include +# include # include # include # include @@ -1053,18 +1055,38 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address (mach_vm_size_t*)&result); if (kr != KERN_SUCCESS) { - switch (kr) { + switch (err_get_code(kr)) { case KERN_PROTECTION_FAILURE: PyErr_Format(PyExc_PermissionError, "Memory protection failure reading from PID %d at address " "0x%lx (size %zu): insufficient permissions", handle->pid, remote_address, len); break; - case KERN_INVALID_ARGUMENT: - PyErr_Format(PyExc_ValueError, - "Invalid argument to mach_vm_read_overwrite for PID %d at " - "address 0x%lx (size %zu)", - handle->pid, remote_address, len); + case KERN_INVALID_ARGUMENT: { + // Perform a task_info check to see if the invalid argument is due + // to the process being terminated + task_basic_info_data_t task_basic_info; + mach_msg_type_number_t task_info_count = TASK_BASIC_INFO_COUNT; + kern_return_t task_valid_check = task_info(handle->task, TASK_BASIC_INFO, + (task_info_t)&task_basic_info, + &task_info_count); + if (task_valid_check == KERN_INVALID_ARGUMENT) { + PyErr_Format(PyExc_ProcessLookupError, + "Process %d is no longer accessible (process terminated)", + handle->pid); + } else { + PyErr_Format(PyExc_ValueError, + "Invalid argument to mach_vm_read_overwrite for PID %d at " + "address 0x%lx (size %zu) - check memory permissions", + handle->pid, remote_address, len); + } + break; + } + case KERN_NO_SPACE: + case KERN_MEMORY_ERROR: + PyErr_Format(PyExc_ProcessLookupError, + "Process %d memory space no longer available (process terminated)", + handle->pid); break; default: PyErr_Format(PyExc_RuntimeError, diff --git a/Python/specialize.c b/Python/specialize.c index fe8d04cf3442f1..38df5741f32520 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -935,8 +935,7 @@ analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr, un PyObject *getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)); has_getattr = getattr != NULL; if (has_custom_getattribute) { - if (getattro_slot == _Py_slot_tp_getattro && - !has_getattr && + if (!has_getattr && Py_IS_TYPE(getattribute, &PyFunction_Type)) { *descr = getattribute; *tp_version = ga_version; @@ -1259,12 +1258,6 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* return -1; case GETATTRIBUTE_IS_PYTHON_FUNCTION: { - #ifndef Py_GIL_DISABLED - // In free-threaded builds it's possible for tp_getattro to change - // after the call to analyze_descriptor. That is fine: the version - // guard will fail. - assert(type->tp_getattro == _Py_slot_tp_getattro); - #endif assert(Py_IS_TYPE(descr, &PyFunction_Type)); _PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1); if (!function_check_args(descr, 2, LOAD_ATTR)) { diff --git a/Python/sysmodule.c b/Python/sysmodule.c index ae6cf306735939..19912b4a4c6198 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1183,9 +1183,10 @@ sys__settraceallthreads(PyObject *module, PyObject *arg) argument = arg; } - - PyEval_SetTraceAllThreads(func, argument); - + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyEval_SetTraceAllThreads(interp, func, argument) < 0) { + return NULL; + } Py_RETURN_NONE; } @@ -1263,8 +1264,10 @@ sys__setprofileallthreads(PyObject *module, PyObject *arg) argument = arg; } - PyEval_SetProfileAllThreads(func, argument); - + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyEval_SetProfileAllThreads(interp, func, argument) < 0) { + return NULL; + } Py_RETURN_NONE; } @@ -2641,6 +2644,47 @@ sys__baserepl_impl(PyObject *module) Py_RETURN_NONE; } +/*[clinic input] +sys._clear_type_descriptors + + type: object(subclass_of='&PyType_Type') + / + +Private function for clearing certain descriptors from a type's dictionary. + +See gh-135228 for context. +[clinic start generated code]*/ + +static PyObject * +sys__clear_type_descriptors_impl(PyObject *module, PyObject *type) +/*[clinic end generated code: output=5ad17851b762b6d9 input=dc536c97fde07251]*/ +{ + PyTypeObject *typeobj = (PyTypeObject *)type; + if (_PyType_HasFeature(typeobj, Py_TPFLAGS_IMMUTABLETYPE)) { + PyErr_SetString(PyExc_TypeError, "argument is immutable"); + return NULL; + } + PyObject *dict = _PyType_GetDict(typeobj); + PyObject *dunder_dict = NULL; + if (PyDict_Pop(dict, &_Py_ID(__dict__), &dunder_dict) < 0) { + return NULL; + } + PyObject *dunder_weakref = NULL; + if (PyDict_Pop(dict, &_Py_ID(__weakref__), &dunder_weakref) < 0) { + PyType_Modified(typeobj); + Py_XDECREF(dunder_dict); + return NULL; + } + PyType_Modified(typeobj); + // We try to hold onto a reference to these until after we call + // PyType_Modified(), in case their deallocation triggers somer user code + // that tries to do something to the type. + Py_XDECREF(dunder_dict); + Py_XDECREF(dunder_weakref); + Py_RETURN_NONE; +} + + /*[clinic input] sys._is_gil_enabled -> bool @@ -2837,6 +2881,7 @@ static PyMethodDef sys_methods[] = { SYS__STATS_DUMP_METHODDEF #endif SYS__GET_CPU_COUNT_CONFIG_METHODDEF + SYS__CLEAR_TYPE_DESCRIPTORS_METHODDEF SYS__IS_GIL_ENABLED_METHODDEF SYS__DUMP_TRACELETS_METHODDEF {NULL, NULL} // sentinel diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index 9815bcfe27d995..668db8df0bd181 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -17,9 +17,11 @@ See --help for more information """ + +from __future__ import annotations + import _imp import argparse -import collections import enum import logging import os @@ -29,13 +31,16 @@ import sysconfig import warnings from collections.abc import Iterable -from importlib._bootstrap import _load as bootstrap_load +from importlib._bootstrap import ( # type: ignore[attr-defined] + _load as bootstrap_load, +) from importlib.machinery import ( BuiltinImporter, ExtensionFileLoader, ModuleSpec, ) from importlib.util import spec_from_file_location, spec_from_loader +from typing import NamedTuple SRC_DIR = pathlib.Path(__file__).parent.parent.parent @@ -112,6 +117,7 @@ ) +@enum.unique class ModuleState(enum.Enum): # Makefile state "yes" BUILTIN = "builtin" @@ -123,11 +129,13 @@ class ModuleState(enum.Enum): # disabled by Setup / makesetup rule DISABLED_SETUP = "disabled_setup" - def __bool__(self): + def __bool__(self) -> bool: return self.value in {"builtin", "shared"} -ModuleInfo = collections.namedtuple("ModuleInfo", "name state") +class ModuleInfo(NamedTuple): + name: str + state: ModuleState class ModuleChecker: @@ -135,9 +143,9 @@ class ModuleChecker: setup_files = ( # see end of configure.ac - "Modules/Setup.local", - "Modules/Setup.stdlib", - "Modules/Setup.bootstrap", + pathlib.Path("Modules/Setup.local"), + pathlib.Path("Modules/Setup.stdlib"), + pathlib.Path("Modules/Setup.bootstrap"), SRC_DIR / "Modules/Setup", ) @@ -149,15 +157,15 @@ def __init__(self, cross_compiling: bool = False, strict: bool = False): self.builddir = self.get_builddir() self.modules = self.get_modules() - self.builtin_ok = [] - self.shared_ok = [] - self.failed_on_import = [] - self.missing = [] - self.disabled_configure = [] - self.disabled_setup = [] - self.notavailable = [] + self.builtin_ok: list[ModuleInfo] = [] + self.shared_ok: list[ModuleInfo] = [] + self.failed_on_import: list[ModuleInfo] = [] + self.missing: list[ModuleInfo] = [] + self.disabled_configure: list[ModuleInfo] = [] + self.disabled_setup: list[ModuleInfo] = [] + self.notavailable: list[ModuleInfo] = [] - def check(self): + def check(self) -> None: if not hasattr(_imp, 'create_dynamic'): logger.warning( ('Dynamic extensions not supported ' @@ -189,10 +197,10 @@ def check(self): assert modinfo.state == ModuleState.SHARED self.shared_ok.append(modinfo) - def summary(self, *, verbose: bool = False): + def summary(self, *, verbose: bool = False) -> None: longest = max([len(e.name) for e in self.modules], default=0) - def print_three_column(modinfos: list[ModuleInfo]): + def print_three_column(modinfos: list[ModuleInfo]) -> None: names = [modinfo.name for modinfo in modinfos] names.sort(key=str.lower) # guarantee zip() doesn't drop anything @@ -262,12 +270,12 @@ def print_three_column(modinfos: list[ModuleInfo]): f"{len(self.failed_on_import)} failed on import)" ) - def check_strict_build(self): + def check_strict_build(self) -> None: """Fail if modules are missing and it's a strict build""" if self.strict_extensions_build and (self.failed_on_import or self.missing): raise RuntimeError("Failed to build some stdlib modules") - def list_module_names(self, *, all: bool = False) -> set: + def list_module_names(self, *, all: bool = False) -> set[str]: names = {modinfo.name for modinfo in self.modules} if all: names.update(WINDOWS_MODULES) @@ -280,9 +288,9 @@ def get_builddir(self) -> pathlib.Path: except FileNotFoundError: logger.error("%s must be run from the top build directory", __file__) raise - builddir = pathlib.Path(builddir) - logger.debug("%s: %s", self.pybuilddir_txt, builddir) - return builddir + builddir_path = pathlib.Path(builddir) + logger.debug("%s: %s", self.pybuilddir_txt, builddir_path) + return builddir_path def get_modules(self) -> list[ModuleInfo]: """Get module info from sysconfig and Modules/Setup* files""" @@ -367,7 +375,7 @@ def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]: case ["*disabled*"]: state = ModuleState.DISABLED case ["*noconfig*"]: - state = None + continue case [*items]: if state == ModuleState.DISABLED: # *disabled* can disable multiple modules per line @@ -384,26 +392,33 @@ def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]: def get_spec(self, modinfo: ModuleInfo) -> ModuleSpec: """Get ModuleSpec for builtin or extension module""" if modinfo.state == ModuleState.SHARED: - location = os.fspath(self.get_location(modinfo)) + mod_location = self.get_location(modinfo) + assert mod_location is not None + location = os.fspath(mod_location) loader = ExtensionFileLoader(modinfo.name, location) - return spec_from_file_location(modinfo.name, location, loader=loader) + spec = spec_from_file_location(modinfo.name, location, loader=loader) + assert spec is not None + return spec elif modinfo.state == ModuleState.BUILTIN: - return spec_from_loader(modinfo.name, loader=BuiltinImporter) + spec = spec_from_loader(modinfo.name, loader=BuiltinImporter) + assert spec is not None + return spec else: raise ValueError(modinfo) - def get_location(self, modinfo: ModuleInfo) -> pathlib.Path: + def get_location(self, modinfo: ModuleInfo) -> pathlib.Path | None: """Get shared library location in build directory""" if modinfo.state == ModuleState.SHARED: return self.builddir / f"{modinfo.name}{self.ext_suffix}" else: return None - def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec): + def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec) -> None: """Check that the module file is present and not empty""" - if spec.loader is BuiltinImporter: + if spec.loader is BuiltinImporter: # type: ignore[comparison-overlap] return try: + assert spec.origin is not None st = os.stat(spec.origin) except FileNotFoundError: logger.error("%s (%s) is missing", modinfo.name, spec.origin) @@ -411,7 +426,7 @@ def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec): if not st.st_size: raise ImportError(f"{spec.origin} is an empty file") - def check_module_import(self, modinfo: ModuleInfo): + def check_module_import(self, modinfo: ModuleInfo) -> None: """Attempt to import module and report errors""" spec = self.get_spec(modinfo) self._check_file(modinfo, spec) @@ -430,7 +445,7 @@ def check_module_import(self, modinfo: ModuleInfo): logger.exception("Importing extension '%s' failed!", modinfo.name) raise - def check_module_cross(self, modinfo: ModuleInfo): + def check_module_cross(self, modinfo: ModuleInfo) -> None: """Sanity check for cross compiling""" spec = self.get_spec(modinfo) self._check_file(modinfo, spec) @@ -443,6 +458,7 @@ def rename_module(self, modinfo: ModuleInfo) -> None: failed_name = f"{modinfo.name}_failed{self.ext_suffix}" builddir_path = self.get_location(modinfo) + assert builddir_path is not None if builddir_path.is_symlink(): symlink = builddir_path module_path = builddir_path.resolve().relative_to(os.getcwd()) @@ -466,7 +482,7 @@ def rename_module(self, modinfo: ModuleInfo) -> None: logger.debug("Rename '%s' -> '%s'", module_path, failed_path) -def main(): +def main() -> None: args = parser.parse_args() if args.debug: args.verbose = True diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index 88414cdbb37a8d..bda72539640611 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -1,9 +1,12 @@ # This script lists the names of standard library modules # to update Python/stdlib_module_names.h +from __future__ import annotations + import _imp import os.path import sys import sysconfig +from typing import TextIO from check_extension_modules import ModuleChecker @@ -48,12 +51,12 @@ } # Built-in modules -def list_builtin_modules(names): +def list_builtin_modules(names: set[str]) -> None: names |= set(sys.builtin_module_names) # Pure Python modules (Lib/*.py) -def list_python_modules(names): +def list_python_modules(names: set[str]) -> None: for filename in os.listdir(STDLIB_PATH): if not filename.endswith(".py"): continue @@ -62,7 +65,7 @@ def list_python_modules(names): # Packages in Lib/ -def list_packages(names): +def list_packages(names: set[str]) -> None: for name in os.listdir(STDLIB_PATH): if name in IGNORE: continue @@ -76,16 +79,16 @@ def list_packages(names): # Built-in and extension modules built by Modules/Setup* # includes Windows and macOS extensions. -def list_modules_setup_extensions(names): +def list_modules_setup_extensions(names: set[str]) -> None: checker = ModuleChecker() names.update(checker.list_module_names(all=True)) # List frozen modules of the PyImport_FrozenModules list (Python/frozen.c). # Use the "./Programs/_testembed list_frozen" command. -def list_frozen(names): +def list_frozen(names: set[str]) -> None: submodules = set() - for name in _imp._frozen_module_names(): + for name in _imp._frozen_module_names(): # type: ignore[attr-defined] # To skip __hello__, __hello_alias__ and etc. if name.startswith('__'): continue @@ -101,8 +104,8 @@ def list_frozen(names): raise Exception(f'unexpected frozen submodules: {sorted(submodules)}') -def list_modules(): - names = set() +def list_modules() -> set[str]: + names: set[str] = set() list_builtin_modules(names) list_modules_setup_extensions(names) @@ -127,7 +130,7 @@ def list_modules(): return names -def write_modules(fp, names): +def write_modules(fp: TextIO, names: set[str]) -> None: print(f"// Auto-generated by {SCRIPT_NAME}.", file=fp) print("// List used to create sys.stdlib_module_names.", file=fp) @@ -138,7 +141,7 @@ def write_modules(fp, names): print("};", file=fp) -def main(): +def main() -> None: if not sysconfig.is_python_build(): print(f"ERROR: {sys.executable} is not a Python build", file=sys.stderr) diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index 123dc895f90a1f..2abb21e51f1cf8 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -3,10 +3,12 @@ # Please, when adding new files here, also add them to: # .github/workflows/mypy.yml files = + Tools/build/check_extension_modules.py, Tools/build/compute-changes.py, Tools/build/deepfreeze.py, Tools/build/generate-build-details.py, Tools/build/generate_sbom.py, + Tools/build/generate_stdlib_module_names.py, Tools/build/verify_ensurepip_wheels.py, Tools/build/update_file.py, Tools/build/umarshal.py diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 6466d2615cd14e..9dd7e5dbfbae7b 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -5,7 +5,7 @@ import re from typing import Optional, Callable -from parser import Stmt, SimpleStmt, BlockStmt, IfStmt, WhileStmt +from parser import Stmt, SimpleStmt, BlockStmt, IfStmt, WhileStmt, ForStmt, MacroIfStmt @dataclass class EscapingCall: @@ -20,6 +20,7 @@ class Properties: error_with_pop: bool error_without_pop: bool deopts: bool + deopts_periodic: bool oparg: bool jumps: bool eval_breaker: bool @@ -58,6 +59,7 @@ def from_list(properties: list["Properties"]) -> "Properties": error_with_pop=any(p.error_with_pop for p in properties), error_without_pop=any(p.error_without_pop for p in properties), deopts=any(p.deopts for p in properties), + deopts_periodic=any(p.deopts_periodic for p in properties), oparg=any(p.oparg for p in properties), jumps=any(p.jumps for p in properties), eval_breaker=any(p.eval_breaker for p in properties), @@ -85,6 +87,7 @@ def infallible(self) -> bool: error_with_pop=False, error_without_pop=False, deopts=False, + deopts_periodic=False, oparg=False, jumps=False, eval_breaker=False, @@ -706,7 +709,7 @@ def visit(stmt: Stmt) -> None: in_if = 0 tkn_iter = iter(stmt.contents) for tkn in tkn_iter: - if tkn.kind == "IDENTIFIER" and tkn.text in ("DEOPT_IF", "ERROR_IF", "EXIT_IF"): + if tkn.kind == "IDENTIFIER" and tkn.text in ("DEOPT_IF", "ERROR_IF", "EXIT_IF", "HANDLE_PENDING_AND_DEOPT_IF", "AT_END_EXIT_IF"): in_if = 1 next(tkn_iter) elif tkn.kind == "LPAREN": @@ -723,53 +726,57 @@ def visit(stmt: Stmt) -> None: if error is not None: raise analysis_error(f"Escaping call '{error.text} in condition", error) +def escaping_call_in_simple_stmt(stmt: SimpleStmt, result: dict[SimpleStmt, EscapingCall]) -> None: + tokens = stmt.contents + for idx, tkn in enumerate(tokens): + try: + next_tkn = tokens[idx+1] + except IndexError: + break + if next_tkn.kind != lexer.LPAREN: + continue + if tkn.kind == lexer.IDENTIFIER: + if tkn.text.upper() == tkn.text: + # simple macro + continue + #if not tkn.text.startswith(("Py", "_Py", "monitor")): + # continue + if tkn.text.startswith(("sym_", "optimize_", "PyJitRef")): + # Optimize functions + continue + if tkn.text.endswith("Check"): + continue + if tkn.text.startswith("Py_Is"): + continue + if tkn.text.endswith("CheckExact"): + continue + if tkn.text in NON_ESCAPING_FUNCTIONS: + continue + elif tkn.kind == "RPAREN": + prev = tokens[idx-1] + if prev.text.endswith("_t") or prev.text == "*" or prev.text == "int": + #cast + continue + elif tkn.kind != "RBRACKET": + continue + if tkn.text in ("PyStackRef_CLOSE", "PyStackRef_XCLOSE"): + if len(tokens) <= idx+2: + raise analysis_error("Unexpected end of file", next_tkn) + kills = tokens[idx+2] + if kills.kind != "IDENTIFIER": + raise analysis_error(f"Expected identifier, got '{kills.text}'", kills) + else: + kills = None + result[stmt] = EscapingCall(stmt, tkn, kills) + + def find_escaping_api_calls(instr: parser.CodeDef) -> dict[SimpleStmt, EscapingCall]: result: dict[SimpleStmt, EscapingCall] = {} def visit(stmt: Stmt) -> None: if not isinstance(stmt, SimpleStmt): return - tokens = stmt.contents - for idx, tkn in enumerate(tokens): - try: - next_tkn = tokens[idx+1] - except IndexError: - break - if next_tkn.kind != lexer.LPAREN: - continue - if tkn.kind == lexer.IDENTIFIER: - if tkn.text.upper() == tkn.text: - # simple macro - continue - #if not tkn.text.startswith(("Py", "_Py", "monitor")): - # continue - if tkn.text.startswith(("sym_", "optimize_", "PyJitRef")): - # Optimize functions - continue - if tkn.text.endswith("Check"): - continue - if tkn.text.startswith("Py_Is"): - continue - if tkn.text.endswith("CheckExact"): - continue - if tkn.text in NON_ESCAPING_FUNCTIONS: - continue - elif tkn.kind == "RPAREN": - prev = tokens[idx-1] - if prev.text.endswith("_t") or prev.text == "*" or prev.text == "int": - #cast - continue - elif tkn.kind != "RBRACKET": - continue - if tkn.text in ("PyStackRef_CLOSE", "PyStackRef_XCLOSE"): - if len(tokens) <= idx+2: - raise analysis_error("Unexpected end of file", next_tkn) - kills = tokens[idx+2] - if kills.kind != "IDENTIFIER": - raise analysis_error(f"Expected identifier, got '{kills.text}'", kills) - else: - kills = None - result[stmt] = EscapingCall(stmt, tkn, kills) + escaping_call_in_simple_stmt(stmt, result) instr.block.accept(visit) check_escaping_calls(instr, result) @@ -822,6 +829,60 @@ def stack_effect_only_peeks(instr: parser.InstDef) -> bool: ) +def stmt_is_simple_exit(stmt: Stmt) -> bool: + if not isinstance(stmt, SimpleStmt): + return False + tokens = stmt.contents + if len(tokens) < 4: + return False + return ( + tokens[0].text in ("ERROR_IF", "DEOPT_IF", "EXIT_IF", "AT_END_EXIT_IF") + and + tokens[1].text == "(" + and + tokens[2].text in ("true", "1") + and + tokens[3].text == ")" + ) + + +def stmt_list_escapes(stmts: list[Stmt]) -> bool: + if not stmts: + return False + if stmt_is_simple_exit(stmts[-1]): + return False + for stmt in stmts: + if stmt_escapes(stmt): + return True + return False + + +def stmt_escapes(stmt: Stmt) -> bool: + if isinstance(stmt, BlockStmt): + return stmt_list_escapes(stmt.body) + elif isinstance(stmt, SimpleStmt): + for tkn in stmt.contents: + if tkn.text == "DECREF_INPUTS": + return True + d: dict[SimpleStmt, EscapingCall] = {} + escaping_call_in_simple_stmt(stmt, d) + return bool(d) + elif isinstance(stmt, IfStmt): + if stmt.else_body and stmt_escapes(stmt.else_body): + return True + return stmt_escapes(stmt.body) + elif isinstance(stmt, MacroIfStmt): + if stmt.else_body and stmt_list_escapes(stmt.else_body): + return True + return stmt_list_escapes(stmt.body) + elif isinstance(stmt, ForStmt): + return stmt_escapes(stmt.body) + elif isinstance(stmt, WhileStmt): + return stmt_escapes(stmt.body) + else: + assert False, "Unexpected statement type" + + def compute_properties(op: parser.CodeDef) -> Properties: escaping_calls = find_escaping_api_calls(op) has_free = ( @@ -831,11 +892,13 @@ def compute_properties(op: parser.CodeDef) -> Properties: or variable_used(op, "PyCell_SwapTakeRef") ) deopts_if = variable_used(op, "DEOPT_IF") - exits_if = variable_used(op, "EXIT_IF") - if deopts_if and exits_if: + exits_if = variable_used(op, "EXIT_IF") or variable_used(op, "AT_END_EXIT_IF") + deopts_periodic = variable_used(op, "HANDLE_PENDING_AND_DEOPT_IF") + exits_and_deopts = sum((deopts_if, exits_if, deopts_periodic)) + if exits_and_deopts > 1: tkn = op.tokens[0] raise lexer.make_syntax_error( - "Op cannot contain both EXIT_IF and DEOPT_IF", + "Op cannot contain more than one of EXIT_IF, DEOPT_IF and HANDLE_PENDING_AND_DEOPT_IF", tkn.filename, tkn.line, tkn.column, @@ -843,7 +906,7 @@ def compute_properties(op: parser.CodeDef) -> Properties: ) error_with_pop = has_error_with_pop(op) error_without_pop = has_error_without_pop(op) - escapes = bool(escaping_calls) or variable_used(op, "DECREF_INPUTS") + escapes = stmt_escapes(op.block) pure = False if isinstance(op, parser.LabelDef) else "pure" in op.annotations no_save_ip = False if isinstance(op, parser.LabelDef) else "no_save_ip" in op.annotations return Properties( @@ -852,6 +915,7 @@ def compute_properties(op: parser.CodeDef) -> Properties: error_with_pop=error_with_pop, error_without_pop=error_without_pop, deopts=deopts_if, + deopts_periodic=deopts_periodic, side_exit=exits_if, oparg=oparg_used(op), jumps=variable_used(op, "JUMPBY"), diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 4c210fbf8d28e9..cc0edcdf6006fd 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -111,7 +111,9 @@ class Emitter: def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False): self._replacers = { "EXIT_IF": self.exit_if, + "AT_END_EXIT_IF": self.exit_if_after, "DEOPT_IF": self.deopt_if, + "HANDLE_PENDING_AND_DEOPT_IF": self.periodic_if, "ERROR_IF": self.error_if, "ERROR_NO_POP": self.error_no_pop, "DECREF_INPUTS": self.decref_inputs, @@ -171,6 +173,29 @@ def deopt_if( exit_if = deopt_if + def periodic_if( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + raise NotImplementedError("HANDLE_PENDING_AND_DEOPT_IF not support in tier 1") + + def exit_if_after( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + storage.clear_inputs("in AT_END_EXIT_IF") + storage.flush(self.out) + storage.stack.clear(self.out) + return self.exit_if(tkn, tkn_iter, uop, storage, inst) + def goto_error(self, offset: int, storage: Storage) -> str: if offset > 0: return f"JUMP_TO_LABEL(pop_{offset}_error);" @@ -692,6 +717,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_EVAL_BREAK_FLAG") if p.deopts: flags.append("HAS_DEOPT_FLAG") + if p.deopts_periodic: + flags.append("HAS_PERIODIC_FLAG") if p.side_exit: flags.append("HAS_EXIT_FLAG") if not p.infallible: diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index 72020133738fa5..29e4e74da72154 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -174,7 +174,13 @@ list of annotations and their meanings are as follows: * `override`. For external use by other interpreter definitions to override the current instruction definition. * `pure`. This instruction has no side effects. -* 'tierN'. This instruction is only used by the tier N interpreter. +* `tierN`. This instruction is only used by the tier N interpreter. +* `specializing`. A prefix for an instructions related to adaptive interpreter. +* `replaced`. This instruction will be replaced in the final bytecode by its directed + version (either forward or backward). +* `register`. Currently does nothing. +* `replicate(N)`. Replicate the instruction N times to store the oparg "inside" the instruction. +* `no_save_ip`. This instruction does not affect the instruction pointer. ### Special functions/macros diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 0bcdc5395dcd8e..b649b38123388d 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -53,10 +53,9 @@ "ESCAPES", "EXIT", "PURE", - "PASSTHROUGH", - "OPARG_AND_1", "ERROR_NO_POP", "NO_SAVE_IP", + "PERIODIC", ] diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 32dc346d5e891a..2fc6794f6a0887 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -132,7 +132,7 @@ def uses_this(inst: Instruction) -> bool: continue for tkn in uop.body.tokens(): if (tkn.kind == "IDENTIFIER" - and (tkn.text in {"DEOPT_IF", "EXIT_IF"})): + and (tkn.text in {"DEOPT_IF", "EXIT_IF", "AT_END_EXIT_IF"})): return True return False diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index fc3bc47286f7f6..1bb5f48658ddfc 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -111,6 +111,8 @@ def exit_if( self.emit("}\n") return not always_true(first_tkn) + periodic_if = deopt_if + def oparg( self, tkn: Token, diff --git a/Tools/clinic/libclinic/_overlong_docstrings.py b/Tools/clinic/libclinic/_overlong_docstrings.py new file mode 100644 index 00000000000000..5ca335fab2875c --- /dev/null +++ b/Tools/clinic/libclinic/_overlong_docstrings.py @@ -0,0 +1,299 @@ +OVERLONG_SUMMARY = frozenset(( + # Lib/test/ + 'test_preprocessor_guarded_if_e_or_f', + + # Modules/ + '_abc._abc_init', + '_abc._abc_instancecheck', + '_abc._abc_register', + '_abc._abc_subclasscheck', + '_codecs.lookup', + '_ctypes.byref', + '_curses.can_change_color', + '_curses.is_term_resized', + '_curses.mousemask', + '_curses.reset_prog_mode', + '_curses.reset_shell_mode', + '_curses.termname', + '_curses.window.enclose', + '_functools.reduce', + '_gdbm.gdbm.setdefault', + '_hashlib.HMAC.hexdigest', + '_hashlib.openssl_shake_128', + '_hashlib.openssl_shake_256', + '_hashlib.pbkdf2_hmac', + '_hmac.HMAC.hexdigest', + '_interpreters.is_shareable', + '_io._BufferedIOBase.read1', + '_lzma._decode_filter_properties', + '_remote_debugging.RemoteUnwinder.__init__', + '_remote_debugging.RemoteUnwinder.get_all_awaited_by', + '_remote_debugging.RemoteUnwinder.get_async_stack_trace', + '_socket.inet_aton', + '_sre.SRE_Match.expand', + '_sre.SRE_Match.groupdict', + '_sre.SRE_Pattern.finditer', + '_sre.SRE_Pattern.search', + '_sre.SRE_Pattern.sub', + '_sre.SRE_Pattern.subn', + '_ssl._SSLContext.sni_callback', + '_ssl._SSLSocket.pending', + '_ssl._SSLSocket.sendfile', + '_ssl.get_default_verify_paths', + '_ssl.RAND_status', + '_sysconfig.config_vars', + '_testcapi.make_exception_with_doc', + '_testcapi.VectorCallClass.set_vectorcall', + '_tkinter.getbusywaitinterval', + '_tkinter.setbusywaitinterval', + '_tracemalloc.reset_peak', + '_zstd.get_frame_size', + '_zstd.set_parameter_types', + '_zstd.ZstdDecompressor.decompress', + 'array.array.buffer_info', + 'array.array.frombytes', + 'array.array.fromfile', + 'array.array.tobytes', + 'cmath.isfinite', + 'datetime.datetime.strptime', + 'gc.get_objects', + 'itertools.chain.from_iterable', + 'itertools.combinations_with_replacement.__new__', + 'itertools.cycle.__new__', + 'itertools.starmap.__new__', + 'itertools.takewhile.__new__', + 'math.comb', + 'math.perm', + 'os.getresgid', + 'os.lstat', + 'os.pread', + 'os.pwritev', + 'os.sched_getaffinity', + 'os.sched_rr_get_interval', + 'os.timerfd_gettime', + 'os.timerfd_gettime_ns', + 'os.urandom', + 'os.WIFEXITED', + 'os.WTERMSIG', + 'pwd.getpwall', + 'pyexpat.xmlparser.ExternalEntityParserCreate', + 'pyexpat.xmlparser.GetReparseDeferralEnabled', + 'pyexpat.xmlparser.SetParamEntityParsing', + 'pyexpat.xmlparser.UseForeignDTD', + 'readline.redisplay', + 'signal.set_wakeup_fd', + 'unicodedata.UCD.combining', + 'unicodedata.UCD.decomposition', + 'zoneinfo.ZoneInfo.dst', + 'zoneinfo.ZoneInfo.tzname', + 'zoneinfo.ZoneInfo.utcoffset', + + # Objects/ + 'B.zfill', + 'bytearray.count', + 'bytearray.endswith', + 'bytearray.extend', + 'bytearray.find', + 'bytearray.index', + 'bytearray.maketrans', + 'bytearray.rfind', + 'bytearray.rindex', + 'bytearray.rsplit', + 'bytearray.split', + 'bytearray.splitlines', + 'bytearray.startswith', + 'bytes.count', + 'bytes.endswith', + 'bytes.find', + 'bytes.index', + 'bytes.maketrans', + 'bytes.rfind', + 'bytes.rindex', + 'bytes.startswith', + 'code.replace', + 'complex.conjugate', + 'dict.pop', + 'float.as_integer_ratio', + 'frame.f_trace', + 'int.bit_count', + 'OrderedDict.fromkeys', + 'OrderedDict.pop', + 'set.symmetric_difference_update', + 'str.count', + 'str.endswith', + 'str.find', + 'str.index', + 'str.isprintable', + 'str.rfind', + 'str.rindex', + 'str.rsplit', + 'str.split', + 'str.startswith', + 'str.strip', + 'str.swapcase', + 'str.zfill', + + # PC/ + 'msvcrt.kbhit', + + # Python/ + '_jit.is_active', + '_jit.is_available', + '_jit.is_enabled', + 'marshal.dumps', + 'sys._current_exceptions', + 'sys._setprofileallthreads', + 'sys._settraceallthreads', +)) + +OVERLONG_BODY = frozenset(( + # Modules/ + '_bz2.BZ2Decompressor.decompress', + '_curses.color_content', + '_curses.flash', + '_curses.longname', + '_curses.resize_term', + '_curses.use_env', + '_curses.window.border', + '_curses.window.derwin', + '_curses.window.getch', + '_curses.window.getkey', + '_curses.window.inch', + '_curses.window.insch', + '_curses.window.insnstr', + '_curses.window.is_linetouched', + '_curses.window.noutrefresh', + '_curses.window.overlay', + '_curses.window.overwrite', + '_curses.window.refresh', + '_curses.window.scroll', + '_curses.window.subwin', + '_curses.window.touchline', + '_curses_panel.panel.hide', + '_functools.reduce', + '_hashlib.HMAC.hexdigest', + '_hmac.HMAC.hexdigest', + '_interpreters.capture_exception', + '_io._IOBase.seek', + '_io._TextIOBase.detach', + '_io.FileIO.read', + '_io.FileIO.readall', + '_io.FileIO.seek', + '_io.open', + '_io.open_code', + '_lzma.LZMADecompressor.decompress', + '_multibytecodec.MultibyteCodec.decode', + '_multibytecodec.MultibyteCodec.encode', + '_posixsubprocess.fork_exec', + '_remote_debugging.RemoteUnwinder.__init__', + '_remote_debugging.RemoteUnwinder.get_all_awaited_by', + '_remote_debugging.RemoteUnwinder.get_async_stack_trace', + '_remote_debugging.RemoteUnwinder.get_stack_trace', + '_socket.socket.send', + '_sqlite3.Blob.read', + '_sqlite3.Blob.seek', + '_sqlite3.Blob.write', + '_sqlite3.Connection.deserialize', + '_sqlite3.Connection.serialize', + '_sqlite3.Connection.set_progress_handler', + '_sqlite3.Connection.setlimit', + '_ssl._SSLContext.sni_callback', + '_ssl._SSLSocket.context', + '_ssl._SSLSocket.get_channel_binding', + '_ssl._SSLSocket.sendfile', + '_tkinter.setbusywaitinterval', + '_zstd.ZstdCompressor.compress', + '_zstd.ZstdCompressor.flush', + '_zstd.ZstdCompressor.set_pledged_input_size', + '_zstd.ZstdDecompressor.__new__', + '_zstd.ZstdDecompressor.decompress', + '_zstd.ZstdDecompressor.unused_data', + '_zstd.ZstdDict.__new__', + '_zstd.ZstdDict.as_digested_dict', + '_zstd.ZstdDict.as_prefix', + '_zstd.ZstdDict.as_undigested_dict', + 'array.array.byteswap', + 'array.array.fromunicode', + 'array.array.tounicode', + 'binascii.a2b_base64', + 'cmath.isclose', + 'datetime.date.fromtimestamp', + 'datetime.datetime.fromtimestamp', + 'datetime.time.strftime', + 'fcntl.ioctl', + 'fcntl.lockf', + 'gc.freeze', + 'itertools.combinations_with_replacement.__new__', + 'math.nextafter', + 'os.fspath', + 'os.link', + 'os.listdir', + 'os.listxattr', + 'os.lseek', + 'os.mknod', + 'os.preadv', + 'os.pwritev', + 'os.readinto', + 'os.rename', + 'os.replace', + 'os.setxattr', + 'pyexpat.xmlparser.GetInputContext', + 'pyexpat.xmlparser.UseForeignDTD', + 'select.devpoll', + 'select.poll', + 'select.select', + 'signal.setitimer', + 'signal.signal', + 'termios.tcsetwinsize', + 'zlib.Decompress.decompress', + 'zlib.ZlibDecompressor.decompress', + + # Objects/ + 'bytearray.maketrans', + 'bytearray.partition', + 'bytearray.replace', + 'bytearray.rpartition', + 'bytearray.rsplit', + 'bytearray.splitlines', + 'bytearray.strip', + 'bytes.maketrans', + 'bytes.partition', + 'bytes.replace', + 'bytes.rpartition', + 'bytes.rsplit', + 'bytes.splitlines', + 'bytes.strip', + 'float.__getformat__', + 'list.sort', + 'memoryview.tobytes', + 'str.capitalize', + 'str.isalnum', + 'str.isalpha', + 'str.isdecimal', + 'str.isdigit', + 'str.isidentifier', + 'str.islower', + 'str.isnumeric', + 'str.isspace', + 'str.isupper', + 'str.join', + 'str.partition', + 'str.removeprefix', + 'str.replace', + 'str.rpartition', + 'str.splitlines', + 'str.title', + 'str.translate', + + # PC/ + '_wmi.exec_query', + + # Python/ + '__import__', + '_contextvars.ContextVar.get', + '_contextvars.ContextVar.reset', + '_contextvars.ContextVar.set', + '_imp.acquire_lock', + 'marshal.dumps', + 'sys._stats_dump', +)) diff --git a/Tools/clinic/libclinic/dsl_parser.py b/Tools/clinic/libclinic/dsl_parser.py index eca41531f7c8e9..58430df6173fd0 100644 --- a/Tools/clinic/libclinic/dsl_parser.py +++ b/Tools/clinic/libclinic/dsl_parser.py @@ -14,6 +14,7 @@ from libclinic import ( ClinicError, VersionTuple, fail, warn, unspecified, unknown, NULL) +from libclinic._overlong_docstrings import OVERLONG_SUMMARY, OVERLONG_BODY from libclinic.function import ( Module, Class, Function, Parameter, FunctionKind, @@ -1515,6 +1516,28 @@ def format_docstring(self) -> str: # between it and the {parameters} we're about to add. lines.append('') + # Fail if the summary line is too long. + # Warn if any of the body lines are too long. + # Existing violations are recorded in OVERLONG_{SUMMARY,BODY}. + max_width = f.docstring_line_width + summary_len = len(lines[0]) + max_body = max(map(len, lines[1:])) + if summary_len > max_width: + if f.full_name not in OVERLONG_SUMMARY: + fail(f"Summary line for {f.full_name!r} is too long!\n" + f"The summary line must be no longer than {max_width} characters.") + else: + if f.full_name in OVERLONG_SUMMARY: + warn(f"Remove {f.full_name!r} from OVERLONG_SUMMARY!\n") + + if max_body > max_width: + if f.full_name not in OVERLONG_BODY: + warn(f"Docstring lines for {f.full_name!r} are too long!\n" + f"Lines should be no longer than {max_width} characters.") + else: + if f.full_name in OVERLONG_BODY: + warn(f"Remove {f.full_name!r} from OVERLONG_BODY!\n") + parameters_marker_count = len(f.docstring.split('{parameters}')) - 1 if parameters_marker_count > 1: fail('You may not specify {parameters} more than once in a docstring!') diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py index e80e2f5f13f648..4280af0c4c9b49 100644 --- a/Tools/clinic/libclinic/function.py +++ b/Tools/clinic/libclinic/function.py @@ -167,6 +167,19 @@ def methoddef_flags(self) -> str | None: flags.append('METH_COEXIST') return '|'.join(flags) + @property + def docstring_line_width(self) -> int: + """Return the maximum line width for docstring lines. + + Pydoc adds indentation when displaying functions and methods. + To keep the total width of within 80 characters, we use a + maximum of 76 characters for global functions and classes, + and 72 characters for methods. + """ + if self.cls is not None and not self.kind.new_or_init: + return 72 + return 76 + def __repr__(self) -> str: return f'' diff --git a/Tools/i18n/msgfmt.py b/Tools/i18n/msgfmt.py index cd5f1ed9f3e268..65254a7c375456 100755 --- a/Tools/i18n/msgfmt.py +++ b/Tools/i18n/msgfmt.py @@ -5,8 +5,7 @@ This program converts a textual Uniforum-style message catalog (.po file) into a binary GNU catalog (.mo file). This is essentially the same function as the -GNU msgfmt program, however, it is a simpler implementation. Currently it -does not handle plural forms but it does handle message contexts. +GNU msgfmt program, however, it is a simpler implementation. Usage: msgfmt.py [OPTIONS] filename.po diff --git a/Tools/jit/template.c b/Tools/jit/template.c index d07f56e9ce6b42..7bbc44a08a0a98 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -51,7 +51,6 @@ do { \ OPT_STAT_INC(traces_executed); \ _PyExecutorObject *_executor = (EXECUTOR); \ - tstate->current_executor = (PyObject *)_executor; \ jit_func_preserve_none jitted = _executor->jit_side_entry; \ __attribute__((musttail)) return jitted(frame, stack_pointer, tstate); \ } while (0) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 0beaab2d3e7157..73236767374378 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,7 +1,7 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.16.1 +mypy==1.17.1 # needed for peg_generator: -types-psutil==7.0.0.20250601 -types-setuptools==80.9.0.20250529 +types-psutil==7.0.0.20250801 +types-setuptools==80.9.0.20250801 diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 93421b623b92f9..6bd31e8e6ecb9d 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -43,8 +43,5 @@ race:list_inplace_repeat_lock_held race:PyObject_Realloc # gh-133467. Some of these could be hard to trigger. -race_top:_Py_slot_tp_getattr_hook -race_top:slot_tp_descr_get -race_top:type_set_name race_top:set_tp_bases race_top:type_set_bases_unlocked diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index fdd93d5c0aa4af..973d78caa0849e 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -20,7 +20,8 @@ assert (CHECKOUT / "configure").is_file(), "Please update the location of the file" CROSS_BUILD_DIR = CHECKOUT / "cross-build" -BUILD_DIR = CROSS_BUILD_DIR / "build" +# Build platform can also be found via `config.guess`. +BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE") LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" LOCAL_SETUP_MARKER = ("# Generated by Tools/wasm/wasi .\n" @@ -80,7 +81,7 @@ def wrapper(context): print("📁", working_dir) if (clean_ok and getattr(context, "clean", False) and working_dir.exists()): - print(f"🚮 Deleting directory (--clean)...") + print("🚮 Deleting directory (--clean)...") shutil.rmtree(working_dir) working_dir.mkdir(parents=True, exist_ok=True) @@ -120,12 +121,6 @@ def call(command, *, context=None, quiet=False, logdir=None, **kwargs): subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) -def build_platform(): - """The name of the build/host platform.""" - # Can also be found via `config.guess`. - return sysconfig.get_config_var("BUILD_GNU_TYPE") - - def build_python_path(): """The path to the build Python binary.""" binary = BUILD_DIR / "python" @@ -274,7 +269,7 @@ def configure_wasi_python(context, working_dir): # executed from within a checkout. configure = [os.path.relpath(CHECKOUT / 'configure', working_dir), f"--host={context.host_triple}", - f"--build={build_platform()}", + f"--build={BUILD_DIR.name}", f"--with-build-python={build_python}"] if build_python_is_pydebug(): configure.append("--with-pydebug") @@ -357,8 +352,8 @@ def main(): "Python)") make_host = subcommands.add_parser("make-host", help="Run `make` for the host/WASI") - clean = subcommands.add_parser("clean", help="Delete files and directories " - "created by this script") + subcommands.add_parser("clean", help="Delete files and directories " + "created by this script") for subcommand in build, configure_build, make_build, configure_host, make_host: subcommand.add_argument("--quiet", action="store_true", default=False, dest="quiet", diff --git a/configure b/configure index 0e7aefed5ee62d..451f72fdfd4ec3 100755 --- a/configure +++ b/configure @@ -725,8 +725,6 @@ LIBHACL_BLAKE2_SIMD128_OBJS LIBHACL_SIMD128_FLAGS LIBHACL_LDFLAGS LIBHACL_CFLAGS -LIBHASHLIB_INTERNAL -LIBHASHLIB_INTERNAL_CFLAGS MODULE_UNICODEDATA_FALSE MODULE_UNICODEDATA_TRUE MODULE__MULTIBYTECODEC_FALSE @@ -19039,15 +19037,27 @@ printf "%s\n" "#define WITH_DTRACE 1" >>confdefs.h # linked into the binary. Correspondingly, dtrace(1) is missing the ELF # generation flag '-G'. We check for presence of this flag, rather than # hardcoding support by OS, in the interest of robustness. + # + # NetBSD DTrace requires the -x nolibs flag to avoid system library conflicts + # and uses header generation for testing instead of object generation. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether DTrace probes require linking" >&5 printf %s "checking whether DTrace probes require linking... " >&6; } if test ${ac_cv_dtrace_link+y} then : printf %s "(cached) " >&6 else case e in #( - e) ac_cv_dtrace_link=no + e) + ac_cv_dtrace_link=no echo 'BEGIN{}' > conftest.d - "$DTRACE" $DFLAGS -G -s conftest.d -o conftest.o > /dev/null 2>&1 && \ + case $host in + *netbsd*) + DTRACE_TEST_FLAGS="-x nolibs -h" + ;; + *) + DTRACE_TEST_FLAGS="-G" + ;; + esac + "$DTRACE" $DFLAGS $DTRACE_TEST_FLAGS -s conftest.d -o conftest.o > /dev/null 2>&1 && \ ac_cv_dtrace_link=yes ;; esac @@ -19057,6 +19067,12 @@ printf "%s\n" "$ac_cv_dtrace_link" >&6; } if test "$ac_cv_dtrace_link" = "yes"; then DTRACE_OBJS="Python/pydtrace.o" fi + # Set NetBSD-specific DTrace flags in DFLAGS + case $host in + *netbsd*) + DFLAGS="$DFLAGS -x nolibs" + ;; + esac fi PLATFORM_HEADERS= @@ -29951,7 +29967,6 @@ SRCDIRS="\ Modules/_decimal \ Modules/_decimal/libmpdec \ Modules/_hacl \ - Modules/_hashlib \ Modules/_io \ Modules/_multiprocessing \ Modules/_sqlite \ @@ -32528,15 +32543,6 @@ then : fi -############################################################################### -# Cryptographic primitives -LIBHASHLIB_INTERNAL_CFLAGS="-I\$(srcdir)/Modules/_hashlib" -LIBHASHLIB_INTERNAL_LDFLAGS="-lm \$(LIBHASHLIB_INTERNAL_A)" -LIBHASHLIB_INTERNAL="\$(LIBHASHLIB_INTERNAL_HEADERS) \$(LIBHASHLIB_INTERNAL_A)" - - - - ############################################################################### # HACL* compilation and linking configuration (contact: @picnixz) # @@ -32785,8 +32791,8 @@ fi if test "x$py_cv_module__md5" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS)$as_nl" fi if test "$py_cv_module__md5" = yes; then @@ -32830,8 +32836,8 @@ fi if test "x$py_cv_module__sha1" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS)$as_nl" fi if test "$py_cv_module__sha1" = yes; then @@ -32875,8 +32881,8 @@ fi if test "x$py_cv_module__sha2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS)$as_nl" fi if test "$py_cv_module__sha2" = yes; then @@ -32920,8 +32926,8 @@ fi if test "x$py_cv_module__sha3" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS)$as_nl" fi if test "$py_cv_module__sha3" = yes; then @@ -32965,8 +32971,8 @@ fi if test "x$py_cv_module__blake2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS)$as_nl" fi if test "$py_cv_module__blake2" = yes; then @@ -33011,8 +33017,8 @@ fi if test "x$py_cv_module__hmac" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS)$as_nl" fi if test "$py_cv_module__hmac" = yes; then @@ -33693,8 +33699,8 @@ fi if test "x$py_cv_module__hashlib" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS$as_nl" fi if test "$py_cv_module__hashlib" = yes; then diff --git a/configure.ac b/configure.ac index 1e590e1d0fd727..f0e9eb2ee88c03 100644 --- a/configure.ac +++ b/configure.ac @@ -5104,16 +5104,33 @@ then # linked into the binary. Correspondingly, dtrace(1) is missing the ELF # generation flag '-G'. We check for presence of this flag, rather than # hardcoding support by OS, in the interest of robustness. + # + # NetBSD DTrace requires the -x nolibs flag to avoid system library conflicts + # and uses header generation for testing instead of object generation. AC_CACHE_CHECK([whether DTrace probes require linking], - [ac_cv_dtrace_link], [dnl + [ac_cv_dtrace_link], [ ac_cv_dtrace_link=no echo 'BEGIN{}' > conftest.d - "$DTRACE" $DFLAGS -G -s conftest.d -o conftest.o > /dev/null 2>&1 && \ + case $host in + *netbsd*) + DTRACE_TEST_FLAGS="-x nolibs -h" + ;; + *) + DTRACE_TEST_FLAGS="-G" + ;; + esac + "$DTRACE" $DFLAGS $DTRACE_TEST_FLAGS -s conftest.d -o conftest.o > /dev/null 2>&1 && \ ac_cv_dtrace_link=yes ]) if test "$ac_cv_dtrace_link" = "yes"; then DTRACE_OBJS="Python/pydtrace.o" fi + # Set NetBSD-specific DTrace flags in DFLAGS + case $host in + *netbsd*) + DFLAGS="$DFLAGS -x nolibs" + ;; + esac fi dnl Platform-specific C and header files. @@ -5219,7 +5236,7 @@ AC_CHECK_FUNCS([ \ posix_spawn_file_actions_addclosefrom_np \ pread preadv preadv2 process_vm_readv \ pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ - pthread_kill pthread_get_name_np pthread_getname_np pthread_set_name_np + pthread_kill pthread_get_name_np pthread_getname_np pthread_set_name_np \ pthread_setname_np pthread_getattr_np \ ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ rtpSpawn sched_get_priority_max sched_rr_get_interval sched_setaffinity \ @@ -7187,7 +7204,6 @@ SRCDIRS="\ Modules/_decimal \ Modules/_decimal/libmpdec \ Modules/_hacl \ - Modules/_hashlib \ Modules/_io \ Modules/_multiprocessing \ Modules/_sqlite \ @@ -7958,15 +7974,6 @@ PY_STDLIB_MOD_SIMPLE([_codecs_tw]) PY_STDLIB_MOD_SIMPLE([_multibytecodec]) PY_STDLIB_MOD_SIMPLE([unicodedata]) -############################################################################### -# Cryptographic primitives -LIBHASHLIB_INTERNAL_CFLAGS="-I\$(srcdir)/Modules/_hashlib" -LIBHASHLIB_INTERNAL_LDFLAGS="-lm \$(LIBHASHLIB_INTERNAL_A)" -LIBHASHLIB_INTERNAL="\$(LIBHASHLIB_INTERNAL_HEADERS) \$(LIBHASHLIB_INTERNAL_A)" - -AC_SUBST([LIBHASHLIB_INTERNAL_CFLAGS]) -AC_SUBST([LIBHASHLIB_INTERNAL]) - ############################################################################### # HACL* compilation and linking configuration (contact: @picnixz) # @@ -8103,9 +8110,7 @@ dnl The EXTNAME is the name of the extension module being built. AC_DEFUN([PY_HACL_CREATE_MODULE], [ AS_VAR_PUSHDEF([v], [[LIBHACL_][$1][_LDFLAGS]]) AS_VAR_SET([v], [[LIBHACL_][$1][_LIB_${LIBHACL_LDEPS_LIBTYPE}]]) - PY_STDLIB_MOD([$2], [$3], [], - [$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS], - [\$($v) $LIBHASHLIB_INTERNAL_LDFLAGS]) + PY_STDLIB_MOD([$2], [$3], [], [$LIBHACL_CFLAGS], [\$($v)]) AS_VAR_POPDEF([v]) ]) @@ -8186,8 +8191,7 @@ dnl OpenSSL bindings PY_STDLIB_MOD([_ssl], [], [test "$ac_cv_working_openssl_ssl" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS]) PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], - [$OPENSSL_INCLUDES $LIBHASHLIB_INTERNAL_CFLAGS], - [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_INTERNAL_LDFLAGS]) + [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) dnl test modules PY_STDLIB_MOD([_testcapi],