diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3422ef835279bc..d40519e40d3cc2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -7,6 +7,9 @@ # GitHub .github/** @ezio-melotti @hugovk +# pre-commit +.pre-commit-config.yaml @hugovk @AlexWaygood + # Build system configure* @erlend-aasland @corona10 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000000000..4481ea80bfd936 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,22 @@ +name: Lint + +on: [push, pull_request, workflow_dispatch] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + - uses: pre-commit/action@v3.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000000..808622f19a3dbf --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-yaml + - id: trailing-whitespace + types_or: [c, python, rst] diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index f02abb01f0220b..0ca8ad624b2034 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -154,7 +154,7 @@ Dictionary Objects .. c:function:: Py_ssize_t PyDict_Size(PyObject *p) - .. index:: builtin: len + .. index:: pair: built-in function; len Return the number of items in the dictionary. This is equivalent to ``len(p)`` on a dictionary. diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 8e5af32b65e02c..79843ba521ab93 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -41,7 +41,7 @@ Importing Modules .. c:function:: PyObject* PyImport_ImportModuleEx(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist) - .. index:: builtin: __import__ + .. index:: pair: built-in function; __import__ Import a module. This is best described by referring to the built-in Python function :func:`__import__`. @@ -120,7 +120,7 @@ Importing Modules .. c:function:: PyObject* PyImport_ExecCodeModule(const char *name, PyObject *co) - .. index:: builtin: compile + .. index:: pair: built-in function; compile Given a module name (possibly of the form ``package.module``) and a code object read from a Python bytecode file or obtained from the built-in function diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 317421f0db8453..dbf35611eccd3e 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -45,7 +45,7 @@ List Objects .. c:function:: Py_ssize_t PyList_Size(PyObject *list) - .. index:: builtin: len + .. index:: pair: built-in function; len Return the length of the list object in *list*; this is equivalent to ``len(list)`` on a list object. @@ -138,7 +138,7 @@ List Objects .. c:function:: PyObject* PyList_AsTuple(PyObject *list) - .. index:: builtin: tuple + .. index:: pair: built-in function; tuple Return a new tuple object containing the contents of *list*; equivalent to ``tuple(list)``. diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index 3c9d282c6d0ab0..cffb0ed50fb77d 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -20,7 +20,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. c:function:: Py_ssize_t PyMapping_Size(PyObject *o) Py_ssize_t PyMapping_Length(PyObject *o) - .. index:: builtin: len + .. index:: pair: built-in function; len Returns the number of keys in object *o* on success, and ``-1`` on failure. This is equivalent to the Python expression ``len(o)``. diff --git a/Doc/c-api/number.rst b/Doc/c-api/number.rst index 70b91f8c2d0ca1..13d3c5af956905 100644 --- a/Doc/c-api/number.rst +++ b/Doc/c-api/number.rst @@ -64,7 +64,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Divmod(PyObject *o1, PyObject *o2) - .. index:: builtin: divmod + .. index:: pair: built-in function; divmod See the built-in function :func:`divmod`. Returns ``NULL`` on failure. This is the equivalent of the Python expression ``divmod(o1, o2)``. @@ -72,7 +72,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Power(PyObject *o1, PyObject *o2, PyObject *o3) - .. index:: builtin: pow + .. index:: pair: built-in function; pow See the built-in function :func:`pow`. Returns ``NULL`` on failure. This is the equivalent of the Python expression ``pow(o1, o2, o3)``, where *o3* is optional. @@ -94,7 +94,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Absolute(PyObject *o) - .. index:: builtin: abs + .. index:: pair: built-in function; abs Returns the absolute value of *o*, or ``NULL`` on failure. This is the equivalent of the Python expression ``abs(o)``. @@ -192,7 +192,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_InPlacePower(PyObject *o1, PyObject *o2, PyObject *o3) - .. index:: builtin: pow + .. index:: pair: built-in function; pow See the built-in function :func:`pow`. Returns ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python @@ -238,7 +238,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Long(PyObject *o) - .. index:: builtin: int + .. index:: pair: built-in function; int Returns the *o* converted to an integer object on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``int(o)``. @@ -246,7 +246,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Float(PyObject *o) - .. index:: builtin: float + .. index:: pair: built-in function; float Returns the *o* converted to a float object on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``float(o)``. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index a0c3194ab0fb90..a25ff244c9f07c 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -190,7 +190,7 @@ Object Protocol .. c:function:: PyObject* PyObject_Repr(PyObject *o) - .. index:: builtin: repr + .. index:: pair: built-in function; repr Compute a string representation of object *o*. Returns the string representation on success, ``NULL`` on failure. This is the equivalent of the @@ -202,7 +202,7 @@ Object Protocol .. c:function:: PyObject* PyObject_ASCII(PyObject *o) - .. index:: builtin: ascii + .. index:: pair: built-in function; ascii As :c:func:`PyObject_Repr`, compute a string representation of object *o*, but escape the non-ASCII characters in the string returned by @@ -227,7 +227,7 @@ Object Protocol .. c:function:: PyObject* PyObject_Bytes(PyObject *o) - .. index:: builtin: bytes + .. index:: pair: built-in function; bytes Compute a bytes representation of object *o*. ``NULL`` is returned on failure and a bytes object on success. This is equivalent to the Python @@ -278,7 +278,7 @@ Object Protocol .. c:function:: Py_hash_t PyObject_Hash(PyObject *o) - .. index:: builtin: hash + .. index:: pair: built-in function; hash Compute and return the hash value of an object *o*. On failure, return ``-1``. This is the equivalent of the Python expression ``hash(o)``. @@ -312,7 +312,7 @@ Object Protocol .. c:function:: PyObject* PyObject_Type(PyObject *o) - .. index:: builtin: type + .. index:: pair: built-in function; type When *o* is non-``NULL``, returns a type object corresponding to the object type of object *o*. On failure, raises :exc:`SystemError` and returns ``NULL``. This @@ -332,7 +332,7 @@ Object Protocol .. c:function:: Py_ssize_t PyObject_Size(PyObject *o) Py_ssize_t PyObject_Length(PyObject *o) - .. index:: builtin: len + .. index:: pair: built-in function; len Return the length of object *o*. If the object *o* provides either the sequence and mapping protocols, the sequence length is returned. On error, ``-1`` is diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index c78d273f9f149f..402a3e5e09ff56 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -18,7 +18,7 @@ Sequence Protocol .. c:function:: Py_ssize_t PySequence_Size(PyObject *o) Py_ssize_t PySequence_Length(PyObject *o) - .. index:: builtin: len + .. index:: pair: built-in function; len Returns the number of objects in sequence *o* on success, and ``-1`` on failure. This is equivalent to the Python expression ``len(o)``. @@ -120,7 +120,7 @@ Sequence Protocol .. c:function:: PyObject* PySequence_Tuple(PyObject *o) - .. index:: builtin: tuple + .. index:: pair: built-in function; tuple Return a tuple object with the same contents as the sequence or iterable *o*, or ``NULL`` on failure. If *o* is a tuple, a new reference will be returned, diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 8e8af60252250e..d642a5f1902e2e 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -107,7 +107,7 @@ or :class:`frozenset` or instances of their subtypes. .. c:function:: Py_ssize_t PySet_Size(PyObject *anyset) - .. index:: builtin: len + .. index:: pair: built-in function; len Return the length of a :class:`set` or :class:`frozenset` object. Equivalent to ``len(anyset)``. Raises a :exc:`PyExc_SystemError` if *anyset* is not a diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 338db6378d240e..aae1b951804491 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -347,7 +347,7 @@ method. .. data:: METH_CLASS - .. index:: builtin: classmethod + .. index:: pair: built-in function; classmethod The method will be passed the type object as the first parameter rather than an instance of the type. This is used to create *class methods*, @@ -357,7 +357,7 @@ method. .. data:: METH_STATIC - .. index:: builtin: staticmethod + .. index:: pair: built-in function; staticmethod The method will be passed ``NULL`` as the first parameter rather than an instance of the type. This is used to create *static methods*, similar to diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index e13db3fb2211e2..0584989233de3f 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -805,7 +805,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_repr - .. index:: builtin: repr + .. index:: pair: built-in function; repr An optional pointer to a function that implements the built-in function :func:`repr`. @@ -870,7 +870,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: hashfunc PyTypeObject.tp_hash - .. index:: builtin: hash + .. index:: pair: built-in function; hash An optional pointer to a function that implements the built-in function :func:`hash`. diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 56b40acdb69fed..6852a385f0c63c 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -149,7 +149,7 @@ done. This can be done using the :c:func:`PyErr_Fetch` and .. index:: single: string; object representation - builtin: repr + pair: built-in function; repr Object Presentation ------------------- diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 8a10fe327358c0..6ebc2d9b0a71a9 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1033,19 +1033,36 @@ you're not permitted to use: Using a return converter ------------------------ -By default the impl function Argument Clinic generates for you returns ``PyObject *``. -But your C function often computes some C type, then converts it into the ``PyObject *`` +By default, the impl function Argument Clinic generates for you returns +:c:type:`PyObject * `. +But your C function often computes some C type, +then converts it into the :c:type:`!PyObject *` at the last moment. Argument Clinic handles converting your inputs from Python types into native C types—why not have it convert your return value from a native C type into a Python type too? That's what a "return converter" does. It changes your impl function to return some C type, then adds code to the generated (non-impl) function to handle converting -that value into the appropriate ``PyObject *``. +that value into the appropriate :c:type:`!PyObject *`. The syntax for return converters is similar to that of parameter converters. You specify the return converter like it was a return annotation on the -function itself. Return converters behave much the same as parameter converters; +function itself, using ``->`` notation. + +For example: + +.. code-block:: c + + /*[clinic input] + add -> int + + a: int + b: int + / + + [clinic start generated code]*/ + +Return converters behave much the same as parameter converters; they take arguments, the arguments are all keyword-only, and if you're not changing any of the default arguments you can omit the parentheses. @@ -1066,19 +1083,17 @@ Currently Argument Clinic supports only a few return converters: .. code-block:: none bool + double + float int - unsigned int long - unsigned int - size_t Py_ssize_t - float - double - DecodeFSDefault + size_t + unsigned int + unsigned long -None of these take parameters. For the first three, return -1 to indicate -error. For ``DecodeFSDefault``, the return type is ``const char *``; return a ``NULL`` -pointer to indicate an error. +None of these take parameters. +For all of these, return ``-1`` to indicate error. To see all the return converters Argument Clinic supports, along with their parameters (if any), diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index f8727b98066990..0f9b9e5668cf4a 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -527,6 +527,8 @@ Running Tasks Concurrently and there is no running event loop. +.. _eager-task-factory: + Eager Task Factory ================== @@ -560,6 +562,13 @@ Eager Task Factory using the provided *custom_task_constructor* when creating a new task instead of the default :class:`Task`. + *custom_task_constructor* must be a *callable* with the signature matching + the signature of :class:`Task.__init__ `. + The callable must return a :class:`asyncio.Task`-compatible object. + + This function returns a *callable* intended to be used as a task factory of an + event loop via :meth:`loop.set_task_factory(factory) `). + .. versionadded:: 3.12 @@ -1014,7 +1023,7 @@ Introspection Task Object =========== -.. class:: Task(coro, *, loop=None, name=None) +.. class:: Task(coro, *, loop=None, name=None, context=None, eager_start=False) A :class:`Future-like ` object that runs a Python :ref:`coroutine `. Not thread-safe. @@ -1049,9 +1058,17 @@ Task Object APIs except :meth:`Future.set_result` and :meth:`Future.set_exception`. - Tasks support the :mod:`contextvars` module. When a Task - is created it copies the current context and later runs its - coroutine in the copied context. + An optional keyword-only *context* argument allows specifying a + custom :class:`contextvars.Context` for the *coro* to run in. + If no *context* is provided, the Task copies the current context + and later runs its coroutine in the copied context. + + An optional keyword-only *eager_start* argument allows eagerly starting + the execution of the :class:`asyncio.Task` at task creation time. + If set to ``True`` and the event loop is running, the task will start + executing the coroutine immediately, until the first time the coroutine + blocks. If the coroutine returns or raises without blocking, the task + will be finished eagerly and will skip scheduling to the event loop. .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. @@ -1063,6 +1080,12 @@ Task Object Deprecation warning is emitted if *loop* is not specified and there is no running event loop. + .. versionchanged:: 3.11 + Added the *context* parameter. + + .. versionchanged:: 3.12 + Added the *eager_start* parameter. + .. method:: done() Return ``True`` if the Task is *done*. @@ -1153,8 +1176,16 @@ Task Object Return the coroutine object wrapped by the :class:`Task`. + .. note:: + + This will return ``None`` for Tasks which have already + completed eagerly. See the :ref:`Eager Task Factory `. + .. versionadded:: 3.8 + .. versionchanged:: 3.12 + Added note on specific eager task execution behavior. + .. method:: get_context() Return the :class:`contextvars.Context` object diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index e3c8c801904b61..8022c596f0af97 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -24,6 +24,8 @@ method to determine whether a value has been found. Instead, the functions only call the :meth:`__lt__` method and will return an insertion point between values in an array. +.. _bisect functions: + The following functions are provided: @@ -55,7 +57,7 @@ The following functions are provided: .. function:: bisect_right(a, x, lo=0, hi=len(a), *, key=None) bisect(a, x, lo=0, hi=len(a), *, key=None) - Similar to :func:`bisect_left`, but returns an insertion point which comes + Similar to :py:func:`~bisect.bisect_left`, but returns an insertion point which comes after (to the right of) any existing entries of *x* in *a*. The returned insertion point *ip* partitions the array *a* into two slices @@ -70,7 +72,7 @@ The following functions are provided: Insert *x* in *a* in sorted order. - This function first runs :func:`bisect_left` to locate an insertion point. + This function first runs :py:func:`~bisect.bisect_left` to locate an insertion point. Next, it runs the :meth:`insert` method on *a* to insert *x* at the appropriate position to maintain sort order. @@ -87,10 +89,10 @@ The following functions are provided: .. function:: insort_right(a, x, lo=0, hi=len(a), *, key=None) insort(a, x, lo=0, hi=len(a), *, key=None) - Similar to :func:`insort_left`, but inserting *x* in *a* after any existing + Similar to :py:func:`~bisect.insort_left`, but inserting *x* in *a* after any existing entries of *x*. - This function first runs :func:`bisect_right` to locate an insertion point. + This function first runs :py:func:`~bisect.bisect_right` to locate an insertion point. Next, it runs the :meth:`insert` method on *a* to insert *x* at the appropriate position to maintain sort order. @@ -120,7 +122,7 @@ thoughts in mind: they are used. Consequently, if the search functions are used in a loop, the key function may be called again and again on the same array elements. If the key function isn't fast, consider wrapping it with - :func:`functools.cache` to avoid duplicate computations. Alternatively, + :py:func:`functools.cache` to avoid duplicate computations. Alternatively, consider searching an array of precomputed keys to locate the insertion point (as shown in the examples section below). @@ -140,7 +142,7 @@ thoughts in mind: Searching Sorted Lists ---------------------- -The above :func:`bisect` functions are useful for finding insertion points but +The above `bisect functions`_ are useful for finding insertion points but can be tricky or awkward to use for common searching tasks. The following five functions show how to transform them into the standard lookups for sorted lists:: @@ -186,8 +188,8 @@ Examples .. _bisect-example: -The :func:`bisect` function can be useful for numeric table lookups. This -example uses :func:`bisect` to look up a letter grade for an exam score (say) +The :py:func:`~bisect.bisect` function can be useful for numeric table lookups. This +example uses :py:func:`~bisect.bisect` to look up a letter grade for an exam score (say) based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is a 'B', and so on:: @@ -198,8 +200,8 @@ a 'B', and so on:: >>> [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]] ['F', 'A', 'C', 'C', 'B', 'A', 'A'] -The :func:`bisect` and :func:`insort` functions also work with lists of -tuples. The *key* argument can serve to extract the field used for ordering +The :py:func:`~bisect.bisect` and :py:func:`~bisect.insort` functions also work with +lists of tuples. The *key* argument can serve to extract the field used for ordering records in a table:: >>> from collections import namedtuple diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 6c3f436ddb1494..296d8a9c66faa4 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1367,7 +1367,7 @@ iterations of the loop. .. opcode:: BUILD_SLICE (argc) - .. index:: builtin: slice + .. index:: pair: built-in function; slice Pushes a slice object on the stack. *argc* must be 2 or 3. If it is 2, implements:: diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 80b7b8b4f4ed07..48a832db60e919 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -562,7 +562,7 @@ are always available. They are listed here in alphabetical order. Raises an :ref:`auditing event ` ``exec`` with the code object as the argument. Code compilation events may also be raised. -.. index:: builtin: exec +.. index:: pair: built-in function; exec .. function:: exec(object, globals=None, locals=None, /, *, closure=None) diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 3097bcf47b627f..d2cc769e2c8400 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -1,11 +1,11 @@ .. _using: -================================= - Using :mod:`!importlib.metadata` -================================= +======================================================== +:mod:`!importlib.metadata` -- Accessing package metadata +======================================================== .. module:: importlib.metadata - :synopsis: The implementation of the importlib metadata. + :synopsis: Accessing package metadata .. versionadded:: 3.8 .. versionchanged:: 3.10 @@ -13,7 +13,7 @@ **Source code:** :source:`Lib/importlib/metadata/__init__.py` -``importlib_metadata`` is a library that provides access to +``importlib.metadata`` is a library that provides access to the metadata of an installed `Distribution Package `_, such as its entry points or its top-level names (`Import Package `_\s, modules, if any). @@ -24,7 +24,7 @@ API`_ and `metadata API`_ of ``pkg_resources``. Along with this package can eliminate the need to use the older and less efficient ``pkg_resources`` package. -``importlib_metadata`` operates on third-party *distribution packages* +``importlib.metadata`` operates on third-party *distribution packages* installed into Python's ``site-packages`` directory via tools such as `pip `_. Specifically, it works with distributions with discoverable @@ -368,7 +368,7 @@ system :ref:`finders `. To find a distribution package's m ``importlib.metadata`` queries the list of :term:`meta path finders ` on :data:`sys.meta_path`. -By default ``importlib_metadata`` installs a finder for distribution packages +By default ``importlib.metadata`` installs a finder for distribution packages found on the file system. This finder doesn't actually find any *distributions*, but it can find their metadata. diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 4c6aa59bf9f58f..755693840fecd8 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -1,5 +1,5 @@ -:mod:`importlib.resources` -- Resources ---------------------------------------- +:mod:`importlib.resources` -- Package resource reading, opening and access +-------------------------------------------------------------------------- .. module:: importlib.resources :synopsis: Package resource reading, opening, and access @@ -97,7 +97,7 @@ for example, a package and its resources can be imported from a zip file using Deprecated functions --------------------- +^^^^^^^^^^^^^^^^^^^^ An older, deprecated set of functions is still available, but is scheduled for removal in a future version of Python. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 14118127835bbe..5ffa33d4e61f19 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -530,10 +530,10 @@ Pure paths provide the following methods and properties: unintended effects. -.. method:: PurePath.joinpath(*other) +.. method:: PurePath.joinpath(*pathsegments) Calling this method is equivalent to combining the path with each of - the *other* arguments in turn:: + the given *pathsegments* in turn:: >>> PurePosixPath('/etc').joinpath('passwd') PurePosixPath('/etc/passwd') @@ -680,6 +680,30 @@ Pure paths provide the following methods and properties: PureWindowsPath('README') +.. method:: PurePath.with_segments(*pathsegments) + + Create a new path object of the same type by combining the given + *pathsegments*. This method is called whenever a derivative path is created, + such as from :attr:`parent` and :meth:`relative_to`. Subclasses may + override this method to pass information to derivative paths, for example:: + + from pathlib import PurePosixPath + + class MyPath(PurePosixPath): + def __init__(self, *pathsegments, session_id): + super().__init__(*pathsegments) + self.session_id = session_id + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, session_id=self.session_id) + + etc = MyPath('/etc', session_id=42) + hosts = etc / 'hosts' + print(hosts.session_id) # 42 + + .. versionadded:: 3.12 + + .. _concrete-paths: diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index 4e29192311fc21..d8269ef48cb36a 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -159,7 +159,7 @@ The :mod:`pprint` module defines one class: .. function:: isreadable(object) - .. index:: builtin: eval + .. index:: pair: built-in function; eval Determine if the formatted representation of *object* is "readable", or can be used to reconstruct the value using :func:`eval`. This always returns ``False`` @@ -218,7 +218,7 @@ created. .. method:: PrettyPrinter.isreadable(object) - .. index:: builtin: eval + .. index:: pair: built-in function; eval Determine if the formatted representation of the object is "readable," or can be used to reconstruct the value using :func:`eval`. Note that this returns diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c739485c5564da..9203afbf6a4e8a 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -244,9 +244,9 @@ and imaginary parts. .. index:: single: arithmetic - builtin: int - builtin: float - builtin: complex + pair: built-in function; int + pair: built-in function; float + pair: built-in function; complex single: operator; + (plus) single: + (plus); unary operator single: + (plus); binary operator @@ -945,9 +945,9 @@ operations have the same priority as the corresponding numeric operations. [3]_ .. index:: triple: operations on; sequence; types - builtin: len - builtin: min - builtin: max + pair: built-in function; len + pair: built-in function; min + pair: built-in function; max pair: concatenation; operation pair: repetition; operation pair: subscript; operation @@ -1113,7 +1113,7 @@ Immutable Sequence Types .. index:: triple: immutable; sequence; types pair: object; tuple - builtin: hash + pair: built-in function; hash The only operation that immutable sequence types generally implement that is not also implemented by mutable sequence types is support for the :func:`hash` @@ -4419,7 +4419,7 @@ Mapping Types --- :class:`dict` triple: operations on; mapping; types triple: operations on; dictionary; type pair: statement; del - builtin: len + pair: built-in function; len A :term:`mapping` object maps :term:`hashable` values to arbitrary objects. Mappings are mutable objects. There is currently only one standard mapping @@ -5348,7 +5348,7 @@ Code Objects ------------ .. index:: - builtin: compile + pair: built-in function; compile single: __code__ (function object attribute) Code objects are used by the implementation to represent "pseudo-compiled" @@ -5362,8 +5362,8 @@ Accessing ``__code__`` raises an :ref:`auditing event ` ``object.__getattr__`` with arguments ``obj`` and ``"__code__"``. .. index:: - builtin: exec - builtin: eval + pair: built-in function; exec + pair: built-in function; eval A code object can be executed or evaluated by passing it (instead of a source string) to the :func:`exec` or :func:`eval` built-in functions. @@ -5377,7 +5377,7 @@ Type Objects ------------ .. index:: - builtin: type + pair: built-in function; type pair: module; types Type objects represent the various object types. An object's type is accessed diff --git a/Doc/library/types.rst b/Doc/library/types.rst index a15fb5cfa49473..8cbe17df16f107 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -186,7 +186,7 @@ Standard names are defined for the following types: .. class:: CodeType(**kwargs) - .. index:: builtin: compile + .. index:: pair: built-in function; compile The type for code objects such as returned by :func:`compile`. diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 162041fc7a846e..ebab1389f07e58 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2130,15 +2130,10 @@ Corresponding to collections in :mod:`collections.abc` .. class:: ByteString(Sequence[int]) - A generic version of :class:`collections.abc.ByteString`. - This type represents the types :class:`bytes`, :class:`bytearray`, and :class:`memoryview` of byte sequences. - As a shorthand for this type, :class:`bytes` can be used to - annotate arguments of any of the types mentioned above. - - .. deprecated:: 3.9 + .. deprecated-removed:: 3.9 3.14 Prefer :class:`collections.abc.Buffer`, or a union like ``bytes | bytearray | memoryview``. .. class:: Collection(Sized, Iterable[T_co], Container[T_co]) @@ -2977,6 +2972,8 @@ convenience. This is subject to change, and not all deprecations are listed. | ``typing`` versions of standard | 3.9 | Undecided | :pep:`585` | | collections | | | | +----------------------------------+---------------+-------------------+----------------+ +| ``typing.ByteString`` | 3.9 | 3.14 | :gh:`91896` | ++----------------------------------+---------------+-------------------+----------------+ | ``typing.Text`` | 3.11 | Undecided | :gh:`92332` | +----------------------------------+---------------+-------------------+----------------+ | ``typing.Hashable`` and | 3.12 | Undecided | :gh:`94309` | diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index e9c1c493ae4278..9d1e5b6c596d9f 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -188,7 +188,7 @@ those made in the suite of the for-loop:: .. index:: - builtin: range + pair: built-in function; range Names in the target list are not deleted when the loop is finished, but if the sequence is empty, they will not have been assigned to at all by the loop. Hint: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 0a9cabc158b9e4..c0734e49f29192 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -21,8 +21,8 @@ conformance to Von Neumann's model of a "stored program computer", code is also represented by objects.) .. index:: - builtin: id - builtin: type + pair: built-in function; id + pair: built-in function; type single: identity of an object single: value of an object single: type of an object @@ -267,7 +267,7 @@ Ellipsis Sequences .. index:: - builtin: len + pair: built-in function; len pair: object; sequence single: index operation single: item selection @@ -308,8 +308,8 @@ Sequences Strings .. index:: - builtin: chr - builtin: ord + pair: built-in function; chr + pair: built-in function; ord single: character single: integer single: Unicode @@ -384,7 +384,7 @@ Sequences Set types .. index:: - builtin: len + pair: built-in function; len pair: object; set type These represent unordered, finite sets of unique, immutable objects. As such, @@ -418,7 +418,7 @@ Set types Mappings .. index:: - builtin: len + pair: built-in function; len single: subscription pair: object; mapping @@ -908,7 +908,7 @@ Class instances I/O objects (also known as file objects) .. index:: - builtin: open + pair: built-in function; open pair: module; io single: popen() (in module os) single: makefile() (socket method) @@ -1177,7 +1177,7 @@ Internal types and the ``tb_next`` attribute of existing instances can be updated. Slice objects - .. index:: builtin: slice + .. index:: pair: built-in function; slice Slice objects are used to represent slices for :meth:`~object.__getitem__` @@ -1411,7 +1411,7 @@ Basic customization .. method:: object.__bytes__(self) - .. index:: builtin: bytes + .. index:: pair: built-in function; bytes Called by :ref:`bytes ` to compute a byte-string representation of an object. This should return a :class:`bytes` object. @@ -1419,7 +1419,7 @@ Basic customization .. index:: single: string; __format__() (object method) pair: string; conversion - builtin: print + pair: built-in function; print .. method:: object.__format__(self, format_spec) @@ -1499,7 +1499,7 @@ Basic customization .. index:: pair: object; dictionary - builtin: hash + pair: built-in function; hash Called by built-in function :func:`hash` and for operations on members of hashed collections including :class:`set`, :class:`frozenset`, and @@ -2050,7 +2050,7 @@ Metaclasses .. index:: single: metaclass - builtin: type + pair: built-in function; type single: = (equals); class definition By default, classes are constructed using :func:`type`. The class body is @@ -2477,7 +2477,7 @@ through the object's keys; for sequences, it should iterate through the values. .. method:: object.__len__(self) .. index:: - builtin: len + pair: built-in function; len single: __bool__() (object method) Called to implement the built-in function :func:`len`. Should return the length @@ -2635,9 +2635,9 @@ left undefined. object.__or__(self, other) .. index:: - builtin: divmod - builtin: pow - builtin: pow + pair: built-in function; divmod + pair: built-in function; pow + pair: built-in function; pow These methods are called to implement the binary arithmetic operations (``+``, ``-``, ``*``, ``@``, ``/``, ``//``, ``%``, :func:`divmod`, @@ -2670,8 +2670,8 @@ left undefined. object.__ror__(self, other) .. index:: - builtin: divmod - builtin: pow + pair: built-in function; divmod + pair: built-in function; pow These methods are called to implement the binary arithmetic operations (``+``, ``-``, ``*``, ``@``, ``/``, ``//``, ``%``, :func:`divmod`, @@ -2683,7 +2683,7 @@ left undefined. ``type(y).__rsub__(y, x)`` is called if ``type(x).__sub__(x, y)`` returns *NotImplemented*. - .. index:: builtin: pow + .. index:: pair: built-in function; pow Note that ternary :func:`pow` will not try calling :meth:`__rpow__` (the coercion rules would become too complicated). @@ -2730,7 +2730,7 @@ left undefined. object.__abs__(self) object.__invert__(self) - .. index:: builtin: abs + .. index:: pair: built-in function; abs Called to implement the unary arithmetic operations (``-``, ``+``, :func:`abs` and ``~``). @@ -2741,9 +2741,9 @@ left undefined. object.__float__(self) .. index:: - builtin: complex - builtin: int - builtin: float + pair: built-in function; complex + pair: built-in function; int + pair: built-in function; float Called to implement the built-in functions :func:`complex`, :func:`int` and :func:`float`. Should return a value @@ -2768,7 +2768,7 @@ left undefined. object.__floor__(self) object.__ceil__(self) - .. index:: builtin: round + .. index:: pair: built-in function; round Called to implement the built-in function :func:`round` and :mod:`math` functions :func:`~math.trunc`, :func:`~math.floor` and :func:`~math.ceil`. diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index ca1aa744389bda..f7a8b44d195417 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -53,7 +53,7 @@ An expression statement evaluates the expression list (which may be a single expression). .. index:: - builtin: repr + pair: built-in function; repr pair: object; None pair: string; conversion single: output @@ -970,9 +970,9 @@ annotation. them or silently change the meaning of the program. .. index:: - builtin: exec - builtin: eval - builtin: compile + pair: built-in function; exec + pair: built-in function; eval + pair: built-in function; compile **Programmer's note:** :keyword:`global` is a directive to the parser. It applies only to code parsed at the same time as the :keyword:`!global` statement. diff --git a/Doc/reference/toplevel_components.rst b/Doc/reference/toplevel_components.rst index ee472ace6e2129..dd3d3d6878e289 100644 --- a/Doc/reference/toplevel_components.rst +++ b/Doc/reference/toplevel_components.rst @@ -98,7 +98,7 @@ Expression input ================ .. index:: single: input -.. index:: builtin: eval +.. index:: pair: built-in function; eval :func:`eval` is used for expression input. It ignores leading whitespace. The string argument to :func:`eval` must have the following form: diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 8fc44cab44455a..cd8d9febb0d13b 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -674,7 +674,14 @@ def process_audit_events(app, doctree, fromdocname): node.replace_self(table) -def patch_pairindextypes(app) -> None: +def patch_pairindextypes(app, _env) -> None: + """Remove all entries from ``pairindextypes`` before writing POT files. + + We want to run this just before writing output files, as the check to + circumvent is in ``I18nBuilder.write_doc()``. + As such, we link this to ``env-check-consistency``, even though it has + nothing to do with the environment consistency check. + """ if app.builder.name != 'gettext': return @@ -688,14 +695,7 @@ def patch_pairindextypes(app) -> None: # the Sphinx-translated pairindextypes values. As we intend to move # away from this, we need Sphinx to believe that these values don't # exist, by deleting them when using the gettext builder. - - pairindextypes.pop('module', None) - pairindextypes.pop('keyword', None) - pairindextypes.pop('operator', None) - pairindextypes.pop('object', None) - pairindextypes.pop('exception', None) - pairindextypes.pop('statement', None) - # pairindextypes.pop('builtin', None) + pairindextypes.clear() def setup(app): @@ -719,7 +719,7 @@ def setup(app): app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod) app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) app.add_directive('miscnews', MiscNews) - app.connect('builder-inited', patch_pairindextypes) + app.connect('env-check-consistency', patch_pairindextypes) app.connect('doctree-resolved', process_audit_events) app.connect('env-merge-info', audit_events_merge) app.connect('env-purge-doc', audit_events_purge) diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 3fcf4e3f43a300..f5cdd84cbadefe 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -285,7 +285,7 @@ Reading and Writing Files ========================= .. index:: - builtin: open + pair: built-in function; open pair: object; file :func:`open` returns a :term:`file object`, and is most commonly used with diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 4f5ada90eb57bc..6bae279c5e9cde 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -24,7 +24,7 @@ Be sure to use the ``import os`` style instead of ``from os import *``. This will keep :func:`os.open` from shadowing the built-in :func:`open` function which operates much differently. -.. index:: builtin: help +.. index:: pair: built-in function; help The built-in :func:`dir` and :func:`help` functions are useful as interactive aids for working with large modules like :mod:`os`:: diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 4f952e2a37ef2f..65b3e9ffb8072d 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -342,12 +342,18 @@ inspect (Contributed by Thomas Krennwallner in :issue:`35759`.) * The performance of :func:`inspect.getattr_static` has been considerably - improved. Most calls to the function should be around 2x faster than they - were in Python 3.11. (Contributed by Alex Waygood in :gh:`103193`.) + improved. Most calls to the function should be at least 2x faster than they + were in Python 3.11, and some may be 6x faster or more. (Contributed by Alex + Waygood in :gh:`103193`.) pathlib ------- +* Add support for subclassing :class:`pathlib.PurePath` and + :class:`~pathlib.Path`, plus their Posix- and Windows-specific variants. + Subclasses may override the :meth:`~pathlib.PurePath.with_segments` method + to pass information between path instances. + * Add :meth:`~pathlib.Path.walk` for walking the directory trees and generating all file or directory names within them, similar to :func:`os.walk`. (Contributed by Stanislav Zmiev in :gh:`90385`.) @@ -592,7 +598,7 @@ typing :func:`runtime-checkable protocols ` has changed significantly. Most ``isinstance()`` checks against protocols with only a few members should be at least 2x faster than in 3.11, and some may be 20x - faster or more. However, ``isinstance()`` checks against protocols with seven + faster or more. However, ``isinstance()`` checks against protocols with fourteen or more members may be slower than in Python 3.11. (Contributed by Alex Waygood in :gh:`74690` and :gh:`103193`.) @@ -839,6 +845,19 @@ Pending Removal in Python 3.14 use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) +* The following :mod:`ast` features have been deprecated in documentation since + Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at runtime + when they are accessed or used, and will be removed in Python 3.14: + + * :class:`!ast.Num` + * :class:`!ast.Str` + * :class:`!ast.Bytes` + * :class:`!ast.NameConstant` + * :class:`!ast.Ellipsis` + + Use :class:`ast.Constant` instead. + (Contributed by Serhiy Storchaka in :gh:`90953`.) + Pending Removal in Future Versions ---------------------------------- diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 4e19535c656f2c..6f3efe36ede5d8 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -4,6 +4,8 @@ # error "this header file must not be included directly" #endif +struct _PyInterpreterFrame; + /* Standard object interface */ PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *, @@ -27,3 +29,18 @@ PyAPI_FUNC(int) _PyFrame_IsEntryFrame(PyFrameObject *frame); PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); + +/* The following functions are for use by debuggers and other tools + * implementing custom frame evaluators with PEP 523. */ + +/* Returns the code object of the frame (strong reference). + * Does not raise an exception. */ +PyAPI_FUNC(PyCodeObject *) PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame); + +/* Returns a byte ofsset into the last executed instruction. + * Does not raise an exception. */ +PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame); + +/* Returns the currently executing line number, or -1 if there is no line number. + * Does not raise an exception. */ +PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame); diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 9c1783d272f1cd..efae2409b50069 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -252,6 +252,7 @@ typedef struct { int allow_threads; int allow_daemon_threads; int check_multi_interp_extensions; + int own_gil; } PyInterpreterConfig; #define _PyInterpreterConfig_INIT \ @@ -262,6 +263,7 @@ typedef struct { .allow_threads = 1, \ .allow_daemon_threads = 0, \ .check_multi_interp_extensions = 1, \ + .own_gil = 1, \ } #define _PyInterpreterConfig_LEGACY_INIT \ @@ -272,6 +274,7 @@ typedef struct { .allow_threads = 1, \ .allow_daemon_threads = 1, \ .check_multi_interp_extensions = 0, \ + .own_gil = 0, \ } /* --- Helper functions --------------------------------------- */ diff --git a/Include/cpython/memoryobject.h b/Include/cpython/memoryobject.h index deab3cc89f726e..3837fa8c6ab5aa 100644 --- a/Include/cpython/memoryobject.h +++ b/Include/cpython/memoryobject.h @@ -24,6 +24,7 @@ typedef struct { #define _Py_MEMORYVIEW_FORTRAN 0x004 /* Fortran contiguous layout */ #define _Py_MEMORYVIEW_SCALAR 0x008 /* scalar: ndim = 0 */ #define _Py_MEMORYVIEW_PIL 0x010 /* PIL-style layout */ +#define _Py_MEMORYVIEW_RESTRICTED 0x020 /* Disallow new references to the memoryview's buffer */ typedef struct { PyObject_VAR_HEAD diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index deda070a6dea79..9fd8571cbc87f4 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -96,11 +96,13 @@ _PyEval_Vector(PyThreadState *tstate, PyObject* const* args, size_t argcount, PyObject *kwnames); -extern int _PyEval_ThreadsInitialized(struct pyruntimestate *runtime); -extern PyStatus _PyEval_InitGIL(PyThreadState *tstate); +extern int _PyEval_ThreadsInitialized(void); +extern PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil); extern void _PyEval_FiniGIL(PyInterpreterState *interp); +extern void _PyEval_AcquireLock(PyThreadState *tstate); extern void _PyEval_ReleaseLock(PyThreadState *tstate); +extern PyThreadState * _PyThreadState_SwapNoGIL(PyThreadState *); extern void _PyEval_DeactivateOpCache(void); diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 9ba42eb03b2676..4781dd5735dcf6 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -49,6 +49,8 @@ struct _ceval_runtime_state { the main thread of the main interpreter can handle signals: see _Py_ThreadCanHandleSignals(). */ _Py_atomic_int signals_pending; + + /* This is (only) used indirectly through PyInterpreterState.ceval.gil. */ struct _gil_runtime_state gil; }; @@ -83,6 +85,8 @@ struct _pending_calls { struct _ceval_state { int recursion_limit; + struct _gil_runtime_state *gil; + int own_gil; /* This single variable consolidates all requests to break out of the fast path in the eval loop. */ _Py_atomic_int eval_breaker; diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 4bd4ef57238f98..d2b12c91fe7a00 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -97,7 +97,8 @@ PyAPI_FUNC(PyObject*) _PyCompile_CodeGen( PyObject *ast, PyObject *filename, PyCompilerFlags *flags, - int optimize); + int optimize, + int compile_mode); PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( PyObject *instructions, diff --git a/Include/internal/pycore_format.h b/Include/internal/pycore_format.h index 1899609e77ef20..1b8d57539ca505 100644 --- a/Include/internal/pycore_format.h +++ b/Include/internal/pycore_format.h @@ -14,14 +14,12 @@ extern "C" { * F_BLANK ' ' * F_ALT '#' * F_ZERO '0' - * F_NO_NEG_0 'z' */ #define F_LJUST (1<<0) #define F_SIGN (1<<1) #define F_BLANK (1<<2) #define F_ALT (1<<3) #define F_ZERO (1<<4) -#define F_NO_NEG_0 (1<<5) #ifdef __cplusplus } diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index d8d7fe9ef2ebde..3d3cbbff7aae81 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -265,8 +265,6 @@ _PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_l return new_frame; } -int _PyInterpreterFrame_GetLine(_PyInterpreterFrame *frame); - static inline PyGenObject *_PyFrame_GetGenerator(_PyInterpreterFrame *frame) { diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9377fd8526e3a2..7e495817981f06 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -847,6 +847,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(command)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(comment_factory)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compile_mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(consts)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(context)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cookie)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index ed9b2bb44ddffc..8ebfee85c87c23 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -335,6 +335,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(code) STRUCT_FOR_ID(command) STRUCT_FOR_ID(comment_factory) + STRUCT_FOR_ID(compile_mode) STRUCT_FOR_ID(consts) STRUCT_FOR_ID(context) STRUCT_FOR_ID(cookie) diff --git a/Include/internal/pycore_memoryobject.h b/Include/internal/pycore_memoryobject.h index acc12c9275172c..fe19e3f9611a16 100644 --- a/Include/internal/pycore_memoryobject.h +++ b/Include/internal/pycore_memoryobject.h @@ -9,7 +9,8 @@ extern "C" { #endif PyObject * -PyMemoryView_FromObjectAndFlags(PyObject *v, int flags); +_PyMemoryView_FromBufferProc(PyObject *v, int flags, + getbufferproc bufferproc); #ifdef __cplusplus } diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 6ade8fb6eade03..7b9c73dd1edf3b 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -841,6 +841,7 @@ extern "C" { INIT_ID(code), \ INIT_ID(command), \ INIT_ID(comment_factory), \ + INIT_ID(compile_mode), \ INIT_ID(consts), \ INIT_ID(context), \ INIT_ID(cookie), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 0b33ea187e60ff..8e086edbdf8193 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -858,6 +858,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(comment_factory); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(compile_mode); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(consts); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 555564ec73b4a2..b8bdfe29d80406 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -78,11 +78,17 @@ struct PyModuleDef_Slot { #define Py_mod_create 1 #define Py_mod_exec 2 +#define Py_mod_multiple_interpreters 3 #ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 2 +#define _Py_mod_LAST_SLOT 3 #endif +/* for Py_mod_multiple_interpreters: */ +#define Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED ((void *)0) +#define Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED ((void *)1) +#define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED ((void *)2) + #endif /* New in 3.5 */ struct PyModuleDef { diff --git a/Lib/argparse.py b/Lib/argparse.py index a819d2650e85f0..f5f44ff02c0d38 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -345,21 +345,22 @@ def _format_usage(self, usage, actions, groups, prefix): def get_lines(parts, indent, prefix=None): lines = [] line = [] + indent_length = len(indent) if prefix is not None: line_len = len(prefix) - 1 else: - line_len = len(indent) - 1 + line_len = indent_length - 1 for part in parts: if line_len + 1 + len(part) > text_width and line: lines.append(indent + ' '.join(line)) line = [] - line_len = len(indent) - 1 + line_len = indent_length - 1 line.append(part) line_len += len(part) + 1 if line: lines.append(indent + ' '.join(line)) if prefix is not None: - lines[0] = lines[0][len(indent):] + lines[0] = lines[0][indent_length:] return lines # if prog is short, follow it with optionals or positionals @@ -2605,9 +2606,11 @@ def print_help(self, file=None): def _print_message(self, message, file=None): if message: - if file is None: - file = _sys.stderr - file.write(message) + file = file or _sys.stderr + try: + file.write(message) + except (AttributeError, OSError): + pass # =============== # Exiting methods diff --git a/Lib/ast.py b/Lib/ast.py index d9733a79d3a78f..65152047a22370 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -294,9 +294,7 @@ def get_docstring(node, clean=True): if not(node.body and isinstance(node.body[0], Expr)): return None node = node.body[0].value - if isinstance(node, Str): - text = node.s - elif isinstance(node, Constant) and isinstance(node.value, str): + if isinstance(node, Constant) and isinstance(node.value, str): text = node.value else: return None @@ -499,20 +497,52 @@ def generic_visit(self, node): return node +_DEPRECATED_VALUE_ALIAS_MESSAGE = ( + "{name} is deprecated and will be removed in Python {remove}; use value instead" +) +_DEPRECATED_CLASS_MESSAGE = ( + "{name} is deprecated and will be removed in Python {remove}; " + "use ast.Constant instead" +) + + # If the ast module is loaded more than once, only add deprecated methods once if not hasattr(Constant, 'n'): # The following code is for backward compatibility. # It will be removed in future. - def _getter(self): + def _n_getter(self): + """Deprecated. Use value instead.""" + import warnings + warnings._deprecated( + "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) + ) + return self.value + + def _n_setter(self, value): + import warnings + warnings._deprecated( + "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) + ) + self.value = value + + def _s_getter(self): """Deprecated. Use value instead.""" + import warnings + warnings._deprecated( + "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) + ) return self.value - def _setter(self, value): + def _s_setter(self, value): + import warnings + warnings._deprecated( + "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) + ) self.value = value - Constant.n = property(_getter, _setter) - Constant.s = property(_getter, _setter) + Constant.n = property(_n_getter, _n_setter) + Constant.s = property(_s_getter, _s_setter) class _ABC(type): @@ -520,6 +550,13 @@ def __init__(cls, *args): cls.__doc__ = """Deprecated AST node class. Use ast.Constant instead""" def __instancecheck__(cls, inst): + if cls in _const_types: + import warnings + warnings._deprecated( + f"ast.{cls.__qualname__}", + message=_DEPRECATED_CLASS_MESSAGE, + remove=(3, 14) + ) if not isinstance(inst, Constant): return False if cls in _const_types: @@ -543,6 +580,10 @@ def _new(cls, *args, **kwargs): if pos < len(args): raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}") if cls in _const_types: + import warnings + warnings._deprecated( + f"ast.{cls.__qualname__}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) + ) return Constant(*args, **kwargs) return Constant.__new__(cls, *args, **kwargs) @@ -565,10 +606,19 @@ class Ellipsis(Constant, metaclass=_ABC): _fields = () def __new__(cls, *args, **kwargs): - if cls is Ellipsis: + if cls is _ast_Ellipsis: + import warnings + warnings._deprecated( + "ast.Ellipsis", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) + ) return Constant(..., *args, **kwargs) return Constant.__new__(cls, *args, **kwargs) +# Keep another reference to Ellipsis in the global namespace +# so it can be referenced in Ellipsis.__new__ +# (The original "Ellipsis" name is removed from the global namespace later on) +_ast_Ellipsis = Ellipsis + _const_types = { Num: (int, float, complex), Str: (str,), @@ -1699,6 +1749,22 @@ def unparse(ast_obj): return unparser.visit(ast_obj) +_deprecated_globals = { + name: globals().pop(name) + for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis') +} + +def __getattr__(name): + if name in _deprecated_globals: + globals()[name] = value = _deprecated_globals[name] + import warnings + warnings._deprecated( + f"ast.{name}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) + ) + return value + raise AttributeError(f"module 'ast' has no attribute '{name}'") + + def main(): import argparse diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 0fdea3697ece3d..06b2e0db86a1fe 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -164,8 +164,14 @@ def create_task(self, coro, *, name=None, context=None): else: task = self._loop.create_task(coro, context=context) tasks._set_task_name(task, name) - task.add_done_callback(self._on_task_done) - self._tasks.add(task) + # optimization: Immediately call the done callback if the task is + # already done (e.g. if the coro was able to complete eagerly), + # and skip scheduling a done callback + if task.done(): + self._on_task_done(task) + else: + self._tasks.add(task) + task.add_done_callback(self._on_task_done) return task # Since Python 3.8 Tasks propagate all exceptions correctly, diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index aa5269ade19a7f..7eb647bd129819 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -813,6 +813,7 @@ def _done_callback(fut): children = [] nfuts = 0 nfinished = 0 + done_futs = [] loop = None outer = None # bpo-46672 for arg in coros_or_futures: @@ -829,7 +830,10 @@ def _done_callback(fut): nfuts += 1 arg_to_fut[arg] = fut - fut.add_done_callback(_done_callback) + if fut.done(): + done_futs.append(fut) + else: + fut.add_done_callback(_done_callback) else: # There's a duplicate Future object in coros_or_futures. @@ -838,6 +842,13 @@ def _done_callback(fut): children.append(fut) outer = _GatheringFuture(children, loop=loop) + # Run done callbacks after GatheringFuture created so any post-processing + # can be performed at this point + # optimization: in the special case that *all* futures finished eagerly, + # this will effectively complete the gather eagerly, with the last + # callback setting the result (or exception) on outer before returning it + for fut in done_futs: + _done_callback(fut) return outer diff --git a/Lib/gzip.py b/Lib/gzip.py index 75c6ddc3f2cffb..8796c8d9fd9a2d 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -22,6 +22,7 @@ _COMPRESS_LEVEL_BEST = 9 READ_BUFFER_SIZE = 128 * 1024 +_WRITE_BUFFER_SIZE = 4 * io.DEFAULT_BUFFER_SIZE def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST, @@ -120,6 +121,21 @@ class BadGzipFile(OSError): """Exception raised in some cases for invalid gzip files.""" +class _WriteBufferStream(io.RawIOBase): + """Minimal object to pass WriteBuffer flushes into GzipFile""" + def __init__(self, gzip_file): + self.gzip_file = gzip_file + + def write(self, data): + return self.gzip_file._write_raw(data) + + def seekable(self): + return False + + def writable(self): + return True + + class GzipFile(_compression.BaseStream): """The GzipFile class simulates most of the methods of a file object with the exception of the truncate() method. @@ -184,6 +200,7 @@ def __init__(self, filename=None, mode=None, if mode is None: mode = getattr(fileobj, 'mode', 'rb') + if mode.startswith('r'): self.mode = READ raw = _GzipReader(fileobj) @@ -206,6 +223,9 @@ def __init__(self, filename=None, mode=None, zlib.DEF_MEM_LEVEL, 0) self._write_mtime = mtime + self._buffer_size = _WRITE_BUFFER_SIZE + self._buffer = io.BufferedWriter(_WriteBufferStream(self), + buffer_size=self._buffer_size) else: raise ValueError("Invalid mode: {!r}".format(mode)) @@ -231,6 +251,11 @@ def _init_write(self, filename): self.bufsize = 0 self.offset = 0 # Current file offset for seek(), tell(), etc + def tell(self): + self._check_not_closed() + self._buffer.flush() + return super().tell() + def _write_gzip_header(self, compresslevel): self.fileobj.write(b'\037\213') # magic header self.fileobj.write(b'\010') # compression method @@ -272,6 +297,10 @@ def write(self,data): if self.fileobj is None: raise ValueError("write() on closed GzipFile object") + return self._buffer.write(data) + + def _write_raw(self, data): + # Called by our self._buffer underlying WriteBufferStream. if isinstance(data, (bytes, bytearray)): length = len(data) else: @@ -322,9 +351,9 @@ def close(self): fileobj = self.fileobj if fileobj is None: return - self.fileobj = None try: if self.mode == WRITE: + self._buffer.flush() fileobj.write(self.compress.flush()) write32u(fileobj, self.crc) # self.size may exceed 2 GiB, or even 4 GiB @@ -332,6 +361,7 @@ def close(self): elif self.mode == READ: self._buffer.close() finally: + self.fileobj = None myfileobj = self.myfileobj if myfileobj: self.myfileobj = None @@ -341,7 +371,7 @@ def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH): self._check_not_closed() if self.mode == WRITE: # Ensure the compressor's buffer is flushed - self.fileobj.write(self.compress.flush(zlib_mode)) + self._buffer.flush() self.fileobj.flush() def fileno(self): @@ -378,10 +408,10 @@ def seek(self, offset, whence=io.SEEK_SET): if offset < self.offset: raise OSError('Negative seek in write mode') count = offset - self.offset - chunk = b'\0' * 1024 - for i in range(count // 1024): + chunk = b'\0' * self._buffer_size + for i in range(count // self._buffer_size): self.write(chunk) - self.write(b'\0' * (count % 1024)) + self.write(b'\0' * (count % self._buffer_size)) elif self.mode == READ: self._check_not_closed() return self._buffer.seek(offset, whence) diff --git a/Lib/http/client.py b/Lib/http/client.py index 74f7bcb68fb6bc..50f2b4680769c8 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -858,6 +858,7 @@ def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, self._tunnel_host = None self._tunnel_port = None self._tunnel_headers = {} + self._proxy_response_headers = None (self.host, self.port) = self._get_hostport(host, port) @@ -944,21 +945,16 @@ def _tunnel(self): try: (version, code, message) = response._read_status() + self._proxy_response_headers = parse_headers(response.fp) + + if self.debuglevel > 0: + for hdr, val in self._proxy_response_headers.items(): + print("header:", hdr + ":", val) + if code != http.HTTPStatus.OK: self.close() raise OSError(f"Tunnel connection failed: {code} {message.strip()}") - while True: - line = response.fp.readline(_MAXLINE + 1) - if len(line) > _MAXLINE: - raise LineTooLong("header line") - if not line: - # for sites which EOF without sending a trailer - break - if line in (b'\r\n', b'\n', b''): - break - if self.debuglevel > 0: - print('header:', line.decode()) finally: response.close() diff --git a/Lib/inspect.py b/Lib/inspect.py index 95da7fb71a3997..a64e85e4fd67a4 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1794,8 +1794,9 @@ def _check_class(klass, attr): return entry.__dict__[attr] return _sentinel -def _shadowed_dict(klass): - for entry in _static_getmro(klass): +@functools.lru_cache() +def _shadowed_dict_from_mro_tuple(mro): + for entry in mro: dunder_dict = _get_dunder_dict_of_class(entry) if '__dict__' in dunder_dict: class_dict = dunder_dict['__dict__'] @@ -1805,6 +1806,9 @@ def _shadowed_dict(klass): return class_dict return _sentinel +def _shadowed_dict(klass): + return _shadowed_dict_from_mro_tuple(_static_getmro(klass)) + def getattr_static(obj, attr, default=_sentinel): """Retrieve attributes without triggering dynamic lookup via the descriptor protocol, __getattr__ or __getattribute__. diff --git a/Lib/pathlib.py b/Lib/pathlib.py index f32e1e2d822834..20ec1ce9d80374 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -64,17 +64,25 @@ def _is_case_sensitive(flavour): @functools.lru_cache() def _make_selector(pattern_parts, flavour, case_sensitive): pat = pattern_parts[0] - child_parts = pattern_parts[1:] if not pat: return _TerminatingSelector() if pat == '**': - cls = _RecursiveWildcardSelector - elif pat == '..': - cls = _ParentSelector - elif '**' in pat: - raise ValueError("Invalid pattern: '**' can only be an entire path component") + child_parts_idx = 1 + while child_parts_idx < len(pattern_parts) and pattern_parts[child_parts_idx] == '**': + child_parts_idx += 1 + child_parts = pattern_parts[child_parts_idx:] + if '**' in child_parts: + cls = _DoubleRecursiveWildcardSelector + else: + cls = _RecursiveWildcardSelector else: - cls = _WildcardSelector + child_parts = pattern_parts[1:] + if pat == '..': + cls = _ParentSelector + elif '**' in pat: + raise ValueError("Invalid pattern: '**' can only be an entire path component") + else: + cls = _WildcardSelector return cls(pat, child_parts, flavour, case_sensitive) @@ -183,20 +191,32 @@ def _iterate_directories(self, parent_path, scandir): def _select_from(self, parent_path, scandir): try: - yielded = set() - try: - successor_select = self.successor._select_from - for starting_point in self._iterate_directories(parent_path, scandir): - for p in successor_select(starting_point, scandir): - if p not in yielded: - yield p - yielded.add(p) - finally: - yielded.clear() + successor_select = self.successor._select_from + for starting_point in self._iterate_directories(parent_path, scandir): + for p in successor_select(starting_point, scandir): + yield p except PermissionError: return +class _DoubleRecursiveWildcardSelector(_RecursiveWildcardSelector): + """ + Like _RecursiveWildcardSelector, but also de-duplicates results from + successive selectors. This is necessary if the pattern contains + multiple non-adjacent '**' segments. + """ + + def _select_from(self, parent_path, scandir): + yielded = set() + try: + for p in super()._select_from(parent_path, scandir): + if p not in yielded: + yield p + yielded.add(p) + finally: + yielded.clear() + + # # Public API # @@ -204,11 +224,10 @@ def _select_from(self, parent_path, scandir): class _PathParents(Sequence): """This object provides sequence-like access to the logical ancestors of a path. Don't try to construct it yourself.""" - __slots__ = ('_pathcls', '_drv', '_root', '_tail') + __slots__ = ('_path', '_drv', '_root', '_tail') def __init__(self, path): - # We don't store the instance to avoid reference cycles - self._pathcls = type(path) + self._path = path self._drv = path.drive self._root = path.root self._tail = path._tail @@ -224,11 +243,11 @@ def __getitem__(self, idx): raise IndexError(idx) if idx < 0: idx += len(self) - return self._pathcls._from_parsed_parts(self._drv, self._root, - self._tail[:-idx - 1]) + return self._path._from_parsed_parts(self._drv, self._root, + self._tail[:-idx - 1]) def __repr__(self): - return "<{}.parents>".format(self._pathcls.__name__) + return "<{}.parents>".format(type(self._path).__name__) class PurePath(object): @@ -316,6 +335,13 @@ def __init__(self, *args): else: self._raw_path = self._flavour.join(*paths) + def with_segments(self, *pathsegments): + """Construct a new path object from any number of path-like objects. + Subclasses may override this method to customize how new path objects + are created from methods like `iterdir()`. + """ + return type(self)(*pathsegments) + @classmethod def _parse_path(cls, path): if not path: @@ -342,15 +368,14 @@ def _load_parts(self): self._root = root self._tail_cached = tail - @classmethod - def _from_parsed_parts(cls, drv, root, tail): - path = cls._format_parsed_parts(drv, root, tail) - self = cls(path) - self._str = path or '.' - self._drv = drv - self._root = root - self._tail_cached = tail - return self + def _from_parsed_parts(self, drv, root, tail): + path_str = self._format_parsed_parts(drv, root, tail) + path = self.with_segments(path_str) + path._str = path_str or '.' + path._drv = drv + path._root = root + path._tail_cached = tail + return path @classmethod def _format_parsed_parts(cls, drv, root, tail): @@ -584,8 +609,7 @@ def relative_to(self, other, /, *_deprecated, walk_up=False): "scheduled for removal in Python {remove}") warnings._deprecated("pathlib.PurePath.relative_to(*args)", msg, remove=(3, 14)) - path_cls = type(self) - other = path_cls(other, *_deprecated) + other = self.with_segments(other, *_deprecated) for step, path in enumerate([other] + list(other.parents)): if self.is_relative_to(path): break @@ -594,7 +618,7 @@ def relative_to(self, other, /, *_deprecated, walk_up=False): if step and not walk_up: raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") parts = ['..'] * step + self._tail[len(path._tail):] - return path_cls(*parts) + return self.with_segments(*parts) def is_relative_to(self, other, /, *_deprecated): """Return True if the path is relative to another path or False. @@ -605,7 +629,7 @@ def is_relative_to(self, other, /, *_deprecated): "scheduled for removal in Python {remove}") warnings._deprecated("pathlib.PurePath.is_relative_to(*args)", msg, remove=(3, 14)) - other = type(self)(other, *_deprecated) + other = self.with_segments(other, *_deprecated) return other == self or other in self.parents @property @@ -617,13 +641,13 @@ def parts(self): else: return tuple(self._tail) - def joinpath(self, *args): + def joinpath(self, *pathsegments): """Combine this path with one or several arguments, and return a new path representing either a subpath (if all arguments are relative paths) or a totally different path (if one of the arguments is anchored). """ - return self.__class__(self, *args) + return self.with_segments(self, *pathsegments) def __truediv__(self, key): try: @@ -633,7 +657,7 @@ def __truediv__(self, key): def __rtruediv__(self, key): try: - return type(self)(key, self) + return self.with_segments(key, self) except TypeError: return NotImplemented @@ -650,6 +674,8 @@ def parent(self): @property def parents(self): """A sequence of this path's logical parents.""" + # The value of this property should not be cached on the path object, + # as doing so would introduce a reference cycle. return _PathParents(self) def is_absolute(self): @@ -658,7 +684,7 @@ def is_absolute(self): # ntpath.isabs() is defective - see GH-44626 . if self._flavour is ntpath: return bool(self.drive and self.root) - return self._flavour.isabs(self) + return self._flavour.isabs(self._raw_path) def is_reserved(self): """Return True if the path contains one of the special names reserved @@ -680,7 +706,7 @@ def match(self, path_pattern): """ Return True if this path matches the given pattern. """ - pat = type(self)(path_pattern) + pat = self.with_segments(path_pattern) if not pat.parts: raise ValueError("empty pattern") pat_parts = pat._parts_normcase @@ -734,211 +760,191 @@ class Path(PurePath): """ __slots__ = () - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) - - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - return object.__new__(cls) - - def _make_child_relpath(self, name): - path_str = str(self) - tail = self._tail - if tail: - path_str = f'{path_str}{self._flavour.sep}{name}' - elif path_str != '.': - path_str = f'{path_str}{name}' - else: - path_str = name - path = type(self)(path_str) - path._str = path_str - path._drv = self.drive - path._root = self.root - path._tail_cached = tail + [name] - return path - - def __enter__(self): - # In previous versions of pathlib, __exit__() marked this path as - # closed; subsequent attempts to perform I/O would raise an IOError. - # This functionality was never documented, and had the effect of - # making Path objects mutable, contrary to PEP 428. - # In Python 3.9 __exit__() was made a no-op. - # In Python 3.11 __enter__() began emitting DeprecationWarning. - # In Python 3.13 __enter__() and __exit__() should be removed. - warnings.warn("pathlib.Path.__enter__() is deprecated and scheduled " - "for removal in Python 3.13; Path objects as a context " - "manager is a no-op", - DeprecationWarning, stacklevel=2) - return self - - def __exit__(self, t, v, tb): - pass - - # Public API - - @classmethod - def cwd(cls): - """Return a new path pointing to the current working directory.""" - # We call 'absolute()' rather than using 'os.getcwd()' directly to - # enable users to replace the implementation of 'absolute()' in a - # subclass and benefit from the new behaviour here. This works because - # os.path.abspath('.') == os.getcwd(). - return cls().absolute() - - @classmethod - def home(cls): - """Return a new path pointing to the user's home directory (as - returned by os.path.expanduser('~')). + def stat(self, *, follow_symlinks=True): """ - return cls("~").expanduser() - - def samefile(self, other_path): - """Return whether other_path is the same or not as this file - (as returned by os.path.samefile()). + Return the result of the stat() system call on this path, like + os.stat() does. """ - st = self.stat() - try: - other_st = other_path.stat() - except AttributeError: - other_st = self.__class__(other_path).stat() - return self._flavour.samestat(st, other_st) - - def iterdir(self): - """Yield path objects of the directory contents. + return os.stat(self, follow_symlinks=follow_symlinks) - The children are yielded in arbitrary order, and the - special entries '.' and '..' are not included. + def lstat(self): """ - for name in os.listdir(self): - yield self._make_child_relpath(name) - - def _scandir(self): - # bpo-24132: a future version of pathlib will support subclassing of - # pathlib.Path to customize how the filesystem is accessed. This - # includes scandir(), which is used to implement glob(). - return os.scandir(self) - - def glob(self, pattern, *, case_sensitive=None): - """Iterate over this subtree and yield all existing files (of any - kind, including directories) matching the given relative pattern. + Like stat(), except if the path points to a symlink, the symlink's + status information is returned, rather than its target's. """ - sys.audit("pathlib.Path.glob", self, pattern) - if not pattern: - raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - drv, root, pattern_parts = self._parse_path(pattern) - if drv or root: - raise NotImplementedError("Non-relative patterns are unsupported") - if pattern[-1] in (self._flavour.sep, self._flavour.altsep): - pattern_parts.append('') - selector = _make_selector(tuple(pattern_parts), self._flavour, case_sensitive) - for p in selector.select_from(self): - yield p + return self.stat(follow_symlinks=False) - def rglob(self, pattern, *, case_sensitive=None): - """Recursively yield all existing files (of any kind, including - directories) matching the given relative pattern, anywhere in - this subtree. - """ - sys.audit("pathlib.Path.rglob", self, pattern) - drv, root, pattern_parts = self._parse_path(pattern) - if drv or root: - raise NotImplementedError("Non-relative patterns are unsupported") - if pattern and pattern[-1] in (self._flavour.sep, self._flavour.altsep): - pattern_parts.append('') - selector = _make_selector(("**",) + tuple(pattern_parts), self._flavour, case_sensitive) - for p in selector.select_from(self): - yield p - def absolute(self): - """Return an absolute version of this path by prepending the current - working directory. No normalization or symlink resolution is performed. + # Convenience functions for querying the stat results - Use resolve() to get the canonical path to a file. + def exists(self, *, follow_symlinks=True): """ - if self.is_absolute(): - return self - elif self.drive: - # There is a CWD on each drive-letter drive. - cwd = self._flavour.abspath(self.drive) - else: - cwd = os.getcwd() - return type(self)(cwd, self) + Whether this path exists. - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. + This method normally follows symlinks; to check whether a symlink exists, + add the argument follow_symlinks=False. """ - - def check_eloop(e): - winerror = getattr(e, 'winerror', 0) - if e.errno == ELOOP or winerror == _WINERROR_CANT_RESOLVE_FILENAME: - raise RuntimeError("Symlink loop from %r" % e.filename) - try: - s = self._flavour.realpath(self, strict=strict) + self.stat(follow_symlinks=follow_symlinks) except OSError as e: - check_eloop(e) - raise - p = type(self)(s) - - # In non-strict mode, realpath() doesn't raise on symlink loops. - # Ensure we get an exception by calling stat() - if not strict: - try: - p.stat() - except OSError as e: - check_eloop(e) - return p + if not _ignore_error(e): + raise + return False + except ValueError: + # Non-encodable path + return False + return True - def stat(self, *, follow_symlinks=True): + def is_dir(self): """ - Return the result of the stat() system call on this path, like - os.stat() does. + Whether this path is a directory. """ - return os.stat(self, follow_symlinks=follow_symlinks) + try: + return S_ISDIR(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False - def owner(self): + def is_file(self): """ - Return the login name of the file owner. + Whether this path is a regular file (also True for symlinks pointing + to regular files). """ try: - import pwd - return pwd.getpwuid(self.stat().st_uid).pw_name - except ImportError: - raise NotImplementedError("Path.owner() is unsupported on this system") + return S_ISREG(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False - def group(self): + def is_mount(self): """ - Return the group name of the file gid. + Check if this path is a mount point """ + return self._flavour.ismount(self) + def is_symlink(self): + """ + Whether this path is a symbolic link. + """ try: - import grp - return grp.getgrgid(self.stat().st_gid).gr_name - except ImportError: - raise NotImplementedError("Path.group() is unsupported on this system") + return S_ISLNK(self.lstat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist + return False + except ValueError: + # Non-encodable path + return False - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): + def is_junction(self): """ - Open the file pointed by this path and return a file object, as - the built-in open() function does. + Whether this path is a junction. """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return io.open(self, mode, buffering, encoding, errors, newline) + return self._flavour.isjunction(self) - def read_bytes(self): + def is_block_device(self): """ - Open the file in bytes mode, read it, and close the file. + Whether this path is a block device. """ - with self.open(mode='rb') as f: + try: + return S_ISBLK(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_char_device(self): + """ + Whether this path is a character device. + """ + try: + return S_ISCHR(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_fifo(self): + """ + Whether this path is a FIFO. + """ + try: + return S_ISFIFO(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_socket(self): + """ + Whether this path is a socket. + """ + try: + return S_ISSOCK(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def samefile(self, other_path): + """Return whether other_path is the same or not as this file + (as returned by os.path.samefile()). + """ + st = self.stat() + try: + other_st = other_path.stat() + except AttributeError: + other_st = self.with_segments(other_path).stat() + return self._flavour.samestat(st, other_st) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def read_bytes(self): + """ + Open the file in bytes mode, read it, and close the file. + """ + with self.open(mode='rb') as f: return f.read() def read_text(self, encoding=None, errors=None): @@ -969,13 +975,239 @@ def write_text(self, data, encoding=None, errors=None, newline=None): with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: return f.write(data) + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + for name in os.listdir(self): + yield self._make_child_relpath(name) + + def _scandir(self): + # bpo-24132: a future version of pathlib will support subclassing of + # pathlib.Path to customize how the filesystem is accessed. This + # includes scandir(), which is used to implement glob(). + return os.scandir(self) + + def _make_child_relpath(self, name): + path_str = str(self) + tail = self._tail + if tail: + path_str = f'{path_str}{self._flavour.sep}{name}' + elif path_str != '.': + path_str = f'{path_str}{name}' + else: + path_str = name + path = self.with_segments(path_str) + path._str = path_str + path._drv = self.drive + path._root = self.root + path._tail_cached = tail + [name] + return path + + def glob(self, pattern, *, case_sensitive=None): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + sys.audit("pathlib.Path.glob", self, pattern) + if not pattern: + raise ValueError("Unacceptable pattern: {!r}".format(pattern)) + drv, root, pattern_parts = self._parse_path(pattern) + if drv or root: + raise NotImplementedError("Non-relative patterns are unsupported") + if pattern[-1] in (self._flavour.sep, self._flavour.altsep): + pattern_parts.append('') + selector = _make_selector(tuple(pattern_parts), self._flavour, case_sensitive) + for p in selector.select_from(self): + yield p + + def rglob(self, pattern, *, case_sensitive=None): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + sys.audit("pathlib.Path.rglob", self, pattern) + drv, root, pattern_parts = self._parse_path(pattern) + if drv or root: + raise NotImplementedError("Non-relative patterns are unsupported") + if pattern and pattern[-1] in (self._flavour.sep, self._flavour.altsep): + pattern_parts.append('') + selector = _make_selector(("**",) + tuple(pattern_parts), self._flavour, case_sensitive) + for p in selector.select_from(self): + yield p + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) + paths = [self] + + while paths: + path = paths.pop() + if isinstance(path, tuple): + yield path + continue + + # We may not have read permission for self, in which case we can't + # get a list of the files the directory contains. os.walk() + # always suppressed the exception in that instance, rather than + # blow up for a minor reason when (say) a thousand readable + # directories are still left to visit. That logic is copied here. + try: + scandir_it = path._scandir() + except OSError as error: + if on_error is not None: + on_error(error) + continue + + with scandir_it: + dirnames = [] + filenames = [] + for entry in scandir_it: + try: + is_dir = entry.is_dir(follow_symlinks=follow_symlinks) + except OSError: + # Carried over from os.path.isdir(). + is_dir = False + + if is_dir: + dirnames.append(entry.name) + else: + filenames.append(entry.name) + + if top_down: + yield path, dirnames, filenames + else: + paths.append((path, dirnames, filenames)) + + paths += [path._make_child_relpath(d) for d in reversed(dirnames)] + + def __init__(self, *args, **kwargs): + if kwargs: + msg = ("support for supplying keyword arguments to pathlib.PurePath " + "is deprecated and scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) + super().__init__(*args) + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + def __enter__(self): + # In previous versions of pathlib, __exit__() marked this path as + # closed; subsequent attempts to perform I/O would raise an IOError. + # This functionality was never documented, and had the effect of + # making Path objects mutable, contrary to PEP 428. + # In Python 3.9 __exit__() was made a no-op. + # In Python 3.11 __enter__() began emitting DeprecationWarning. + # In Python 3.13 __enter__() and __exit__() should be removed. + warnings.warn("pathlib.Path.__enter__() is deprecated and scheduled " + "for removal in Python 3.13; Path objects as a context " + "manager is a no-op", + DeprecationWarning, stacklevel=2) + return self + + def __exit__(self, t, v, tb): + pass + + # Public API + + @classmethod + def cwd(cls): + """Return a new path pointing to the current working directory.""" + # We call 'absolute()' rather than using 'os.getcwd()' directly to + # enable users to replace the implementation of 'absolute()' in a + # subclass and benefit from the new behaviour here. This works because + # os.path.abspath('.') == os.getcwd(). + return cls().absolute() + + @classmethod + def home(cls): + """Return a new path pointing to the user's home directory (as + returned by os.path.expanduser('~')). + """ + return cls("~").expanduser() + + def absolute(self): + """Return an absolute version of this path by prepending the current + working directory. No normalization or symlink resolution is performed. + + Use resolve() to get the canonical path to a file. + """ + if self.is_absolute(): + return self + elif self.drive: + # There is a CWD on each drive-letter drive. + cwd = self._flavour.abspath(self.drive) + else: + cwd = os.getcwd() + # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). + # We pass only one argument to with_segments() to avoid the cost + # of joining, and we exploit the fact that getcwd() returns a + # fully-normalized string by storing it in _str. This is used to + # implement Path.cwd(). + if not self.root and not self._tail: + result = self.with_segments(cwd) + result._str = cwd + return result + return self.with_segments(cwd, self) + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + + def check_eloop(e): + winerror = getattr(e, 'winerror', 0) + if e.errno == ELOOP or winerror == _WINERROR_CANT_RESOLVE_FILENAME: + raise RuntimeError("Symlink loop from %r" % e.filename) + + try: + s = self._flavour.realpath(self, strict=strict) + except OSError as e: + check_eloop(e) + raise + p = self.with_segments(s) + + # In non-strict mode, realpath() doesn't raise on symlink loops. + # Ensure we get an exception by calling stat() + if not strict: + try: + p.stat() + except OSError as e: + check_eloop(e) + return p + + def owner(self): + """ + Return the login name of the file owner. + """ + try: + import pwd + return pwd.getpwuid(self.stat().st_uid).pw_name + except ImportError: + raise NotImplementedError("Path.owner() is unsupported on this system") + + def group(self): + """ + Return the group name of the file gid. + """ + + try: + import grp + return grp.getgrgid(self.stat().st_gid).gr_name + except ImportError: + raise NotImplementedError("Path.group() is unsupported on this system") + def readlink(self): """ Return the path to which the symbolic link points. """ if not hasattr(os, "readlink"): raise NotImplementedError("os.readlink() not available on this system") - return type(self)(os.readlink(self)) + return self.with_segments(os.readlink(self)) def touch(self, mode=0o666, exist_ok=True): """ @@ -1046,13 +1278,6 @@ def rmdir(self): """ os.rmdir(self) - def lstat(self): - """ - Like stat(), except if the path points to a symlink, the symlink's - status information is returned, rather than its target's. - """ - return self.stat(follow_symlinks=False) - def rename(self, target): """ Rename this path to the target path. @@ -1064,7 +1289,7 @@ def rename(self, target): Returns the new Path instance pointing to the target path. """ os.rename(self, target) - return self.__class__(target) + return self.with_segments(target) def replace(self, target): """ @@ -1077,7 +1302,7 @@ def replace(self, target): Returns the new Path instance pointing to the target path. """ os.replace(self, target) - return self.__class__(target) + return self.with_segments(target) def symlink_to(self, target, target_is_directory=False): """ @@ -1098,151 +1323,6 @@ def hardlink_to(self, target): raise NotImplementedError("os.link() not available on this system") os.link(target, self) - - # Convenience functions for querying the stat results - - def exists(self, *, follow_symlinks=True): - """ - Whether this path exists. - - This method normally follows symlinks; to check whether a symlink exists, - add the argument follow_symlinks=False. - """ - try: - self.stat(follow_symlinks=follow_symlinks) - except OSError as e: - if not _ignore_error(e): - raise - return False - except ValueError: - # Non-encodable path - return False - return True - - def is_dir(self): - """ - Whether this path is a directory. - """ - try: - return S_ISDIR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_file(self): - """ - Whether this path is a regular file (also True for symlinks pointing - to regular files). - """ - try: - return S_ISREG(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_mount(self): - """ - Check if this path is a mount point - """ - return self._flavour.ismount(self) - - def is_symlink(self): - """ - Whether this path is a symbolic link. - """ - try: - return S_ISLNK(self.lstat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist - return False - except ValueError: - # Non-encodable path - return False - - def is_junction(self): - """ - Whether this path is a junction. - """ - return self._flavour.isjunction(self) - - def is_block_device(self): - """ - Whether this path is a block device. - """ - try: - return S_ISBLK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_char_device(self): - """ - Whether this path is a character device. - """ - try: - return S_ISCHR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_fifo(self): - """ - Whether this path is a FIFO. - """ - try: - return S_ISFIFO(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_socket(self): - """ - Whether this path is a socket. - """ - try: - return S_ISSOCK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - def expanduser(self): """ Return a new path with expanded ~ and ~user constructs (as returned by os.path.expanduser) @@ -1257,51 +1337,6 @@ def expanduser(self): return self - def walk(self, top_down=True, on_error=None, follow_symlinks=False): - """Walk the directory tree from this directory, similar to os.walk().""" - sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) - paths = [self] - - while paths: - path = paths.pop() - if isinstance(path, tuple): - yield path - continue - - # We may not have read permission for self, in which case we can't - # get a list of the files the directory contains. os.walk() - # always suppressed the exception in that instance, rather than - # blow up for a minor reason when (say) a thousand readable - # directories are still left to visit. That logic is copied here. - try: - scandir_it = path._scandir() - except OSError as error: - if on_error is not None: - on_error(error) - continue - - with scandir_it: - dirnames = [] - filenames = [] - for entry in scandir_it: - try: - is_dir = entry.is_dir(follow_symlinks=follow_symlinks) - except OSError: - # Carried over from os.path.isdir(). - is_dir = False - - if is_dir: - dirnames.append(entry.name) - else: - filenames.append(entry.name) - - if top_down: - yield path, dirnames, filenames - else: - paths.append((path, dirnames, filenames)) - - paths += [path._make_child_relpath(d) for d in reversed(dirnames)] - class PosixPath(Path, PurePosixPath): """Path subclass for non-Windows systems. diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 53e5df5ba872ed..564205274edd73 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -4102,3 +4102,172 @@ exit: static PyObject * test_paramname_module_impl(PyObject *module, PyObject *mod) /*[clinic end generated code: output=4a2a849ecbcc8b53 input=afefe259667f13ba]*/ + +/*[clinic input] +mangle1 + + args: object + kwnames: object + return_value: object + _keywords: object + _parser: object + argsbuf: object + fastargs: object + nargs: object + noptargs: object + +[clinic start generated code]*/ + +PyDoc_STRVAR(mangle1__doc__, +"mangle1($module, /, args, kwnames, return_value, _keywords, _parser,\n" +" argsbuf, fastargs, nargs, noptargs)\n" +"--\n" +"\n"); + +#define MANGLE1_METHODDEF \ + {"mangle1", _PyCFunction_CAST(mangle1), METH_FASTCALL|METH_KEYWORDS, mangle1__doc__}, + +static PyObject * +mangle1_impl(PyObject *module, PyObject *args, PyObject *kwnames, + PyObject *return_value, PyObject *_keywords, PyObject *_parser, + PyObject *argsbuf, PyObject *fastargs, PyObject *nargs, + PyObject *noptargs); + +static PyObject * +mangle1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 9 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(args), &_Py_ID(kwnames), &_Py_ID(return_value), &_Py_ID(_keywords), &_Py_ID(_parser), &_Py_ID(argsbuf), &_Py_ID(fastargs), &_Py_ID(nargs), &_Py_ID(noptargs), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"args", "kwnames", "return_value", "_keywords", "_parser", "argsbuf", "fastargs", "nargs", "noptargs", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "mangle1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[9]; + PyObject *__clinic_args; + PyObject *__clinic_kwnames; + PyObject *__clinic_return_value; + PyObject *__clinic__keywords; + PyObject *__clinic__parser; + PyObject *__clinic_argsbuf; + PyObject *__clinic_fastargs; + PyObject *__clinic_nargs; + PyObject *__clinic_noptargs; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 9, 9, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + __clinic_kwnames = args[1]; + __clinic_return_value = args[2]; + __clinic__keywords = args[3]; + __clinic__parser = args[4]; + __clinic_argsbuf = args[5]; + __clinic_fastargs = args[6]; + __clinic_nargs = args[7]; + __clinic_noptargs = args[8]; + return_value = mangle1_impl(module, __clinic_args, __clinic_kwnames, __clinic_return_value, __clinic__keywords, __clinic__parser, __clinic_argsbuf, __clinic_fastargs, __clinic_nargs, __clinic_noptargs); + +exit: + return return_value; +} + +static PyObject * +mangle1_impl(PyObject *module, PyObject *args, PyObject *kwnames, + PyObject *return_value, PyObject *_keywords, PyObject *_parser, + PyObject *argsbuf, PyObject *fastargs, PyObject *nargs, + PyObject *noptargs) +/*[clinic end generated code: output=083e5076be9987c3 input=a3ed51bdedf8a3c7]*/ + +/*[clinic input] +mangle2 + + args: object + kwargs: object + return_value: object + +[clinic start generated code]*/ + +PyDoc_STRVAR(mangle2__doc__, +"mangle2($module, /, args, kwargs, return_value)\n" +"--\n" +"\n"); + +#define MANGLE2_METHODDEF \ + {"mangle2", _PyCFunction_CAST(mangle2), METH_FASTCALL|METH_KEYWORDS, mangle2__doc__}, + +static PyObject * +mangle2_impl(PyObject *module, PyObject *args, PyObject *kwargs, + PyObject *return_value); + +static PyObject * +mangle2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(args), &_Py_ID(kwargs), &_Py_ID(return_value), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"args", "kwargs", "return_value", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "mangle2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *__clinic_args; + PyObject *__clinic_kwargs; + PyObject *__clinic_return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + __clinic_kwargs = args[1]; + __clinic_return_value = args[2]; + return_value = mangle2_impl(module, __clinic_args, __clinic_kwargs, __clinic_return_value); + +exit: + return return_value; +} + +static PyObject * +mangle2_impl(PyObject *module, PyObject *args, PyObject *kwargs, + PyObject *return_value) +/*[clinic end generated code: output=2ebb62aaefe7590a input=391766fee51bad7a]*/ diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 861da2326d1214..0659d244d35686 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -1,5 +1,7 @@ # Author: Steven J. Bethard . +import contextlib +import functools import inspect import io import operator @@ -35,6 +37,35 @@ def getvalue(self): return self.buffer.raw.getvalue().decode('utf-8') +class StdStreamTest(unittest.TestCase): + + def test_skip_invalid_stderr(self): + parser = argparse.ArgumentParser() + with ( + contextlib.redirect_stderr(None), + mock.patch('argparse._sys.exit') + ): + parser.exit(status=0, message='foo') + + def test_skip_invalid_stdout(self): + parser = argparse.ArgumentParser() + for func in ( + parser.print_usage, + parser.print_help, + functools.partial(parser.parse_args, ['-h']) + ): + with ( + self.subTest(func=func), + contextlib.redirect_stdout(None), + # argparse uses stderr as a fallback + StdIOBuffer() as mocked_stderr, + contextlib.redirect_stderr(mocked_stderr), + mock.patch('argparse._sys.exit'), + ): + func() + self.assertRegex(mocked_stderr.getvalue(), r'usage:') + + class TestCase(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 8eef7baec70118..fdd21aca06ffdd 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -8,9 +8,11 @@ import unittest import warnings import weakref +from functools import partial from textwrap import dedent from test import support +from test.support.import_helper import import_fresh_module from test.support import os_helper, script_helper from test.support.ast_helper import ASTTestMixin @@ -267,6 +269,7 @@ def to_tuple(t): # excepthandler, arguments, keywords, alias class AST_Tests(unittest.TestCase): + maxDiff = None def _is_ast_node(self, name, node): if not isinstance(node, type): @@ -435,16 +438,42 @@ def test_base_classes(self): self.assertTrue(issubclass(ast.comprehension, ast.AST)) self.assertTrue(issubclass(ast.Gt, ast.AST)) + def test_import_deprecated(self): + ast = import_fresh_module('ast') + depr_regex = ( + r'ast\.{} is deprecated and will be removed in Python 3.14; ' + r'use ast\.Constant instead' + ) + for name in 'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis': + with self.assertWarnsRegex(DeprecationWarning, depr_regex.format(name)): + getattr(ast, name) + + def test_field_attr_existence_deprecated(self): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num, Str, Bytes, NameConstant, Ellipsis + + for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'): + item = getattr(ast, name) + if self._is_ast_node(name, item): + with self.subTest(item): + with self.assertWarns(DeprecationWarning): + x = item() + if isinstance(x, ast.AST): + self.assertIs(type(x._fields), tuple) + def test_field_attr_existence(self): for name, item in ast.__dict__.items(): + # These emit DeprecationWarnings + if name in {'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'}: + continue + # constructor has a different signature + if name == 'Index': + continue if self._is_ast_node(name, item): - if name == 'Index': - # Index(value) just returns value now. - # The argument is required. - continue x = item() if isinstance(x, ast.AST): - self.assertEqual(type(x._fields), tuple) + self.assertIs(type(x._fields), tuple) def test_arguments(self): x = ast.arguments() @@ -459,25 +488,108 @@ def test_arguments(self): self.assertEqual(x.args, 2) self.assertEqual(x.vararg, 3) + def test_field_attr_writable_deprecated(self): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + x = ast.Num() + # We can assign to _fields + x._fields = 666 + self.assertEqual(x._fields, 666) + def test_field_attr_writable(self): - x = ast.Num() + x = ast.Constant() # We can assign to _fields x._fields = 666 self.assertEqual(x._fields, 666) + def test_classattrs_deprecated(self): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num, Str, Bytes, NameConstant, Ellipsis + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + x = ast.Num() + self.assertEqual(x._fields, ('value', 'kind')) + + with self.assertRaises(AttributeError): + x.value + + with self.assertRaises(AttributeError): + x.n + + x = ast.Num(42) + self.assertEqual(x.value, 42) + self.assertEqual(x.n, 42) + + with self.assertRaises(AttributeError): + x.lineno + + with self.assertRaises(AttributeError): + x.foobar + + x = ast.Num(lineno=2) + self.assertEqual(x.lineno, 2) + + x = ast.Num(42, lineno=0) + self.assertEqual(x.lineno, 0) + self.assertEqual(x._fields, ('value', 'kind')) + self.assertEqual(x.value, 42) + self.assertEqual(x.n, 42) + + self.assertRaises(TypeError, ast.Num, 1, None, 2) + self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) + + # Arbitrary keyword arguments are supported + self.assertEqual(ast.Num(1, foo='bar').foo, 'bar') + + with self.assertRaisesRegex(TypeError, "Num got multiple values for argument 'n'"): + ast.Num(1, n=2) + + self.assertEqual(ast.Num(42).n, 42) + self.assertEqual(ast.Num(4.25).n, 4.25) + self.assertEqual(ast.Num(4.25j).n, 4.25j) + self.assertEqual(ast.Str('42').s, '42') + self.assertEqual(ast.Bytes(b'42').s, b'42') + self.assertIs(ast.NameConstant(True).value, True) + self.assertIs(ast.NameConstant(False).value, False) + self.assertIs(ast.NameConstant(None).value, None) + + self.assertEqual([str(w.message) for w in wlog], [ + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + ]) + def test_classattrs(self): - x = ast.Num() + x = ast.Constant() self.assertEqual(x._fields, ('value', 'kind')) with self.assertRaises(AttributeError): x.value - with self.assertRaises(AttributeError): - x.n - - x = ast.Num(42) + x = ast.Constant(42) self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) with self.assertRaises(AttributeError): x.lineno @@ -485,36 +597,23 @@ def test_classattrs(self): with self.assertRaises(AttributeError): x.foobar - x = ast.Num(lineno=2) + x = ast.Constant(lineno=2) self.assertEqual(x.lineno, 2) - x = ast.Num(42, lineno=0) + x = ast.Constant(42, lineno=0) self.assertEqual(x.lineno, 0) self.assertEqual(x._fields, ('value', 'kind')) self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - self.assertRaises(TypeError, ast.Num, 1, None, 2) - self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) + self.assertRaises(TypeError, ast.Constant, 1, None, 2) + self.assertRaises(TypeError, ast.Constant, 1, None, 2, lineno=0) # Arbitrary keyword arguments are supported self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') - self.assertEqual(ast.Num(1, foo='bar').foo, 'bar') - with self.assertRaisesRegex(TypeError, "Num got multiple values for argument 'n'"): - ast.Num(1, n=2) with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"): ast.Constant(1, value=2) - self.assertEqual(ast.Num(42).n, 42) - self.assertEqual(ast.Num(4.25).n, 4.25) - self.assertEqual(ast.Num(4.25j).n, 4.25j) - self.assertEqual(ast.Str('42').s, '42') - self.assertEqual(ast.Bytes(b'42').s, b'42') - self.assertIs(ast.NameConstant(True).value, True) - self.assertIs(ast.NameConstant(False).value, False) - self.assertIs(ast.NameConstant(None).value, None) - self.assertEqual(ast.Constant(42).value, 42) self.assertEqual(ast.Constant(4.25).value, 4.25) self.assertEqual(ast.Constant(4.25j).value, 4.25j) @@ -526,85 +625,211 @@ def test_classattrs(self): self.assertIs(ast.Constant(...).value, ...) def test_realtype(self): - self.assertEqual(type(ast.Num(42)), ast.Constant) - self.assertEqual(type(ast.Num(4.25)), ast.Constant) - self.assertEqual(type(ast.Num(4.25j)), ast.Constant) - self.assertEqual(type(ast.Str('42')), ast.Constant) - self.assertEqual(type(ast.Bytes(b'42')), ast.Constant) - self.assertEqual(type(ast.NameConstant(True)), ast.Constant) - self.assertEqual(type(ast.NameConstant(False)), ast.Constant) - self.assertEqual(type(ast.NameConstant(None)), ast.Constant) - self.assertEqual(type(ast.Ellipsis()), ast.Constant) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num, Str, Bytes, NameConstant, Ellipsis + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + self.assertIs(type(ast.Num(42)), ast.Constant) + self.assertIs(type(ast.Num(4.25)), ast.Constant) + self.assertIs(type(ast.Num(4.25j)), ast.Constant) + self.assertIs(type(ast.Str('42')), ast.Constant) + self.assertIs(type(ast.Bytes(b'42')), ast.Constant) + self.assertIs(type(ast.NameConstant(True)), ast.Constant) + self.assertIs(type(ast.NameConstant(False)), ast.Constant) + self.assertIs(type(ast.NameConstant(None)), ast.Constant) + self.assertIs(type(ast.Ellipsis()), ast.Constant) + + self.assertEqual([str(w.message) for w in wlog], [ + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Ellipsis is deprecated and will be removed in Python 3.14; use ast.Constant instead', + ]) def test_isinstance(self): - self.assertTrue(isinstance(ast.Num(42), ast.Num)) - self.assertTrue(isinstance(ast.Num(4.2), ast.Num)) - self.assertTrue(isinstance(ast.Num(4.2j), ast.Num)) - self.assertTrue(isinstance(ast.Str('42'), ast.Str)) - self.assertTrue(isinstance(ast.Bytes(b'42'), ast.Bytes)) - self.assertTrue(isinstance(ast.NameConstant(True), ast.NameConstant)) - self.assertTrue(isinstance(ast.NameConstant(False), ast.NameConstant)) - self.assertTrue(isinstance(ast.NameConstant(None), ast.NameConstant)) - self.assertTrue(isinstance(ast.Ellipsis(), ast.Ellipsis)) - - self.assertTrue(isinstance(ast.Constant(42), ast.Num)) - self.assertTrue(isinstance(ast.Constant(4.2), ast.Num)) - self.assertTrue(isinstance(ast.Constant(4.2j), ast.Num)) - self.assertTrue(isinstance(ast.Constant('42'), ast.Str)) - self.assertTrue(isinstance(ast.Constant(b'42'), ast.Bytes)) - self.assertTrue(isinstance(ast.Constant(True), ast.NameConstant)) - self.assertTrue(isinstance(ast.Constant(False), ast.NameConstant)) - self.assertTrue(isinstance(ast.Constant(None), ast.NameConstant)) - self.assertTrue(isinstance(ast.Constant(...), ast.Ellipsis)) - - self.assertFalse(isinstance(ast.Str('42'), ast.Num)) - self.assertFalse(isinstance(ast.Num(42), ast.Str)) - self.assertFalse(isinstance(ast.Str('42'), ast.Bytes)) - self.assertFalse(isinstance(ast.Num(42), ast.NameConstant)) - self.assertFalse(isinstance(ast.Num(42), ast.Ellipsis)) - self.assertFalse(isinstance(ast.NameConstant(True), ast.Num)) - self.assertFalse(isinstance(ast.NameConstant(False), ast.Num)) - - self.assertFalse(isinstance(ast.Constant('42'), ast.Num)) - self.assertFalse(isinstance(ast.Constant(42), ast.Str)) - self.assertFalse(isinstance(ast.Constant('42'), ast.Bytes)) - self.assertFalse(isinstance(ast.Constant(42), ast.NameConstant)) - self.assertFalse(isinstance(ast.Constant(42), ast.Ellipsis)) - self.assertFalse(isinstance(ast.Constant(True), ast.Num)) - self.assertFalse(isinstance(ast.Constant(False), ast.Num)) - - self.assertFalse(isinstance(ast.Constant(), ast.Num)) - self.assertFalse(isinstance(ast.Constant(), ast.Str)) - self.assertFalse(isinstance(ast.Constant(), ast.Bytes)) - self.assertFalse(isinstance(ast.Constant(), ast.NameConstant)) - self.assertFalse(isinstance(ast.Constant(), ast.Ellipsis)) + from ast import Constant + + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num, Str, Bytes, NameConstant, Ellipsis + + cls_depr_msg = ( + 'ast.{} is deprecated and will be removed in Python 3.14; ' + 'use ast.Constant instead' + ) + + assertNumDeprecated = partial( + self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Num") + ) + assertStrDeprecated = partial( + self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Str") + ) + assertBytesDeprecated = partial( + self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Bytes") + ) + assertNameConstantDeprecated = partial( + self.assertWarnsRegex, + DeprecationWarning, + cls_depr_msg.format("NameConstant") + ) + assertEllipsisDeprecated = partial( + self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Ellipsis") + ) + + for arg in 42, 4.2, 4.2j: + with self.subTest(arg=arg): + with assertNumDeprecated(): + n = Num(arg) + with assertNumDeprecated(): + self.assertIsInstance(n, Num) + + with assertStrDeprecated(): + s = Str('42') + with assertStrDeprecated(): + self.assertIsInstance(s, Str) + + with assertBytesDeprecated(): + b = Bytes(b'42') + with assertBytesDeprecated(): + self.assertIsInstance(b, Bytes) + + for arg in True, False, None: + with self.subTest(arg=arg): + with assertNameConstantDeprecated(): + n = NameConstant(arg) + with assertNameConstantDeprecated(): + self.assertIsInstance(n, NameConstant) + + with assertEllipsisDeprecated(): + e = Ellipsis() + with assertEllipsisDeprecated(): + self.assertIsInstance(e, Ellipsis) + + for arg in 42, 4.2, 4.2j: + with self.subTest(arg=arg): + with assertNumDeprecated(): + self.assertIsInstance(Constant(arg), Num) + + with assertStrDeprecated(): + self.assertIsInstance(Constant('42'), Str) + + with assertBytesDeprecated(): + self.assertIsInstance(Constant(b'42'), Bytes) + + for arg in True, False, None: + with self.subTest(arg=arg): + with assertNameConstantDeprecated(): + self.assertIsInstance(Constant(arg), NameConstant) + + with assertEllipsisDeprecated(): + self.assertIsInstance(Constant(...), Ellipsis) + + with assertStrDeprecated(): + s = Str('42') + assertNumDeprecated(self.assertNotIsInstance, s, Num) + assertBytesDeprecated(self.assertNotIsInstance, s, Bytes) + + with assertNumDeprecated(): + n = Num(42) + assertStrDeprecated(self.assertNotIsInstance, n, Str) + assertNameConstantDeprecated(self.assertNotIsInstance, n, NameConstant) + assertEllipsisDeprecated(self.assertNotIsInstance, n, Ellipsis) + + with assertNameConstantDeprecated(): + n = NameConstant(True) + with assertNumDeprecated(): + self.assertNotIsInstance(n, Num) + + with assertNameConstantDeprecated(): + n = NameConstant(False) + with assertNumDeprecated(): + self.assertNotIsInstance(n, Num) + + for arg in '42', True, False: + with self.subTest(arg=arg): + with assertNumDeprecated(): + self.assertNotIsInstance(Constant(arg), Num) + + assertStrDeprecated(self.assertNotIsInstance, Constant(42), Str) + assertBytesDeprecated(self.assertNotIsInstance, Constant('42'), Bytes) + assertNameConstantDeprecated(self.assertNotIsInstance, Constant(42), NameConstant) + assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis) + assertNumDeprecated(self.assertNotIsInstance, Constant(), Num) + assertStrDeprecated(self.assertNotIsInstance, Constant(), Str) + assertBytesDeprecated(self.assertNotIsInstance, Constant(), Bytes) + assertNameConstantDeprecated(self.assertNotIsInstance, Constant(), NameConstant) + assertEllipsisDeprecated(self.assertNotIsInstance, Constant(), Ellipsis) class S(str): pass - self.assertTrue(isinstance(ast.Constant(S('42')), ast.Str)) - self.assertFalse(isinstance(ast.Constant(S('42')), ast.Num)) + with assertStrDeprecated(): + self.assertIsInstance(Constant(S('42')), Str) + with assertNumDeprecated(): + self.assertNotIsInstance(Constant(S('42')), Num) + + def test_constant_subclasses_deprecated(self): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num - def test_subclasses(self): - class N(ast.Num): + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + class N(ast.Num): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.z = 'spam' + class N2(ast.Num): + pass + + n = N(42) + self.assertEqual(n.n, 42) + self.assertEqual(n.z, 'spam') + self.assertIs(type(n), N) + self.assertIsInstance(n, N) + self.assertIsInstance(n, ast.Num) + self.assertNotIsInstance(n, N2) + self.assertNotIsInstance(ast.Num(42), N) + n = N(n=42) + self.assertEqual(n.n, 42) + self.assertIs(type(n), N) + + self.assertEqual([str(w.message) for w in wlog], [ + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + ]) + + def test_constant_subclasses(self): + class N(ast.Constant): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.z = 'spam' - class N2(ast.Num): + class N2(ast.Constant): pass n = N(42) - self.assertEqual(n.n, 42) + self.assertEqual(n.value, 42) self.assertEqual(n.z, 'spam') self.assertEqual(type(n), N) self.assertTrue(isinstance(n, N)) - self.assertTrue(isinstance(n, ast.Num)) + self.assertTrue(isinstance(n, ast.Constant)) self.assertFalse(isinstance(n, N2)) - self.assertFalse(isinstance(ast.Num(42), N)) - n = N(n=42) - self.assertEqual(n.n, 42) + self.assertFalse(isinstance(ast.Constant(42), N)) + n = N(value=42) + self.assertEqual(n.value, 42) self.assertEqual(type(n), N) def test_module(self): - body = [ast.Num(42)] + body = [ast.Constant(42)] x = ast.Module(body, []) self.assertEqual(x.body, body) @@ -617,8 +842,8 @@ def test_nodeclasses(self): x.foobarbaz = 5 self.assertEqual(x.foobarbaz, 5) - n1 = ast.Num(1) - n3 = ast.Num(3) + n1 = ast.Constant(1) + n3 = ast.Constant(3) addop = ast.Add() x = ast.BinOp(n1, addop, n3) self.assertEqual(x.left, n1) @@ -987,7 +1212,7 @@ def test_dump_incomplete(self): def test_copy_location(self): src = ast.parse('1 + 1', mode='eval') - src.body.right = ast.copy_location(ast.Num(2), src.body.right) + src.body.right = ast.copy_location(ast.Constant(2), src.body.right) self.assertEqual(ast.dump(src, include_attributes=True), 'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, ' 'end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, ' @@ -1004,7 +1229,7 @@ def test_copy_location(self): def test_fix_missing_locations(self): src = ast.parse('write("spam")') src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()), - [ast.Str('eggs')], []))) + [ast.Constant('eggs')], []))) self.assertEqual(src, ast.fix_missing_locations(src)) self.maxDiff = None self.assertEqual(ast.dump(src, include_attributes=True), @@ -1317,9 +1542,9 @@ def arguments(args=None, posonlyargs=None, vararg=None, check(arguments(args=args), "must have Load context") check(arguments(posonlyargs=args), "must have Load context") check(arguments(kwonlyargs=args), "must have Load context") - check(arguments(defaults=[ast.Num(3)]), + check(arguments(defaults=[ast.Constant(3)]), "more positional defaults than args") - check(arguments(kw_defaults=[ast.Num(4)]), + check(arguments(kw_defaults=[ast.Constant(4)]), "length of kwonlyargs is not the same as kw_defaults") args = [ast.arg("x", ast.Name("x", ast.Load()))] check(arguments(args=args, defaults=[ast.Name("x", ast.Store())]), @@ -1372,9 +1597,9 @@ def test_delete(self): "must have Del context") def test_assign(self): - self.stmt(ast.Assign([], ast.Num(3)), "empty targets on Assign") - self.stmt(ast.Assign([None], ast.Num(3)), "None disallowed") - self.stmt(ast.Assign([ast.Name("x", ast.Load())], ast.Num(3)), + self.stmt(ast.Assign([], ast.Constant(3)), "empty targets on Assign") + self.stmt(ast.Assign([None], ast.Constant(3)), "None disallowed") + self.stmt(ast.Assign([ast.Name("x", ast.Load())], ast.Constant(3)), "must have Store context") self.stmt(ast.Assign([ast.Name("x", ast.Store())], ast.Name("y", ast.Store())), @@ -1402,39 +1627,39 @@ def test_for(self): self.stmt(ast.For(x, y, [p], [e]), "must have Load context") def test_while(self): - self.stmt(ast.While(ast.Num(3), [], []), "empty body on While") + self.stmt(ast.While(ast.Constant(3), [], []), "empty body on While") self.stmt(ast.While(ast.Name("x", ast.Store()), [ast.Pass()], []), "must have Load context") - self.stmt(ast.While(ast.Num(3), [ast.Pass()], + self.stmt(ast.While(ast.Constant(3), [ast.Pass()], [ast.Expr(ast.Name("x", ast.Store()))]), "must have Load context") def test_if(self): - self.stmt(ast.If(ast.Num(3), [], []), "empty body on If") + self.stmt(ast.If(ast.Constant(3), [], []), "empty body on If") i = ast.If(ast.Name("x", ast.Store()), [ast.Pass()], []) self.stmt(i, "must have Load context") - i = ast.If(ast.Num(3), [ast.Expr(ast.Name("x", ast.Store()))], []) + i = ast.If(ast.Constant(3), [ast.Expr(ast.Name("x", ast.Store()))], []) self.stmt(i, "must have Load context") - i = ast.If(ast.Num(3), [ast.Pass()], + i = ast.If(ast.Constant(3), [ast.Pass()], [ast.Expr(ast.Name("x", ast.Store()))]) self.stmt(i, "must have Load context") def test_with(self): p = ast.Pass() self.stmt(ast.With([], [p]), "empty items on With") - i = ast.withitem(ast.Num(3), None) + i = ast.withitem(ast.Constant(3), None) self.stmt(ast.With([i], []), "empty body on With") i = ast.withitem(ast.Name("x", ast.Store()), None) self.stmt(ast.With([i], [p]), "must have Load context") - i = ast.withitem(ast.Num(3), ast.Name("x", ast.Load())) + i = ast.withitem(ast.Constant(3), ast.Name("x", ast.Load())) self.stmt(ast.With([i], [p]), "must have Store context") def test_raise(self): - r = ast.Raise(None, ast.Num(3)) + r = ast.Raise(None, ast.Constant(3)) self.stmt(r, "Raise with cause but no exception") r = ast.Raise(ast.Name("x", ast.Store()), None) self.stmt(r, "must have Load context") - r = ast.Raise(ast.Num(4), ast.Name("x", ast.Store())) + r = ast.Raise(ast.Constant(4), ast.Name("x", ast.Store())) self.stmt(r, "must have Load context") def test_try(self): @@ -1505,11 +1730,11 @@ def test_expr(self): def test_boolop(self): b = ast.BoolOp(ast.And(), []) self.expr(b, "less than 2 values") - b = ast.BoolOp(ast.And(), [ast.Num(3)]) + b = ast.BoolOp(ast.And(), [ast.Constant(3)]) self.expr(b, "less than 2 values") - b = ast.BoolOp(ast.And(), [ast.Num(4), None]) + b = ast.BoolOp(ast.And(), [ast.Constant(4), None]) self.expr(b, "None disallowed") - b = ast.BoolOp(ast.And(), [ast.Num(4), ast.Name("x", ast.Store())]) + b = ast.BoolOp(ast.And(), [ast.Constant(4), ast.Name("x", ast.Store())]) self.expr(b, "must have Load context") def test_unaryop(self): @@ -1597,11 +1822,11 @@ def test_compare(self): left = ast.Name("x", ast.Load()) comp = ast.Compare(left, [ast.In()], []) self.expr(comp, "no comparators") - comp = ast.Compare(left, [ast.In()], [ast.Num(4), ast.Num(5)]) + comp = ast.Compare(left, [ast.In()], [ast.Constant(4), ast.Constant(5)]) self.expr(comp, "different number of comparators and operands") - comp = ast.Compare(ast.Num("blah"), [ast.In()], [left]) + comp = ast.Compare(ast.Constant("blah"), [ast.In()], [left]) self.expr(comp) - comp = ast.Compare(left, [ast.In()], [ast.Num("blah")]) + comp = ast.Compare(left, [ast.In()], [ast.Constant("blah")]) self.expr(comp) def test_call(self): @@ -1617,23 +1842,37 @@ def test_call(self): self.expr(call, "must have Load context") def test_num(self): - class subint(int): - pass - class subfloat(float): - pass - class subcomplex(complex): - pass - for obj in "0", "hello": - self.expr(ast.Num(obj)) - for obj in subint(), subfloat(), subcomplex(): - self.expr(ast.Num(obj), "invalid type", exc=TypeError) + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + class subint(int): + pass + class subfloat(float): + pass + class subcomplex(complex): + pass + for obj in "0", "hello": + self.expr(ast.Num(obj)) + for obj in subint(), subfloat(), subcomplex(): + self.expr(ast.Num(obj), "invalid type", exc=TypeError) + + self.assertEqual([str(w.message) for w in wlog], [ + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + ]) def test_attribute(self): attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) self.expr(attr, "must have Load context") def test_subscript(self): - sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Num(3), + sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Constant(3), ast.Load()) self.expr(sub, "must have Load context") x = ast.Name("x", ast.Load()) @@ -1653,7 +1892,7 @@ def test_subscript(self): def test_starred(self): left = ast.List([ast.Starred(ast.Name("x", ast.Load()), ast.Store())], ast.Store()) - assign = ast.Assign([left], ast.Num(4)) + assign = ast.Assign([left], ast.Constant(4)) self.stmt(assign, "must have Store context") def _sequence(self, fac): @@ -1668,7 +1907,17 @@ def test_tuple(self): self._sequence(ast.Tuple) def test_nameconstant(self): - self.expr(ast.NameConstant(4)) + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import NameConstant + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + self.expr(ast.NameConstant(4)) + + self.assertEqual([str(w.message) for w in wlog], [ + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + ]) def test_stdlib_validates(self): stdlib = os.path.dirname(ast.__file__) @@ -2357,10 +2606,15 @@ def visit_Ellipsis(self, node): ]) self.assertEqual([str(w.message) for w in wlog], [ 'visit_Num is deprecated; add visit_Constant', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'visit_Num is deprecated; add visit_Constant', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'visit_Num is deprecated; add visit_Constant', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'visit_Str is deprecated; add visit_Constant', + 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', 'visit_Bytes is deprecated; add visit_Constant', + 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', 'visit_NameConstant is deprecated; add visit_Constant', 'visit_NameConstant is deprecated; add visit_Constant', 'visit_Ellipsis is deprecated; add visit_Constant', diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index b6e82ad4db266a..2c65ae8114818f 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4579,6 +4579,176 @@ def test_c_buffer(self): buf.__release_buffer__(mv) self.assertEqual(buf.references, 0) + def test_inheritance(self): + class A(bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + a = A(b"hello") + mv = memoryview(a) + self.assertEqual(mv.tobytes(), b"hello") + + def test_inheritance_releasebuffer(self): + rb_call_count = 0 + class B(bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + def __release_buffer__(self, view): + nonlocal rb_call_count + rb_call_count += 1 + super().__release_buffer__(view) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(rb_call_count, 0) + self.assertEqual(rb_call_count, 1) + + def test_inherit_but_return_something_else(self): + class A(bytearray): + def __buffer__(self, flags): + return memoryview(b"hello") + + a = A(b"hello") + with memoryview(a) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + rb_call_count = 0 + rb_raised = False + class B(bytearray): + def __buffer__(self, flags): + return memoryview(b"hello") + def __release_buffer__(self, view): + nonlocal rb_call_count + rb_call_count += 1 + try: + super().__release_buffer__(view) + except ValueError: + nonlocal rb_raised + rb_raised = True + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(rb_call_count, 0) + self.assertEqual(rb_call_count, 1) + self.assertIs(rb_raised, True) + + def test_override_only_release(self): + class C(bytearray): + def __release_buffer__(self, buffer): + super().__release_buffer__(buffer) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + def test_release_saves_reference(self): + smuggled_buffer = None + + class C(bytearray): + def __release_buffer__(s, buffer: memoryview): + with self.assertRaises(ValueError): + memoryview(buffer) + with self.assertRaises(ValueError): + buffer.cast("b") + with self.assertRaises(ValueError): + buffer.toreadonly() + with self.assertRaises(ValueError): + buffer[:1] + with self.assertRaises(ValueError): + buffer.__buffer__(0) + nonlocal smuggled_buffer + smuggled_buffer = buffer + self.assertEqual(buffer.tobytes(), b"hello") + super().__release_buffer__(buffer) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + c.clear() + with self.assertRaises(ValueError): + smuggled_buffer.tobytes() + + def test_release_saves_reference_no_subclassing(self): + ba = bytearray(b"hello") + + class C: + def __buffer__(self, flags): + return memoryview(ba) + + def __release_buffer__(self, buffer): + self.buffer = buffer + + c = C() + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(c.buffer.tobytes(), b"hello") + + with self.assertRaises(BufferError): + ba.clear() + c.buffer.release() + ba.clear() + + def test_multiple_inheritance_buffer_last(self): + class A: + def __buffer__(self, flags): + return memoryview(b"hello A") + + class B(A, bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello A") + + class Releaser: + def __release_buffer__(self, buffer): + self.buffer = buffer + + class C(Releaser, bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + c = C(b"hello C") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello C") + c.clear() + with self.assertRaises(ValueError): + c.buffer.tobytes() + + def test_multiple_inheritance_buffer_last(self): + class A: + def __buffer__(self, flags): + raise RuntimeError("should not be called") + + def __release_buffer__(self, buffer): + raise RuntimeError("should not be called") + + class B(bytearray, A): + def __buffer__(self, flags): + return super().__buffer__(flags) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + class Releaser: + buffer = None + def __release_buffer__(self, buffer): + self.buffer = buffer + + class C(bytearray, Releaser): + def __buffer__(self, flags): + return super().__buffer__(flags) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + c.clear() + self.assertIs(c.buffer, None) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 22be3c0814278e..3fc2c07f933061 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1401,23 +1401,37 @@ def test_configured_settings(self): DAEMON_THREADS = 1<<11 FORK = 1<<15 EXEC = 1<<16 - - features = ['obmalloc', 'fork', 'exec', 'threads', 'daemon_threads', - 'extensions'] + ALL_FLAGS = (OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS + | EXTENSIONS); + + features = [ + 'obmalloc', + 'fork', + 'exec', + 'threads', + 'daemon_threads', + 'extensions', + 'own_gil', + ] kwlist = [f'allow_{n}' for n in features] kwlist[0] = 'use_main_obmalloc' - kwlist[-1] = 'check_multi_interp_extensions' + kwlist[-2] = 'check_multi_interp_extensions' + kwlist[-1] = 'own_gil' # expected to work for config, expected in { - (True, True, True, True, True, True): - OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS, - (True, False, False, False, False, False): OBMALLOC, - (False, False, False, True, False, True): THREADS | EXTENSIONS, + (True, True, True, True, True, True, True): + (ALL_FLAGS, True), + (True, False, False, False, False, False, False): + (OBMALLOC, False), + (False, False, False, True, False, True, False): + (THREADS | EXTENSIONS, False), }.items(): kwargs = dict(zip(kwlist, config)) + exp_flags, exp_gil = expected expected = { - 'feature_flags': expected, + 'feature_flags': exp_flags, + 'own_gil': exp_gil, } with self.subTest(config): r, w = os.pipe() @@ -1437,7 +1451,7 @@ def test_configured_settings(self): # expected to fail for config in [ - (False, False, False, False, False, False), + (False, False, False, False, False, False, False), ]: kwargs = dict(zip(kwlist, config)) with self.subTest(config): @@ -1473,6 +1487,7 @@ def test_overridden_setting_extensions_subinterp_check(self): 'allow_exec': True, 'allow_threads': True, 'allow_daemon_threads': True, + 'own_gil': False, } def check(enabled, override): @@ -1483,6 +1498,7 @@ def check(enabled, override): flags = BASE_FLAGS | EXTENSIONS if enabled else BASE_FLAGS settings = { 'feature_flags': flags, + 'own_gil': False, } expected = { diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 4abf739cf52ca3..6aaf4d1ed8d560 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -99,8 +99,9 @@ def test_eol(self): # the last line of the block got corrupted. c = clinic.Clinic(clinic.CLanguage(None), filename="file") raw = "/*[clinic]\nfoo\n[clinic]*/" - cooked = c.parse(raw).splitlines() - end_line = cooked[2].rstrip() + cooked, _ = c.parse(raw) + lines = cooked.splitlines() + end_line = lines[2].rstrip() # this test is redundant, it's just here explicitly to catch # the regression test so we don't forget what it looked like self.assertNotEqual(end_line, "[clinic]*/[clinic]*/") @@ -259,7 +260,7 @@ def _test_clinic(self, input, output): c = clinic.Clinic(language, filename="file") c.parsers['inert'] = InertParser(c) c.parsers['copy'] = CopyParser(c) - computed = c.parse(input) + computed, _ = c.parse(input) self.assertEqual(output, computed) def test_clinic_1(self): @@ -1284,6 +1285,19 @@ def test_gh_99240_double_free(self): with self.assertRaisesRegex(TypeError, expected_error): ac_tester.gh_99240_double_free('a', '\0b') + def test_cloned_func_exception_message(self): + incorrect_arg = -1 # f1() and f2() accept a single str + with self.assertRaisesRegex(TypeError, "clone_f1"): + ac_tester.clone_f1(incorrect_arg) + with self.assertRaisesRegex(TypeError, "clone_f2"): + ac_tester.clone_f2(incorrect_arg) + + def test_cloned_func_with_converter_exception_message(self): + for name in "clone_with_conv_f1", "clone_with_conv_f2": + with self.subTest(name=name): + func = getattr(ac_tester, name) + self.assertEqual(func(), name) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 022753e0c99483..ea57df9cd2400b 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -25,6 +25,8 @@ def test_if_expression(self): ('LOAD_CONST', 2, 1), exit_lbl, ('POP_TOP', None), + ('LOAD_CONST', 3), + ('RETURN_VALUE', None), ] self.codegen_test(snippet, expected) @@ -46,5 +48,7 @@ def test_for_loop(self): ('JUMP', loop_lbl), exit_lbl, ('END_FOR', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), ] self.codegen_test(snippet, expected) diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index 98648528bc81f2..484b8f8e3a365c 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -25,7 +25,6 @@ def test_bad_counter_during_dealloc(self): with support.catch_unraisable_exception() as cm: obj = _lsprof.Profiler(lambda: int) obj.enable() - obj = _lsprof.Profiler(1) obj.disable() obj.clear() @@ -37,10 +36,11 @@ def test_profile_enable_disable(self): self.addCleanup(prof.disable) prof.enable() - self.assertIs(sys.getprofile(), prof) + self.assertEqual( + sys.monitoring.get_tool(sys.monitoring.PROFILER_ID), "cProfile") prof.disable() - self.assertIs(sys.getprofile(), None) + self.assertIs(sys.monitoring.get_tool(sys.monitoring.PROFILER_ID), None) def test_profile_as_context_manager(self): prof = self.profilerclass() @@ -53,10 +53,19 @@ def test_profile_as_context_manager(self): # profile should be set as the global profiler inside the # with-block - self.assertIs(sys.getprofile(), prof) + self.assertEqual( + sys.monitoring.get_tool(sys.monitoring.PROFILER_ID), "cProfile") # profile shouldn't be set once we leave the with-block. - self.assertIs(sys.getprofile(), None) + self.assertIs(sys.monitoring.get_tool(sys.monitoring.PROFILER_ID), None) + + def test_second_profiler(self): + pr = self.profilerclass() + pr2 = self.profilerclass() + pr.enable() + self.assertRaises(ValueError, pr2.enable) + pr.disable() + class TestCommandLine(unittest.TestCase): def test_sort(self): diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 8fb97bc0c1a1a7..de7ac97d72cb8e 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -10,7 +10,7 @@ import gc import pickle from test import support -from test.support import warnings_helper +from test.support import warnings_helper, import_helper, check_disallow_instantiation from itertools import permutations from textwrap import dedent from collections import OrderedDict @@ -1430,5 +1430,12 @@ def test_subclassable(self): # issue 44089 class Foo(csv.Error): ... + @support.cpython_only + def test_disallow_instantiation(self): + _csv = import_helper.import_module("_csv") + for tp in _csv.Reader, _csv.Writer: + with self.subTest(tp=tp): + check_disallow_instantiation(self, tp) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index c9691bbf304915..582392ecddcb91 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1666,6 +1666,7 @@ def test_init_main_interpreter_settings(self): # All optional features should be enabled. 'feature_flags': OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS, + 'own_gil': True, } out, err = self.run_embedded_interpreter( 'test_init_main_interpreter_settings', diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index cc782ea1ee5dff..31680b5a92e0f3 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -2141,11 +2141,10 @@ def printsolution(self, x): ... SyntaxError: 'yield' outside function -# Pegen does not produce this error message yet -# >>> def f(): x = yield = y -# Traceback (most recent call last): -# ... -# SyntaxError: assignment to yield expression not possible +>>> def f(): x = yield = y +Traceback (most recent call last): + ... +SyntaxError: assignment to yield expression not possible >>> def f(): (yield bar) = y Traceback (most recent call last): diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 37f77fe0a320c7..4b1d355f550b49 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -2390,6 +2390,20 @@ def test_tunnel_debuglog(self): lines = output.getvalue().splitlines() self.assertIn('header: {}'.format(expected_header), lines) + def test_proxy_response_headers(self): + expected_header = ('X-Dummy', '1') + response_text = ( + 'HTTP/1.0 200 OK\r\n' + '{0}\r\n\r\n'.format(':'.join(expected_header)) + ) + + self.conn._create_connection = self._create_connection(response_text) + self.conn.set_tunnel('destination.com') + + self.conn.request('PUT', '/', '') + headers = self.conn._proxy_response_headers + self.assertIn(expected_header, headers.items()) + def test_tunnel_leak(self): sock = None diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 41dfdaabe24664..e2384a08ecaa90 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1640,6 +1640,7 @@ class SubinterpImportTests(unittest.TestCase): ) ISOLATED = dict( use_main_obmalloc=False, + own_gil=True, ) NOT_ISOLATED = {k: not v for k, v in ISOLATED.items()} @@ -1652,26 +1653,44 @@ def pipe(self): os.set_blocking(r, False) return (r, w) - def import_script(self, name, fd, check_override=None): + def import_script(self, name, fd, filename=None, check_override=None): override_text = '' if check_override is not None: override_text = f''' - import _imp - _imp._override_multi_interp_extensions_check({check_override}) - ''' - return textwrap.dedent(f''' - import os, sys - {override_text} - try: - import {name} - except ImportError as exc: - text = 'ImportError: ' + str(exc) - else: - text = 'okay' - os.write({fd}, text.encode('utf-8')) - ''') + import _imp + _imp._override_multi_interp_extensions_check({check_override}) + ''' + if filename: + return textwrap.dedent(f''' + from importlib.util import spec_from_loader, module_from_spec + from importlib.machinery import ExtensionFileLoader + import os, sys + {override_text} + loader = ExtensionFileLoader({name!r}, {filename!r}) + spec = spec_from_loader({name!r}, loader) + try: + module = module_from_spec(spec) + loader.exec_module(module) + except ImportError as exc: + text = 'ImportError: ' + str(exc) + else: + text = 'okay' + os.write({fd}, text.encode('utf-8')) + ''') + else: + return textwrap.dedent(f''' + import os, sys + {override_text} + try: + import {name} + except ImportError as exc: + text = 'ImportError: ' + str(exc) + else: + text = 'okay' + os.write({fd}, text.encode('utf-8')) + ''') - def run_here(self, name, *, + def run_here(self, name, filename=None, *, check_singlephase_setting=False, check_singlephase_override=None, isolated=False, @@ -1700,26 +1719,30 @@ def run_here(self, name, *, ) r, w = self.pipe() - script = self.import_script(name, w, check_singlephase_override) + script = self.import_script(name, w, filename, + check_singlephase_override) ret = run_in_subinterp_with_config(script, **kwargs) self.assertEqual(ret, 0) return os.read(r, 100) - def check_compatible_here(self, name, *, strict=False, isolated=False): + def check_compatible_here(self, name, filename=None, *, + strict=False, + isolated=False, + ): # Verify that the named module may be imported in a subinterpreter. # (See run_here() for more info.) - out = self.run_here(name, + out = self.run_here(name, filename, check_singlephase_setting=strict, isolated=isolated, ) self.assertEqual(out, b'okay') - def check_incompatible_here(self, name, *, isolated=False): + def check_incompatible_here(self, name, filename=None, *, isolated=False): # Differences from check_compatible_here(): # * verify that import fails # * "strict" is always True - out = self.run_here(name, + out = self.run_here(name, filename, check_singlephase_setting=True, isolated=isolated, ) @@ -1820,6 +1843,44 @@ def test_multi_init_extension_compat(self): with self.subTest(f'{module}: strict, fresh'): self.check_compatible_fresh(module, strict=True) + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_non_isolated_compat(self): + modname = '_test_non_isolated' + filename = _testmultiphase.__file__ + loader = ExtensionFileLoader(modname, filename) + spec = importlib.util.spec_from_loader(modname, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + sys.modules[modname] = module + + require_extension(module) + with self.subTest(f'{modname}: isolated'): + self.check_incompatible_here(modname, filename, isolated=True) + with self.subTest(f'{modname}: not isolated'): + self.check_incompatible_here(modname, filename, isolated=False) + with self.subTest(f'{modname}: not strict'): + self.check_compatible_here(modname, filename, strict=False) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_per_interpreter_gil_compat(self): + modname = '_test_shared_gil_only' + filename = _testmultiphase.__file__ + loader = ExtensionFileLoader(modname, filename) + spec = importlib.util.spec_from_loader(modname, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + sys.modules[modname] = module + + require_extension(module) + with self.subTest(f'{modname}: isolated, strict'): + self.check_incompatible_here(modname, filename, isolated=True) + with self.subTest(f'{modname}: not isolated, strict'): + self.check_compatible_here(modname, filename, + strict=True, isolated=False) + with self.subTest(f'{modname}: not isolated, not strict'): + self.check_compatible_here(modname, filename, + strict=False, isolated=False) + def test_python_compat(self): module = 'threading' require_pure_python(module) diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 3bf2bbdcdcc4e6..3a74b821eaee49 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -348,6 +348,8 @@ def test_bad_modules(self): 'exec_err', 'exec_raise', 'exec_unreported_exception', + 'multiple_create_slots', + 'multiple_multiple_interpreters_slots', ]: with self.subTest(name_base): name = self.name + '_' + name_base diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 42e3d709bd683f..d2b2f3171e785d 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2111,6 +2111,28 @@ def __dict__(self): self.assertEqual(inspect.getattr_static(foo, 'a'), 3) self.assertFalse(test.called) + def test_mutated_mro(self): + test = self + test.called = False + + class Foo(dict): + a = 3 + @property + def __dict__(self): + test.called = True + return {} + + class Bar(dict): + a = 4 + + class Baz(Bar): pass + + baz = Baz() + self.assertEqual(inspect.getattr_static(baz, 'a'), 4) + Baz.__bases__ = (Foo,) + self.assertEqual(inspect.getattr_static(baz, 'a'), 3) + self.assertFalse(test.called) + def test_custom_object_dict(self): test = self test.called = False @@ -2165,6 +2187,35 @@ class Thing(metaclass=Meta): inspect.getattr_static(Thing, "spam") self.assertFalse(Thing.executed) + def test_custom___getattr__(self): + test = self + test.called = False + + class Foo: + def __getattr__(self, attr): + test.called = True + return {} + + with self.assertRaises(AttributeError): + inspect.getattr_static(Foo(), 'whatever') + + self.assertFalse(test.called) + + def test_custom___getattribute__(self): + test = self + test.called = False + + class Foo: + def __getattribute__(self, attr): + test.called = True + return {} + + with self.assertRaises(AttributeError): + inspect.getattr_static(Foo(), 'really_could_be_anything') + + self.assertFalse(test.called) + + class TestGetGeneratorState(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index a932e03df4236d..ee0ef9a34c385c 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -29,11 +29,12 @@ # class _BasePurePathSubclass(object): - init_called = False + def __init__(self, *pathsegments, session_id): + super().__init__(*pathsegments) + self.session_id = session_id - def __init__(self, *args): - super().__init__(*args) - self.init_called = True + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, session_id=self.session_id) class _BasePurePathTest(object): @@ -121,20 +122,21 @@ def test_str_subclass_common(self): self._check_str_subclass('a/b.txt') self._check_str_subclass('/a/b.txt') - def test_init_called_common(self): + def test_with_segments_common(self): class P(_BasePurePathSubclass, self.cls): pass - p = P('foo', 'bar') - self.assertTrue((p / 'foo').init_called) - self.assertTrue(('foo' / p).init_called) - self.assertTrue(p.joinpath('foo').init_called) - self.assertTrue(p.with_name('foo').init_called) - self.assertTrue(p.with_stem('foo').init_called) - self.assertTrue(p.with_suffix('.foo').init_called) - self.assertTrue(p.relative_to('foo').init_called) - self.assertTrue(p.parent.init_called) + p = P('foo', 'bar', session_id=42) + self.assertEqual(42, (p / 'foo').session_id) + self.assertEqual(42, ('foo' / p).session_id) + self.assertEqual(42, p.joinpath('foo').session_id) + self.assertEqual(42, p.with_name('foo').session_id) + self.assertEqual(42, p.with_stem('foo').session_id) + self.assertEqual(42, p.with_suffix('.foo').session_id) + self.assertEqual(42, p.with_segments('foo').session_id) + self.assertEqual(42, p.relative_to('foo').session_id) + self.assertEqual(42, p.parent.session_id) for parent in p.parents: - self.assertTrue(parent.init_called) + self.assertEqual(42, parent.session_id) def _get_drive_root_parts(self, parts): path = self.cls(*parts) @@ -1647,6 +1649,27 @@ def test_home(self): env['HOME'] = os.path.join(BASE, 'home') self._test_home(self.cls.home()) + def test_with_segments(self): + class P(_BasePurePathSubclass, self.cls): + pass + p = P(BASE, session_id=42) + self.assertEqual(42, p.absolute().session_id) + self.assertEqual(42, p.resolve().session_id) + if not is_wasi: # WASI has no user accounts. + self.assertEqual(42, p.with_segments('~').expanduser().session_id) + self.assertEqual(42, (p / 'fileA').rename(p / 'fileB').session_id) + self.assertEqual(42, (p / 'fileB').replace(p / 'fileA').session_id) + if os_helper.can_symlink(): + self.assertEqual(42, (p / 'linkA').readlink().session_id) + for path in p.iterdir(): + self.assertEqual(42, path.session_id) + for path in p.glob('*'): + self.assertEqual(42, path.session_id) + for path in p.rglob('*'): + self.assertEqual(42, path.session_id) + for dirpath, dirnames, filenames in p.walk(): + self.assertEqual(42, dirpath.session_id) + def test_samefile(self): fileA_path = os.path.join(BASE, 'fileA') fileB_path = os.path.join(BASE, 'dirB', 'fileB') @@ -1830,13 +1853,14 @@ def _check(path, pattern, case_sensitive, expected): def test_rglob_common(self): def _check(glob, expected): - self.assertEqual(set(glob), { P(BASE, q) for q in expected }) + self.assertEqual(sorted(glob), sorted(P(BASE, q) for q in expected)) P = self.cls p = P(BASE) it = p.rglob("fileA") self.assertIsInstance(it, collections.abc.Iterator) _check(it, ["fileA"]) _check(p.rglob("fileB"), ["dirB/fileB"]) + _check(p.rglob("**/fileB"), ["dirB/fileB"]) _check(p.rglob("*/fileA"), []) if not os_helper.can_symlink(): _check(p.rglob("*/fileB"), ["dirB/fileB"]) @@ -1860,9 +1884,12 @@ def _check(glob, expected): _check(p.rglob("*"), ["dirC/fileC", "dirC/novel.txt", "dirC/dirD", "dirC/dirD/fileD"]) _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p.rglob("dir*/**"), ["dirC/dirD"]) _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) _check(p.rglob("*/"), ["dirC/dirD"]) _check(p.rglob(""), ["dirC", "dirC/dirD"]) + _check(p.rglob("**"), ["dirC", "dirC/dirD"]) # gh-91616, a re module regression _check(p.rglob("*.txt"), ["dirC/novel.txt"]) _check(p.rglob("*.*"), ["dirC/novel.txt"]) diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 2b977d78d39324..5418f9f35485f8 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -1,4 +1,5 @@ import unittest +import string import subprocess import sys import sysconfig @@ -70,9 +71,14 @@ def baz(): perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") self.assertTrue(perf_file.exists()) perf_file_contents = perf_file.read_text() - self.assertIn(f"py::foo:{script}", perf_file_contents) - self.assertIn(f"py::bar:{script}", perf_file_contents) - self.assertIn(f"py::baz:{script}", perf_file_contents) + perf_lines = perf_file_contents.splitlines(); + expected_symbols = [f"py::foo:{script}", f"py::bar:{script}", f"py::baz:{script}"] + for expected_symbol in expected_symbols: + perf_line = next((line for line in perf_lines if expected_symbol in line), None) + self.assertIsNotNone(perf_line, f"Could not find {expected_symbol} in perf file") + perf_addr = perf_line.split(" ")[0] + self.assertFalse(perf_addr.startswith("0x"), "Address should not be prefixed with 0x") + self.assertTrue(set(perf_addr).issubset(string.hexdigits), "Address should contain only hex characters") def test_trampoline_works_with_forks(self): code = """if 1: diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 1bb0e13e356e78..328b0467e7fa3d 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1495,6 +1495,14 @@ def test_blob_closed_db_read(self): "Cannot operate on a closed database", blob.read) + def test_blob_32bit_rowid(self): + # gh-100370: we should not get an OverflowError for 32-bit rowids + with memory_database() as cx: + rowid = 2**32 + cx.execute("create table t(t blob)") + cx.execute("insert into t(rowid, t) values (?, zeroblob(1))", (rowid,)) + cx.blobopen('t', 't', rowid) + @threading_helper.requires_working_threading() class ThreadTests(unittest.TestCase): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index fdd74c37e26235..97165264b34bbe 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1349,6 +1349,7 @@ def func(): allow_threads={allowed}, allow_daemon_threads={daemon_allowed}, check_multi_interp_extensions=False, + own_gil=False, ) """) with test.support.SuppressCrashReport(): diff --git a/Lib/turtle.py b/Lib/turtle.py index 2de406e0f517af..cf111158b7c149 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -21,7 +21,6 @@ # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. - """ Turtle graphics is a popular way for introducing programming to kids. It was part of the original Logo programming language developed @@ -97,13 +96,8 @@ Behind the scenes there are some features included with possible extensions in mind. These will be commented and documented elsewhere. - """ -_ver = "turtle 1.1b- - for Python 3.1 - 4. 5. 2009" - -# print(_ver) - import tkinter as TK import types import math @@ -141,7 +135,7 @@ _tg_utilities = ['write_docstringdict', 'done'] __all__ = (_tg_classes + _tg_screen_functions + _tg_turtle_functions + - _tg_utilities + ['Terminator']) # + _math_functions) + _tg_utilities + ['Terminator']) _alias_list = ['addshape', 'backward', 'bk', 'fd', 'ht', 'lt', 'pd', 'pos', 'pu', 'rt', 'seth', 'setpos', 'setposition', 'st', @@ -598,9 +592,6 @@ def _write(self, pos, txt, align, font, pencolor): x0, y0, x1, y1 = self.cv.bbox(item) return item, x1-1 -## def _dot(self, pos, size, color): -## """may be implemented for some other graphics toolkit""" - def _onclick(self, item, fun, num=1, add=None): """Bind fun to mouse-click event on turtle. fun must be a function with two arguments, the coordinates @@ -2726,7 +2717,7 @@ def _cc(self, args): if not ((0 <= r <= 255) and (0 <= g <= 255) and (0 <= b <= 255)): raise TurtleGraphicsError("bad color sequence: %s" % str(args)) return "#%02x%02x%02x" % (r, g, b) - + def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: """Instantly move turtle to an absolute position. @@ -2738,14 +2729,14 @@ def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: call: teleport(x, y) # two coordinates --or: teleport(x) # teleport to x position, keeping y as is --or: teleport(y=y) # teleport to y position, keeping x as is - --or: teleport(x, y, fill_gap=True) + --or: teleport(x, y, fill_gap=True) # teleport but fill the gap in between Move turtle to an absolute position. Unlike goto(x, y), a line will not be drawn. The turtle's orientation does not change. If currently filling, the polygon(s) teleported from will be filled after leaving, and filling will begin again after teleporting. This can be disabled - with fill_gap=True, which makes the imaginary line traveled during + with fill_gap=True, which makes the imaginary line traveled during teleporting act as a fill barrier like in goto(x, y). Example (for a Turtle instance named turtle): @@ -2773,7 +2764,7 @@ def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: self._position = Vec2D(new_x, new_y) self.pen(pendown=pendown) if was_filling and not fill_gap: - self.begin_fill() + self.begin_fill() def clone(self): """Create and return a clone of the turtle. @@ -3455,27 +3446,22 @@ def dot(self, size=None, *color): if size is None: size = self._pensize + max(self._pensize, 4) color = self._colorstr(color) - if hasattr(self.screen, "_dot"): - item = self.screen._dot(self._position, size, color) - self.items.append(item) - if self.undobuffer: - self.undobuffer.push(("dot", item)) - else: - pen = self.pen() - if self.undobuffer: - self.undobuffer.push(["seq"]) - self.undobuffer.cumulate = True - try: - if self.resizemode() == 'auto': - self.ht() - self.pendown() - self.pensize(size) - self.pencolor(color) - self.forward(0) - finally: - self.pen(pen) - if self.undobuffer: - self.undobuffer.cumulate = False + # If screen were to gain a dot function, see GH #104218. + pen = self.pen() + if self.undobuffer: + self.undobuffer.push(["seq"]) + self.undobuffer.cumulate = True + try: + if self.resizemode() == 'auto': + self.ht() + self.pendown() + self.pensize(size) + self.pencolor(color) + self.forward(0) + finally: + self.pen(pen) + if self.undobuffer: + self.undobuffer.cumulate = False def _write(self, txt, align, font): """Performs the writing for write() @@ -3751,11 +3737,6 @@ class _Screen(TurtleScreen): _title = _CFG["title"] def __init__(self): - # XXX there is no need for this code to be conditional, - # as there will be only a single _Screen instance, anyway - # XXX actually, the turtle demo is injecting root window, - # so perhaps the conditional creation of a root should be - # preserved (perhaps by passing it as an optional parameter) if _Screen._root is None: _Screen._root = self._root = _Root() self._root.title(_Screen._title) diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py index caea022da4a688..f6c9d6aa6f9a32 100755 --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -203,10 +203,10 @@ def __init__(self, filename=None): def onResize(self, event): - cwidth = self._canvas.winfo_width() - cheight = self._canvas.winfo_height() - self._canvas.xview_moveto(0.5*(self.canvwidth-cwidth)/self.canvwidth) - self._canvas.yview_moveto(0.5*(self.canvheight-cheight)/self.canvheight) + cwidth = self.canvas.winfo_width() + cheight = self.canvas.winfo_height() + self.canvas.xview_moveto(0.5*(self.canvwidth-cwidth)/self.canvwidth) + self.canvas.yview_moveto(0.5*(self.canvheight-cheight)/self.canvheight) def makeTextFrame(self, root): self.text_frame = text_frame = Frame(root) @@ -237,19 +237,23 @@ def makeTextFrame(self, root): return text_frame def makeGraphFrame(self, root): + # t._Screen is a singleton class instantiated or retrieved + # by calling Screen. Since tdemo canvas needs a different + # configuration, we manually set class attributes before + # calling Screen and manually call superclass init after. turtle._Screen._root = root + self.canvwidth = 1000 self.canvheight = 800 - turtle._Screen._canvas = self._canvas = canvas = turtle.ScrolledCanvas( + turtle._Screen._canvas = self.canvas = canvas = turtle.ScrolledCanvas( root, 800, 600, self.canvwidth, self.canvheight) canvas.adjustScrolls() canvas._rootwindow.bind('', self.onResize) canvas._canvas['borderwidth'] = 0 - self.screen = _s_ = turtle.Screen() - turtle.TurtleScreen.__init__(_s_, _s_._canvas) - self.scanvas = _s_._canvas - turtle.RawTurtle.screens = [_s_] + self.screen = screen = turtle.Screen() + turtle.TurtleScreen.__init__(screen, canvas) + turtle.RawTurtle.screens = [screen] return canvas def set_txtsize(self, size): @@ -373,7 +377,7 @@ def startDemo(self): def clearCanvas(self): self.refreshCanvas() self.screen._delete("all") - self.scanvas.config(cursor="") + self.canvas.config(cursor="") self.configGUI(NORMAL, DISABLED, DISABLED) def stopIt(self): diff --git a/Makefile.pre.in b/Makefile.pre.in index 736a520d0e8fb6..329466580b9cb0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2698,7 +2698,7 @@ MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_MD5.h Modules/_hacl/Hacl_Hash_MD5.c MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA1.h Modules/_hacl/Hacl_Hash_SHA1.c MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_A) -MODULE__SHA3_DEPS=$(srcdir)/Modules/_sha3/sha3.c $(srcdir)/Modules/_sha3/sha3.h $(srcdir)/Modules/hashlib.h +MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA3.h Modules/_hacl/Hacl_Hash_SHA3.c MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h diff --git a/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst b/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst new file mode 100644 index 00000000000000..6fc56d2249f581 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst @@ -0,0 +1,6 @@ +Add unstable C-API functions to get the code object, lasti and line number from +the internal ``_PyInterpreterFrame`` in the limited API. The functions are: + +* ``PyCodeObject * PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame)`` +* ``int PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame)`` +* ``int PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame)`` diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst new file mode 100644 index 00000000000000..5434660e9d6ffb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst @@ -0,0 +1 @@ +Change the perf map format to remove the '0x' prefix from the addresses diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst new file mode 100644 index 00000000000000..dad843636493ae --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst @@ -0,0 +1,6 @@ +Multi-phase init extension modules may now indicate whether or not they +actually support multiple interpreters. By default such modules are +expected to support use in multiple interpreters. In the uncommon case that +one does not, it may use the new ``Py_mod_multiple_interpreters`` module def +slot. A value of ``0`` means the module does not support them. ``1`` means +it does. The default is ``1``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst new file mode 100644 index 00000000000000..afd26750846167 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst @@ -0,0 +1,11 @@ +Multi-phase init extension modules may now indicate that they support +running in subinterpreters that have their own GIL. This is done by using +``Py_MOD_PER_INTERPRETER_GIL_SUPPORTED`` as the value for the +``Py_mod_multiple_interpreters`` module def slot. Otherwise the module, by +default, cannot be imported in such subinterpreters. (This does not affect +the main interpreter or subinterpreters that do not have their own GIL.) In +addition to the isolation that multi-phase init already normally requires, +support for per-interpreter GIL involves one additional constraint: +thread-safety. If the module has external (linked) dependencies and those +libraries have any state that isn't thread-safe then the module must do the +additional work to add thread-safety. This should be an uncommon case. diff --git a/Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst b/Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst new file mode 100644 index 00000000000000..ba113673b7fbe5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst @@ -0,0 +1,3 @@ +Added attribute '_proxy_response_headers' to HTTPConnection class. This +attribute contains the headers of the proxy server response to the CONNECT +request. diff --git a/Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst b/Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst new file mode 100644 index 00000000000000..6539efbc9d0eb0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst @@ -0,0 +1,4 @@ +Deprecation warnings are now emitted for :class:`!ast.Num`, +:class:`!ast.Bytes`, :class:`!ast.Str`, :class:`!ast.NameConstant` and +:class:`!ast.Ellipsis`. These have been documented as deprecated since Python +3.8, and will be removed in Python 3.14. diff --git a/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst b/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst new file mode 100644 index 00000000000000..556db0eae00c0b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst @@ -0,0 +1,2 @@ +Decrease execution time of some :mod:`gzip` file writes by 15% by +adding more appropriate buffering. diff --git a/Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst b/Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst new file mode 100644 index 00000000000000..917cf0f97b9e06 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst @@ -0,0 +1 @@ +:class:`argparse.ArgumentParser` now catches errors when writing messages, such as when :data:`sys.stderr` is ``None``. Patch by Oleg Iarygin. diff --git a/Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst b/Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst new file mode 100644 index 00000000000000..58db90480d2ff0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst @@ -0,0 +1,4 @@ +Add :meth:`pathlib.PurePath.with_segments`, which creates a path object from +arguments. This method is called whenever a derivative path is created, such +as from :attr:`pathlib.PurePath.parent`. Subclasses may override this method +to share information between path objects. diff --git a/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst b/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst new file mode 100644 index 00000000000000..1008ea076c71a0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst @@ -0,0 +1 @@ +Update :mod:`cProfile` to use PEP 669 API diff --git a/Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst b/Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst new file mode 100644 index 00000000000000..238f2868867472 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst @@ -0,0 +1,4 @@ +Improve performance of :meth:`pathlib.Path.absolute` and +:meth:`~pathlib.Path.cwd` by joining paths only when necessary. Also improve +performance of :meth:`pathlib.PurePath.is_absolute` on Posix by skipping path +parsing and normalization. diff --git a/Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst b/Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst new file mode 100644 index 00000000000000..9022d55c48cb11 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst @@ -0,0 +1,2 @@ +Fix potential :exc:`OverflowError` in :meth:`sqlite3.Connection.blobopen` +for 32-bit builds. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst b/Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst new file mode 100644 index 00000000000000..b975d48ed3385c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst @@ -0,0 +1,3 @@ +Optimize :func:`asyncio.gather` when using :func:`asyncio.eager_task_factory` +to complete eagerly if all fututres completed eagerly. +Avoid scheduling done callbacks for futures that complete eagerly. diff --git a/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst b/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst new file mode 100644 index 00000000000000..ced3b7cea04954 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst @@ -0,0 +1,2 @@ +Optimize :class:`asyncio.TaskGroup` when using :func:`asyncio.eager_task_factory`. +Skip scheduling a done callback if a TaskGroup task completes eagerly. diff --git a/Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst b/Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst new file mode 100644 index 00000000000000..54b0190192863c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst @@ -0,0 +1 @@ +Remove unneeded comments and code in turtle.py. diff --git a/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst b/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst new file mode 100644 index 00000000000000..01f8b948d2cb65 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst @@ -0,0 +1,3 @@ +Improve performance of :meth:`pathlib.Path.glob` when expanding recursive +wildcards ("``**``") by merging adjacent wildcards and de-duplicating +results only when necessary. diff --git a/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst b/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst new file mode 100644 index 00000000000000..9c582844bf909b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst @@ -0,0 +1,4 @@ +Prevent possible crash by disallowing instantiation of the +:class:`!_csv.Reader` and :class:`!_csv.Writer` types. +The regression was introduced in 3.10.0a4 with PR 23224 (:issue:`14935`). +Patch by Radislav Chugunov. diff --git a/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst b/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst new file mode 100644 index 00000000000000..f259acf753831c --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst @@ -0,0 +1,2 @@ +Upgrade built-in :mod:`hashlib` SHA3 implementation to a verified implementation +from the ``HACL*`` project. Used when OpenSSL is not present or lacks SHA3. diff --git a/Modules/Setup b/Modules/Setup index 1c6f2f7ea5182d..312e99fea530dc 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -166,7 +166,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_md5 md5module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_MD5.c -D_BSD_SOURCE -D_DEFAULT_SOURCE #_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE #_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a -#_sha3 _sha3/sha3module.c +#_sha3 sha3module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA3.c -D_BSD_SOURCE -D_DEFAULT_SOURCE # text encodings and unicode #_codecs_cn cjkcodecs/_codecs_cn.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index a7803cf7c00e68..8e66576b5c5f00 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -80,7 +80,7 @@ @MODULE__MD5_TRUE@_md5 md5module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_MD5.c -D_BSD_SOURCE -D_DEFAULT_SOURCE @MODULE__SHA1_TRUE@_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE @MODULE__SHA2_TRUE@_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a -@MODULE__SHA3_TRUE@_sha3 _sha3/sha3module.c +@MODULE__SHA3_TRUE@_sha3 sha3module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA3.c -D_BSD_SOURCE -D_DEFAULT_SOURCE @MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c ############################################################################ diff --git a/Modules/_abc.c b/Modules/_abc.c index 9694331339aa78..d3e405dadb664a 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -944,6 +944,7 @@ _abcmodule_free(void *module) static PyModuleDef_Slot _abcmodule_slots[] = { {Py_mod_exec, _abcmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 8b1a29b6d33e8b..3830245abe87b3 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -8,7 +8,6 @@ #include "pycore_runtime_init.h" // _Py_ID() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "structmember.h" // PyMemberDef -#include "cpython/context.h" #include // offsetof() @@ -18,6 +17,10 @@ module _asyncio /*[clinic end generated code: output=da39a3ee5e6b4b0d input=8fd17862aa989c69]*/ +#define FI_FREELIST_MAXLEN 255 + +typedef struct futureiterobject futureiterobject; + /* State of the _asyncio module */ typedef struct { PyTypeObject *FutureIterType; @@ -67,6 +70,9 @@ typedef struct { /* Counter for autogenerated Task names */ uint64_t task_name_counter; + + futureiterobject *fi_freelist; + Py_ssize_t fi_freelist_len; } asyncio_state; static inline asyncio_state * @@ -1574,28 +1580,24 @@ FutureObj_dealloc(PyObject *self) /*********************** Future Iterator **************************/ -typedef struct { +typedef struct futureiterobject { PyObject_HEAD FutureObj *future; } futureiterobject; -#define FI_FREELIST_MAXLEN 255 -static futureiterobject *fi_freelist = NULL; -static Py_ssize_t fi_freelist_len = 0; - - static void FutureIter_dealloc(futureiterobject *it) { PyTypeObject *tp = Py_TYPE(it); + asyncio_state *state = get_asyncio_state_by_def((PyObject *)it); PyObject_GC_UnTrack(it); tp->tp_clear((PyObject *)it); - if (fi_freelist_len < FI_FREELIST_MAXLEN) { - fi_freelist_len++; - it->future = (FutureObj*) fi_freelist; - fi_freelist = it; + if (state->fi_freelist_len < FI_FREELIST_MAXLEN) { + state->fi_freelist_len++; + it->future = (FutureObj*) state->fi_freelist; + state->fi_freelist = it; } else { PyObject_GC_Del(it); @@ -1799,17 +1801,12 @@ future_new_iter(PyObject *fut) futureiterobject *it; asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - if (!Future_Check(state, fut)) { - PyErr_BadInternalCall(); - return NULL; - } - ENSURE_FUTURE_ALIVE(state, fut) - if (fi_freelist_len) { - fi_freelist_len--; - it = fi_freelist; - fi_freelist = (futureiterobject*) it->future; + if (state->fi_freelist_len) { + state->fi_freelist_len--; + it = state->fi_freelist; + state->fi_freelist = (futureiterobject*) it->future; it->future = NULL; _Py_NewReference((PyObject*) it); } @@ -2066,21 +2063,6 @@ swap_current_task(asyncio_state *state, PyObject *loop, PyObject *task) return prev_task; } -static int -is_loop_running(PyObject *loop) -{ - PyObject *func = PyObject_GetAttr(loop, &_Py_ID(is_running)); - if (func == NULL) { - PyErr_Format(PyExc_TypeError, "Loop missing is_running()"); - return -1; - } - PyObject *res = PyObject_CallNoArgs(func); - int retval = Py_IsTrue(res); - Py_DECREF(func); - Py_DECREF(res); - return !!retval; -} - /* ----- Task */ /*[clinic input] @@ -2151,11 +2133,13 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, } if (eager_start) { - int loop_running = is_loop_running(self->task_loop); - if (loop_running == -1) { + PyObject *res = PyObject_CallMethodNoArgs(loop, &_Py_ID(is_running)); + if (res == NULL) { return -1; } - if (loop_running) { + int is_loop_running = Py_IsTrue(res); + Py_DECREF(res); + if (is_loop_running) { if (task_eager_start(state, self)) { return -1; } @@ -3556,22 +3540,22 @@ _asyncio_current_task_impl(PyObject *module, PyObject *loop) static void -module_free_freelists(void) +module_free_freelists(asyncio_state *state) { PyObject *next; PyObject *current; - next = (PyObject*) fi_freelist; + next = (PyObject*) state->fi_freelist; while (next != NULL) { - assert(fi_freelist_len > 0); - fi_freelist_len--; + assert(state->fi_freelist_len > 0); + state->fi_freelist_len--; current = next; next = (PyObject*) ((futureiterobject*) current)->future; PyObject_GC_Del(current); } - assert(fi_freelist_len == 0); - fi_freelist = NULL; + assert(state->fi_freelist_len == 0); + state->fi_freelist = NULL; } static int @@ -3603,7 +3587,7 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) Py_VISIT(state->context_kwname); // Visit freelist. - PyObject *next = (PyObject*) fi_freelist; + PyObject *next = (PyObject*) state->fi_freelist; while (next != NULL) { PyObject *current = next; Py_VISIT(current); @@ -3640,7 +3624,7 @@ module_clear(PyObject *mod) Py_CLEAR(state->context_kwname); - module_free_freelists(); + module_free_freelists(state); return 0; } @@ -3805,6 +3789,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 30801c2f87eee7..0773bbd191931d 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -457,6 +457,7 @@ bisect_modexec(PyObject *m) static PyModuleDef_Slot bisect_slots[] = { {Py_mod_exec, bisect_modexec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c index 44d783b40d0453..0d1d88c6603684 100644 --- a/Modules/_blake2/blake2module.c +++ b/Modules/_blake2/blake2module.c @@ -127,6 +127,7 @@ blake2_exec(PyObject *m) static PyModuleDef_Slot _blake2_slots[] = { {Py_mod_exec, blake2_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; @@ -146,4 +147,4 @@ PyMODINIT_FUNC PyInit__blake2(void) { return PyModuleDef_Init(&blake2_module); -} \ No newline at end of file +} diff --git a/Modules/_blake2/blake2module.h b/Modules/_blake2/blake2module.h index aa8f281178eadc..c8144ec9d48d29 100644 --- a/Modules/_blake2/blake2module.h +++ b/Modules/_blake2/blake2module.h @@ -38,6 +38,6 @@ #endif // HAVE_LIBB2 // for secure_zero_memory(), store32(), store48(), and store64() -#include "impl/blake2-impl.h" +#include "impl/blake2-impl.h" #endif // Py_BLAKE2MODULE_H diff --git a/Modules/_blake2/impl/blake2b-round.h b/Modules/_blake2/impl/blake2b-round.h index cebc22550da4cd..5b452c4d63babe 100644 --- a/Modules/_blake2/impl/blake2b-round.h +++ b/Modules/_blake2/impl/blake2b-round.h @@ -62,7 +62,7 @@ \ row2l = _mm_roti_epi64(row2l, -24); \ row2h = _mm_roti_epi64(row2h, -24); \ - + #define G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1) \ row1l = _mm_add_epi64(_mm_add_epi64(row1l, b0), row2l); \ row1h = _mm_add_epi64(_mm_add_epi64(row1h, b1), row2h); \ @@ -81,7 +81,7 @@ \ row2l = _mm_roti_epi64(row2l, -63); \ row2h = _mm_roti_epi64(row2h, -63); \ - + #if defined(HAVE_SSSE3) #define DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ t0 = _mm_alignr_epi8(row2h, row2l, 8); \ diff --git a/Modules/_blake2/impl/blake2s-load-xop.h b/Modules/_blake2/impl/blake2s-load-xop.h index ac591a77d191a7..14d9e7f7640672 100644 --- a/Modules/_blake2/impl/blake2s-load-xop.h +++ b/Modules/_blake2/impl/blake2s-load-xop.h @@ -166,7 +166,7 @@ buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(1),TOB(7)) ); #define LOAD_MSG_8_3(buf) \ t0 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(6),TOB(1),TOB(0),TOB(0)) ); \ buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(5),TOB(4)) ); \ - + #define LOAD_MSG_8_4(buf) \ buf = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(5),TOB(4),TOB(7),TOB(2)) ); diff --git a/Modules/_blake2/impl/blake2s-round.h b/Modules/_blake2/impl/blake2s-round.h index 1e2f2b7f59bd6c..3af4be35bee5d4 100644 --- a/Modules/_blake2/impl/blake2s-round.h +++ b/Modules/_blake2/impl/blake2s-round.h @@ -86,6 +86,6 @@ LOAD_MSG_ ##r ##_4(buf4); \ G2(row1,row2,row3,row4,buf4); \ UNDIAGONALIZE(row1,row2,row3,row4); \ - + #endif diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 8e7b8e8078af4e..97bd44b4ac9694 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -799,6 +799,7 @@ _bz2_free(void *module) static struct PyModuleDef_Slot _bz2_slots[] = { {Py_mod_exec, _bz2_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index d5035d20600ae2..777c753bd7c2a9 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -1049,6 +1049,7 @@ static PyMethodDef _codecs_functions[] = { }; static PyModuleDef_Slot _codecs_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index a9b1425177c3d7..9a81531bdffb16 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2571,6 +2571,7 @@ collections_exec(PyObject *module) { static struct PyModuleDef_Slot collections_slots[] = { {Py_mod_exec, collections_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_contextvarsmodule.c b/Modules/_contextvarsmodule.c index d13b5962c13c44..f621c1de6d42d6 100644 --- a/Modules/_contextvarsmodule.c +++ b/Modules/_contextvarsmodule.c @@ -44,6 +44,7 @@ _contextvars_exec(PyObject *m) static struct PyModuleDef_Slot _contextvars_slots[] = { {Py_mod_exec, _contextvars_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_cryptmodule.c b/Modules/_cryptmodule.c index 72a4f44600d92c..75035084c9cd29 100644 --- a/Modules/_cryptmodule.c +++ b/Modules/_cryptmodule.c @@ -58,6 +58,7 @@ static PyMethodDef crypt_methods[] = { }; static PyModuleDef_Slot _crypt_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_csv.c b/Modules/_csv.c index 2217cc2ca7a775..9ab2ad266c2739 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -1000,7 +1000,7 @@ PyType_Spec Reader_Type_spec = { .name = "_csv.reader", .basicsize = sizeof(ReaderObj), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), .slots = Reader_Type_slots }; @@ -1431,7 +1431,7 @@ PyType_Spec Writer_Type_spec = { .name = "_csv.writer", .basicsize = sizeof(WriterObj), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), .slots = Writer_Type_slots, }; @@ -1798,6 +1798,7 @@ csv_exec(PyObject *module) { static PyModuleDef_Slot csv_slots[] = { {Py_mod_exec, csv_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index a8811d03cc91a2..ddfb2c8a332a9e 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1036,7 +1036,7 @@ EXPORT (HRESULT) KeepObject(IUnknown *punk) #ifdef MS_WIN32 -// i38748: c stub for testing stack corruption +// i38748: c stub for testing stack corruption // When executing a Python callback with a long and a long long typedef long(__stdcall *_test_i38748_funcType)(long, long long); @@ -1054,6 +1054,7 @@ _testfunc_pylist_append(PyObject *list, PyObject *item) } static struct PyModuleDef_Slot _ctypes_test_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 2144345de01ba3..a3124ff80551e0 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -690,6 +690,9 @@ _curses_panel_exec(PyObject *mod) static PyModuleDef_Slot _curses_slots[] = { {Py_mod_exec, _curses_panel_exec}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 54376022dcb182..9908174c94c450 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -583,6 +583,7 @@ _dbm_module_free(void *module) static PyModuleDef_Slot _dbmmodule_slots[] = { {Py_mod_exec, _dbm_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 97be89a167104f..42de3c675c2e5a 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -4419,6 +4419,9 @@ module_exec(PyObject *m) static struct PyModuleDef_Slot elementtree_slots[] = { {Py_mod_exec, module_exec}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 4032ba79374fa4..a8001d71223fdc 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -1520,6 +1520,7 @@ _functools_free(void *module) static struct PyModuleDef_Slot _functools_slots[] = { {Py_mod_exec, _functools_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index 4e8acdefc722b2..4dbb5741b2ede8 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -793,6 +793,7 @@ _gdbm_module_free(void *module) static PyModuleDef_Slot _gdbm_module_slots[] = { {Py_mod_exec, _gdbm_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_hacl/Hacl_Hash_SHA3.c b/Modules/_hacl/Hacl_Hash_SHA3.c new file mode 100644 index 00000000000000..100afe7c2c6d1f --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_SHA3.c @@ -0,0 +1,826 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#include "internal/Hacl_Hash_SHA3.h" + +static uint32_t block_len(Spec_Hash_Definitions_hash_alg a) +{ + switch (a) + { + case Spec_Hash_Definitions_SHA3_224: + { + return (uint32_t)144U; + } + case Spec_Hash_Definitions_SHA3_256: + { + return (uint32_t)136U; + } + case Spec_Hash_Definitions_SHA3_384: + { + return (uint32_t)104U; + } + case Spec_Hash_Definitions_SHA3_512: + { + return (uint32_t)72U; + } + case Spec_Hash_Definitions_Shake128: + { + return (uint32_t)168U; + } + case Spec_Hash_Definitions_Shake256: + { + return (uint32_t)136U; + } + default: + { + KRML_HOST_EPRINTF("KaRaMeL incomplete match at %s:%d\n", __FILE__, __LINE__); + KRML_HOST_EXIT(253U); + } + } +} + +static uint32_t hash_len(Spec_Hash_Definitions_hash_alg a) +{ + switch (a) + { + case Spec_Hash_Definitions_SHA3_224: + { + return (uint32_t)28U; + } + case Spec_Hash_Definitions_SHA3_256: + { + return (uint32_t)32U; + } + case Spec_Hash_Definitions_SHA3_384: + { + return (uint32_t)48U; + } + case Spec_Hash_Definitions_SHA3_512: + { + return (uint32_t)64U; + } + default: + { + KRML_HOST_EPRINTF("KaRaMeL incomplete match at %s:%d\n", __FILE__, __LINE__); + KRML_HOST_EXIT(253U); + } + } +} + +void +Hacl_Hash_SHA3_update_multi_sha3( + Spec_Hash_Definitions_hash_alg a, + uint64_t *s, + uint8_t *blocks, + uint32_t n_blocks +) +{ + for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + { + uint8_t *block = blocks + i * block_len(a); + Hacl_Impl_SHA3_absorb_inner(block_len(a), block, s); + } +} + +void +Hacl_Hash_SHA3_update_last_sha3( + Spec_Hash_Definitions_hash_alg a, + uint64_t *s, + uint8_t *input, + uint32_t input_len +) +{ + uint8_t suffix; + if (a == Spec_Hash_Definitions_Shake128 || a == Spec_Hash_Definitions_Shake256) + { + suffix = (uint8_t)0x1fU; + } + else + { + suffix = (uint8_t)0x06U; + } + uint32_t len = block_len(a); + if (input_len == len) + { + Hacl_Impl_SHA3_absorb_inner(len, input, s); + uint8_t *uu____0 = input + input_len; + uint8_t lastBlock_[200U] = { 0U }; + uint8_t *lastBlock = lastBlock_; + memcpy(lastBlock, uu____0, (uint32_t)0U * sizeof (uint8_t)); + lastBlock[0U] = suffix; + Hacl_Impl_SHA3_loadState(len, lastBlock, s); + if (!((suffix & (uint8_t)0x80U) == (uint8_t)0U) && (uint32_t)0U == len - (uint32_t)1U) + { + Hacl_Impl_SHA3_state_permute(s); + } + uint8_t nextBlock_[200U] = { 0U }; + uint8_t *nextBlock = nextBlock_; + nextBlock[len - (uint32_t)1U] = (uint8_t)0x80U; + Hacl_Impl_SHA3_loadState(len, nextBlock, s); + Hacl_Impl_SHA3_state_permute(s); + return; + } + uint8_t lastBlock_[200U] = { 0U }; + uint8_t *lastBlock = lastBlock_; + memcpy(lastBlock, input, input_len * sizeof (uint8_t)); + lastBlock[input_len] = suffix; + Hacl_Impl_SHA3_loadState(len, lastBlock, s); + if (!((suffix & (uint8_t)0x80U) == (uint8_t)0U) && input_len == len - (uint32_t)1U) + { + Hacl_Impl_SHA3_state_permute(s); + } + uint8_t nextBlock_[200U] = { 0U }; + uint8_t *nextBlock = nextBlock_; + nextBlock[len - (uint32_t)1U] = (uint8_t)0x80U; + Hacl_Impl_SHA3_loadState(len, nextBlock, s); + Hacl_Impl_SHA3_state_permute(s); +} + +typedef struct hash_buf2_s +{ + Hacl_Streaming_Keccak_hash_buf fst; + Hacl_Streaming_Keccak_hash_buf snd; +} +hash_buf2; + +Spec_Hash_Definitions_hash_alg Hacl_Streaming_Keccak_get_alg(Hacl_Streaming_Keccak_state *s) +{ + Hacl_Streaming_Keccak_state scrut = *s; + Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; + return block_state.fst; +} + +Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_hash_alg a) +{ + KRML_CHECK_SIZE(sizeof (uint8_t), block_len(a)); + uint8_t *buf0 = (uint8_t *)KRML_HOST_CALLOC(block_len(a), sizeof (uint8_t)); + uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC((uint32_t)25U, sizeof (uint64_t)); + Hacl_Streaming_Keccak_hash_buf block_state = { .fst = a, .snd = buf }; + Hacl_Streaming_Keccak_state + s = { .block_state = block_state, .buf = buf0, .total_len = (uint64_t)(uint32_t)0U }; + Hacl_Streaming_Keccak_state + *p = (Hacl_Streaming_Keccak_state *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_Keccak_state)); + p[0U] = s; + uint64_t *s1 = block_state.snd; + for (uint32_t _i = 0U; _i < (uint32_t)25U; ++_i) + ((void **)s1)[_i] = (void *)(uint64_t)0U; + return p; +} + +void Hacl_Streaming_Keccak_free(Hacl_Streaming_Keccak_state *s) +{ + Hacl_Streaming_Keccak_state scrut = *s; + uint8_t *buf = scrut.buf; + Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; + uint64_t *s1 = block_state.snd; + KRML_HOST_FREE(s1); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(s); +} + +Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_copy(Hacl_Streaming_Keccak_state *s0) +{ + Hacl_Streaming_Keccak_state scrut0 = *s0; + Hacl_Streaming_Keccak_hash_buf block_state0 = scrut0.block_state; + uint8_t *buf0 = scrut0.buf; + uint64_t total_len0 = scrut0.total_len; + Spec_Hash_Definitions_hash_alg i = block_state0.fst; + KRML_CHECK_SIZE(sizeof (uint8_t), block_len(i)); + uint8_t *buf1 = (uint8_t *)KRML_HOST_CALLOC(block_len(i), sizeof (uint8_t)); + memcpy(buf1, buf0, block_len(i) * sizeof (uint8_t)); + uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC((uint32_t)25U, sizeof (uint64_t)); + Hacl_Streaming_Keccak_hash_buf block_state = { .fst = i, .snd = buf }; + hash_buf2 scrut = { .fst = block_state0, .snd = block_state }; + uint64_t *s_dst = scrut.snd.snd; + uint64_t *s_src = scrut.fst.snd; + memcpy(s_dst, s_src, (uint32_t)25U * sizeof (uint64_t)); + Hacl_Streaming_Keccak_state + s = { .block_state = block_state, .buf = buf1, .total_len = total_len0 }; + Hacl_Streaming_Keccak_state + *p = (Hacl_Streaming_Keccak_state *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_Keccak_state)); + p[0U] = s; + return p; +} + +void Hacl_Streaming_Keccak_reset(Hacl_Streaming_Keccak_state *s) +{ + Hacl_Streaming_Keccak_state scrut = *s; + uint8_t *buf = scrut.buf; + Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; + uint64_t *s1 = block_state.snd; + for (uint32_t _i = 0U; _i < (uint32_t)25U; ++_i) + ((void **)s1)[_i] = (void *)(uint64_t)0U; + Hacl_Streaming_Keccak_state + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s[0U] = tmp; +} + +uint32_t +Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint32_t len) +{ + Hacl_Streaming_Keccak_state s = *p; + Hacl_Streaming_Keccak_hash_buf block_state = s.block_state; + uint64_t total_len = s.total_len; + Spec_Hash_Definitions_hash_alg i = block_state.fst; + if ((uint64_t)len > (uint64_t)0xffffffffU - total_len) + { + return (uint32_t)1U; + } + uint32_t sz; + if (total_len % (uint64_t)block_len(i) == (uint64_t)0U && total_len > (uint64_t)0U) + { + sz = block_len(i); + } + else + { + sz = (uint32_t)(total_len % (uint64_t)block_len(i)); + } + if (len <= block_len(i) - sz) + { + Hacl_Streaming_Keccak_state s1 = *p; + Hacl_Streaming_Keccak_hash_buf block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = block_len(i); + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, data, len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)len; + *p + = + ( + (Hacl_Streaming_Keccak_state){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == (uint32_t)0U) + { + Hacl_Streaming_Keccak_state s1 = *p; + Hacl_Streaming_Keccak_hash_buf block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = block_len(i); + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); + } + if (!(sz1 == (uint32_t)0U)) + { + Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; + uint64_t *s2 = block_state1.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, buf, block_len(i) / block_len(a1)); + } + uint32_t ite; + if ((uint64_t)len % (uint64_t)block_len(i) == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + { + ite = block_len(i); + } + else + { + ite = (uint32_t)((uint64_t)len % (uint64_t)block_len(i)); + } + uint32_t n_blocks = (len - ite) / block_len(i); + uint32_t data1_len = n_blocks * block_len(i); + uint32_t data2_len = len - data1_len; + uint8_t *data1 = data; + uint8_t *data2 = data + data1_len; + Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; + uint64_t *s2 = block_state1.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data1, data1_len / block_len(a1)); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_Keccak_state){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)len + } + ); + } + else + { + uint32_t diff = block_len(i) - sz; + uint8_t *data1 = data; + uint8_t *data2 = data + diff; + Hacl_Streaming_Keccak_state s1 = *p; + Hacl_Streaming_Keccak_hash_buf block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)block_len(i) == (uint64_t)0U && total_len10 > (uint64_t)0U) + { + sz10 = block_len(i); + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)block_len(i)); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, data1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *p + = + ( + (Hacl_Streaming_Keccak_state){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Streaming_Keccak_state s10 = *p; + Hacl_Streaming_Keccak_hash_buf block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = block_len(i); + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); + } + if (!(sz1 == (uint32_t)0U)) + { + Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; + uint64_t *s2 = block_state1.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, buf, block_len(i) / block_len(a1)); + } + uint32_t ite; + if + ( + (uint64_t)(len - diff) + % (uint64_t)block_len(i) + == (uint64_t)0U + && (uint64_t)(len - diff) > (uint64_t)0U + ) + { + ite = block_len(i); + } + else + { + ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)block_len(i)); + } + uint32_t n_blocks = (len - diff - ite) / block_len(i); + uint32_t data1_len = n_blocks * block_len(i); + uint32_t data2_len = len - diff - data1_len; + uint8_t *data11 = data2; + uint8_t *data21 = data2 + data1_len; + Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; + uint64_t *s2 = block_state1.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data11, data1_len / block_len(a1)); + uint8_t *dst = buf; + memcpy(dst, data21, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_Keccak_state){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(len - diff) + } + ); + } + return (uint32_t)0U; +} + +static void +finish_( + Spec_Hash_Definitions_hash_alg a, + Hacl_Streaming_Keccak_state *p, + uint8_t *dst, + uint32_t l +) +{ + Hacl_Streaming_Keccak_state scrut0 = *p; + Hacl_Streaming_Keccak_hash_buf block_state = scrut0.block_state; + uint8_t *buf_ = scrut0.buf; + uint64_t total_len = scrut0.total_len; + uint32_t r; + if (total_len % (uint64_t)block_len(a) == (uint64_t)0U && total_len > (uint64_t)0U) + { + r = block_len(a); + } + else + { + r = (uint32_t)(total_len % (uint64_t)block_len(a)); + } + uint8_t *buf_1 = buf_; + uint64_t buf[25U] = { 0U }; + Hacl_Streaming_Keccak_hash_buf tmp_block_state = { .fst = a, .snd = buf }; + hash_buf2 scrut = { .fst = block_state, .snd = tmp_block_state }; + uint64_t *s_dst = scrut.snd.snd; + uint64_t *s_src = scrut.fst.snd; + memcpy(s_dst, s_src, (uint32_t)25U * sizeof (uint64_t)); + uint32_t ite0; + if (r % block_len(a) == (uint32_t)0U && r > (uint32_t)0U) + { + ite0 = block_len(a); + } + else + { + ite0 = r % block_len(a); + } + uint8_t *buf_last = buf_1 + r - ite0; + uint8_t *buf_multi = buf_1; + Spec_Hash_Definitions_hash_alg a1 = tmp_block_state.fst; + uint64_t *s0 = tmp_block_state.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s0, buf_multi, (uint32_t)0U / block_len(a1)); + Spec_Hash_Definitions_hash_alg a10 = tmp_block_state.fst; + uint64_t *s1 = tmp_block_state.snd; + Hacl_Hash_SHA3_update_last_sha3(a10, s1, buf_last, r); + Spec_Hash_Definitions_hash_alg a11 = tmp_block_state.fst; + uint64_t *s = tmp_block_state.snd; + if (a11 == Spec_Hash_Definitions_Shake128 || a11 == Spec_Hash_Definitions_Shake256) + { + uint32_t ite; + if (a11 == Spec_Hash_Definitions_Shake128 || a11 == Spec_Hash_Definitions_Shake256) + { + ite = l; + } + else + { + ite = hash_len(a11); + } + Hacl_Impl_SHA3_squeeze(s, block_len(a11), ite, dst); + return; + } + Hacl_Impl_SHA3_squeeze(s, block_len(a11), hash_len(a11), dst); +} + +Hacl_Streaming_Keccak_error_code +Hacl_Streaming_Keccak_finish(Hacl_Streaming_Keccak_state *s, uint8_t *dst) +{ + Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + if (a1 == Spec_Hash_Definitions_Shake128 || a1 == Spec_Hash_Definitions_Shake256) + { + return Hacl_Streaming_Keccak_InvalidAlgorithm; + } + finish_(a1, s, dst, hash_len(a1)); + return Hacl_Streaming_Keccak_Success; +} + +Hacl_Streaming_Keccak_error_code +Hacl_Streaming_Keccak_squeeze(Hacl_Streaming_Keccak_state *s, uint8_t *dst, uint32_t l) +{ + Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + if (!(a1 == Spec_Hash_Definitions_Shake128 || a1 == Spec_Hash_Definitions_Shake256)) + { + return Hacl_Streaming_Keccak_InvalidAlgorithm; + } + if (l == (uint32_t)0U) + { + return Hacl_Streaming_Keccak_InvalidLength; + } + finish_(a1, s, dst, l); + return Hacl_Streaming_Keccak_Success; +} + +uint32_t Hacl_Streaming_Keccak_block_len(Hacl_Streaming_Keccak_state *s) +{ + Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + return block_len(a1); +} + +uint32_t Hacl_Streaming_Keccak_hash_len(Hacl_Streaming_Keccak_state *s) +{ + Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + return hash_len(a1); +} + +bool Hacl_Streaming_Keccak_is_shake(Hacl_Streaming_Keccak_state *s) +{ + Spec_Hash_Definitions_hash_alg uu____0 = Hacl_Streaming_Keccak_get_alg(s); + return uu____0 == Spec_Hash_Definitions_Shake128 || uu____0 == Spec_Hash_Definitions_Shake256; +} + +void +Hacl_SHA3_shake128_hacl( + uint32_t inputByteLen, + uint8_t *input, + uint32_t outputByteLen, + uint8_t *output +) +{ + Hacl_Impl_SHA3_keccak((uint32_t)1344U, + (uint32_t)256U, + inputByteLen, + input, + (uint8_t)0x1FU, + outputByteLen, + output); +} + +void +Hacl_SHA3_shake256_hacl( + uint32_t inputByteLen, + uint8_t *input, + uint32_t outputByteLen, + uint8_t *output +) +{ + Hacl_Impl_SHA3_keccak((uint32_t)1088U, + (uint32_t)512U, + inputByteLen, + input, + (uint8_t)0x1FU, + outputByteLen, + output); +} + +void Hacl_SHA3_sha3_224(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +{ + Hacl_Impl_SHA3_keccak((uint32_t)1152U, + (uint32_t)448U, + inputByteLen, + input, + (uint8_t)0x06U, + (uint32_t)28U, + output); +} + +void Hacl_SHA3_sha3_256(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +{ + Hacl_Impl_SHA3_keccak((uint32_t)1088U, + (uint32_t)512U, + inputByteLen, + input, + (uint8_t)0x06U, + (uint32_t)32U, + output); +} + +void Hacl_SHA3_sha3_384(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +{ + Hacl_Impl_SHA3_keccak((uint32_t)832U, + (uint32_t)768U, + inputByteLen, + input, + (uint8_t)0x06U, + (uint32_t)48U, + output); +} + +void Hacl_SHA3_sha3_512(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +{ + Hacl_Impl_SHA3_keccak((uint32_t)576U, + (uint32_t)1024U, + inputByteLen, + input, + (uint8_t)0x06U, + (uint32_t)64U, + output); +} + +static const +uint32_t +keccak_rotc[24U] = + { + (uint32_t)1U, (uint32_t)3U, (uint32_t)6U, (uint32_t)10U, (uint32_t)15U, (uint32_t)21U, + (uint32_t)28U, (uint32_t)36U, (uint32_t)45U, (uint32_t)55U, (uint32_t)2U, (uint32_t)14U, + (uint32_t)27U, (uint32_t)41U, (uint32_t)56U, (uint32_t)8U, (uint32_t)25U, (uint32_t)43U, + (uint32_t)62U, (uint32_t)18U, (uint32_t)39U, (uint32_t)61U, (uint32_t)20U, (uint32_t)44U + }; + +static const +uint32_t +keccak_piln[24U] = + { + (uint32_t)10U, (uint32_t)7U, (uint32_t)11U, (uint32_t)17U, (uint32_t)18U, (uint32_t)3U, + (uint32_t)5U, (uint32_t)16U, (uint32_t)8U, (uint32_t)21U, (uint32_t)24U, (uint32_t)4U, + (uint32_t)15U, (uint32_t)23U, (uint32_t)19U, (uint32_t)13U, (uint32_t)12U, (uint32_t)2U, + (uint32_t)20U, (uint32_t)14U, (uint32_t)22U, (uint32_t)9U, (uint32_t)6U, (uint32_t)1U + }; + +static const +uint64_t +keccak_rndc[24U] = + { + (uint64_t)0x0000000000000001U, (uint64_t)0x0000000000008082U, (uint64_t)0x800000000000808aU, + (uint64_t)0x8000000080008000U, (uint64_t)0x000000000000808bU, (uint64_t)0x0000000080000001U, + (uint64_t)0x8000000080008081U, (uint64_t)0x8000000000008009U, (uint64_t)0x000000000000008aU, + (uint64_t)0x0000000000000088U, (uint64_t)0x0000000080008009U, (uint64_t)0x000000008000000aU, + (uint64_t)0x000000008000808bU, (uint64_t)0x800000000000008bU, (uint64_t)0x8000000000008089U, + (uint64_t)0x8000000000008003U, (uint64_t)0x8000000000008002U, (uint64_t)0x8000000000000080U, + (uint64_t)0x000000000000800aU, (uint64_t)0x800000008000000aU, (uint64_t)0x8000000080008081U, + (uint64_t)0x8000000000008080U, (uint64_t)0x0000000080000001U, (uint64_t)0x8000000080008008U + }; + +void Hacl_Impl_SHA3_state_permute(uint64_t *s) +{ + for (uint32_t i0 = (uint32_t)0U; i0 < (uint32_t)24U; i0++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + _C[i] = + s[i + + (uint32_t)0U] + ^ + (s[i + + (uint32_t)5U] + ^ (s[i + (uint32_t)10U] ^ (s[i + (uint32_t)15U] ^ s[i + (uint32_t)20U])));); + KRML_MAYBE_FOR5(i1, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + uint64_t uu____0 = _C[(i1 + (uint32_t)1U) % (uint32_t)5U]; + uint64_t + _D = + _C[(i1 + (uint32_t)4U) + % (uint32_t)5U] + ^ (uu____0 << (uint32_t)1U | uu____0 >> (uint32_t)63U); + KRML_MAYBE_FOR5(i, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + s[i1 + (uint32_t)5U * i] = s[i1 + (uint32_t)5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = (uint32_t)0U; i < (uint32_t)24U; i++) + { + uint32_t _Y = keccak_piln[i]; + uint32_t r = keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> ((uint32_t)64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + uint64_t + v0 = + s[(uint32_t)0U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)1U + (uint32_t)5U * i] & s[(uint32_t)2U + (uint32_t)5U * i]); + uint64_t + v1 = + s[(uint32_t)1U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)2U + (uint32_t)5U * i] & s[(uint32_t)3U + (uint32_t)5U * i]); + uint64_t + v2 = + s[(uint32_t)2U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)3U + (uint32_t)5U * i] & s[(uint32_t)4U + (uint32_t)5U * i]); + uint64_t + v3 = + s[(uint32_t)3U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)4U + (uint32_t)5U * i] & s[(uint32_t)0U + (uint32_t)5U * i]); + uint64_t + v4 = + s[(uint32_t)4U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)0U + (uint32_t)5U * i] & s[(uint32_t)1U + (uint32_t)5U * i]); + s[(uint32_t)0U + (uint32_t)5U * i] = v0; + s[(uint32_t)1U + (uint32_t)5U * i] = v1; + s[(uint32_t)2U + (uint32_t)5U * i] = v2; + s[(uint32_t)3U + (uint32_t)5U * i] = v3; + s[(uint32_t)4U + (uint32_t)5U * i] = v4;); + uint64_t c = keccak_rndc[i0]; + s[0U] = s[0U] ^ c; + } +} + +void Hacl_Impl_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s) +{ + uint8_t block[200U] = { 0U }; + memcpy(block, input, rateInBytes * sizeof (uint8_t)); + for (uint32_t i = (uint32_t)0U; i < (uint32_t)25U; i++) + { + uint64_t u = load64_le(block + i * (uint32_t)8U); + uint64_t x = u; + s[i] = s[i] ^ x; + } +} + +static void storeState(uint32_t rateInBytes, uint64_t *s, uint8_t *res) +{ + uint8_t block[200U] = { 0U }; + for (uint32_t i = (uint32_t)0U; i < (uint32_t)25U; i++) + { + uint64_t sj = s[i]; + store64_le(block + i * (uint32_t)8U, sj); + } + memcpy(res, block, rateInBytes * sizeof (uint8_t)); +} + +void Hacl_Impl_SHA3_absorb_inner(uint32_t rateInBytes, uint8_t *block, uint64_t *s) +{ + Hacl_Impl_SHA3_loadState(rateInBytes, block, s); + Hacl_Impl_SHA3_state_permute(s); +} + +static void +absorb( + uint64_t *s, + uint32_t rateInBytes, + uint32_t inputByteLen, + uint8_t *input, + uint8_t delimitedSuffix +) +{ + uint32_t n_blocks = inputByteLen / rateInBytes; + uint32_t rem = inputByteLen % rateInBytes; + for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + { + uint8_t *block = input + i * rateInBytes; + Hacl_Impl_SHA3_absorb_inner(rateInBytes, block, s); + } + uint8_t *last = input + n_blocks * rateInBytes; + uint8_t lastBlock_[200U] = { 0U }; + uint8_t *lastBlock = lastBlock_; + memcpy(lastBlock, last, rem * sizeof (uint8_t)); + lastBlock[rem] = delimitedSuffix; + Hacl_Impl_SHA3_loadState(rateInBytes, lastBlock, s); + if (!((delimitedSuffix & (uint8_t)0x80U) == (uint8_t)0U) && rem == rateInBytes - (uint32_t)1U) + { + Hacl_Impl_SHA3_state_permute(s); + } + uint8_t nextBlock_[200U] = { 0U }; + uint8_t *nextBlock = nextBlock_; + nextBlock[rateInBytes - (uint32_t)1U] = (uint8_t)0x80U; + Hacl_Impl_SHA3_loadState(rateInBytes, nextBlock, s); + Hacl_Impl_SHA3_state_permute(s); +} + +void +Hacl_Impl_SHA3_squeeze( + uint64_t *s, + uint32_t rateInBytes, + uint32_t outputByteLen, + uint8_t *output +) +{ + uint32_t outBlocks = outputByteLen / rateInBytes; + uint32_t remOut = outputByteLen % rateInBytes; + uint8_t *last = output + outputByteLen - remOut; + uint8_t *blocks = output; + for (uint32_t i = (uint32_t)0U; i < outBlocks; i++) + { + storeState(rateInBytes, s, blocks + i * rateInBytes); + Hacl_Impl_SHA3_state_permute(s); + } + storeState(remOut, s, last); +} + +void +Hacl_Impl_SHA3_keccak( + uint32_t rate, + uint32_t capacity, + uint32_t inputByteLen, + uint8_t *input, + uint8_t delimitedSuffix, + uint32_t outputByteLen, + uint8_t *output +) +{ + uint32_t rateInBytes = rate / (uint32_t)8U; + uint64_t s[25U] = { 0U }; + absorb(s, rateInBytes, inputByteLen, input, delimitedSuffix); + Hacl_Impl_SHA3_squeeze(s, rateInBytes, outputByteLen, output); +} + diff --git a/Modules/_hacl/Hacl_Hash_SHA3.h b/Modules/_hacl/Hacl_Hash_SHA3.h new file mode 100644 index 00000000000000..2a5cf4b1844b9d --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_SHA3.h @@ -0,0 +1,136 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Hash_SHA3_H +#define __Hacl_Hash_SHA3_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "Hacl_Streaming_Types.h" + +typedef struct Hacl_Streaming_Keccak_hash_buf_s +{ + Spec_Hash_Definitions_hash_alg fst; + uint64_t *snd; +} +Hacl_Streaming_Keccak_hash_buf; + +typedef struct Hacl_Streaming_Keccak_state_s +{ + Hacl_Streaming_Keccak_hash_buf block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Streaming_Keccak_state; + +Spec_Hash_Definitions_hash_alg Hacl_Streaming_Keccak_get_alg(Hacl_Streaming_Keccak_state *s); + +Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_hash_alg a); + +void Hacl_Streaming_Keccak_free(Hacl_Streaming_Keccak_state *s); + +Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_copy(Hacl_Streaming_Keccak_state *s0); + +void Hacl_Streaming_Keccak_reset(Hacl_Streaming_Keccak_state *s); + +uint32_t +Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint32_t len); + +#define Hacl_Streaming_Keccak_Success 0 +#define Hacl_Streaming_Keccak_InvalidAlgorithm 1 +#define Hacl_Streaming_Keccak_InvalidLength 2 + +typedef uint8_t Hacl_Streaming_Keccak_error_code; + +Hacl_Streaming_Keccak_error_code +Hacl_Streaming_Keccak_finish(Hacl_Streaming_Keccak_state *s, uint8_t *dst); + +Hacl_Streaming_Keccak_error_code +Hacl_Streaming_Keccak_squeeze(Hacl_Streaming_Keccak_state *s, uint8_t *dst, uint32_t l); + +uint32_t Hacl_Streaming_Keccak_block_len(Hacl_Streaming_Keccak_state *s); + +uint32_t Hacl_Streaming_Keccak_hash_len(Hacl_Streaming_Keccak_state *s); + +bool Hacl_Streaming_Keccak_is_shake(Hacl_Streaming_Keccak_state *s); + +void +Hacl_SHA3_shake128_hacl( + uint32_t inputByteLen, + uint8_t *input, + uint32_t outputByteLen, + uint8_t *output +); + +void +Hacl_SHA3_shake256_hacl( + uint32_t inputByteLen, + uint8_t *input, + uint32_t outputByteLen, + uint8_t *output +); + +void Hacl_SHA3_sha3_224(uint32_t inputByteLen, uint8_t *input, uint8_t *output); + +void Hacl_SHA3_sha3_256(uint32_t inputByteLen, uint8_t *input, uint8_t *output); + +void Hacl_SHA3_sha3_384(uint32_t inputByteLen, uint8_t *input, uint8_t *output); + +void Hacl_SHA3_sha3_512(uint32_t inputByteLen, uint8_t *input, uint8_t *output); + +void Hacl_Impl_SHA3_absorb_inner(uint32_t rateInBytes, uint8_t *block, uint64_t *s); + +void +Hacl_Impl_SHA3_squeeze( + uint64_t *s, + uint32_t rateInBytes, + uint32_t outputByteLen, + uint8_t *output +); + +void +Hacl_Impl_SHA3_keccak( + uint32_t rate, + uint32_t capacity, + uint32_t inputByteLen, + uint8_t *input, + uint8_t delimitedSuffix, + uint32_t outputByteLen, + uint8_t *output +); + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Hash_SHA3_H_DEFINED +#endif diff --git a/Modules/_hacl/Hacl_Streaming_Types.h b/Modules/_hacl/Hacl_Streaming_Types.h index 51057611ca978d..8a60b707bc4958 100644 --- a/Modules/_hacl/Hacl_Streaming_Types.h +++ b/Modules/_hacl/Hacl_Streaming_Types.h @@ -35,6 +35,23 @@ extern "C" { #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" +#define Spec_Hash_Definitions_SHA2_224 0 +#define Spec_Hash_Definitions_SHA2_256 1 +#define Spec_Hash_Definitions_SHA2_384 2 +#define Spec_Hash_Definitions_SHA2_512 3 +#define Spec_Hash_Definitions_SHA1 4 +#define Spec_Hash_Definitions_MD5 5 +#define Spec_Hash_Definitions_Blake2S 6 +#define Spec_Hash_Definitions_Blake2B 7 +#define Spec_Hash_Definitions_SHA3_256 8 +#define Spec_Hash_Definitions_SHA3_224 9 +#define Spec_Hash_Definitions_SHA3_384 10 +#define Spec_Hash_Definitions_SHA3_512 11 +#define Spec_Hash_Definitions_Shake128 12 +#define Spec_Hash_Definitions_Shake256 13 + +typedef uint8_t Spec_Hash_Definitions_hash_alg; + typedef struct Hacl_Streaming_MD_state_32_s { uint32_t *block_state; diff --git a/Modules/_hacl/include/krml/internal/target.h b/Modules/_hacl/include/krml/internal/target.h index dcbe7007b17be8..5a2f94eb2ec8da 100644 --- a/Modules/_hacl/include/krml/internal/target.h +++ b/Modules/_hacl/include/krml/internal/target.h @@ -19,6 +19,28 @@ # define inline __inline__ #endif +/******************************************************************************/ +/* Macros that KaRaMeL will generate. */ +/******************************************************************************/ + +/* For "bare" targets that do not have a C stdlib, the user might want to use + * [-add-early-include '"mydefinitions.h"'] and override these. */ +#ifndef KRML_HOST_PRINTF +# define KRML_HOST_PRINTF printf +#endif + +#if ( \ + (defined __STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ + (!(defined KRML_HOST_EPRINTF))) +# define KRML_HOST_EPRINTF(...) fprintf(stderr, __VA_ARGS__) +#elif !(defined KRML_HOST_EPRINTF) && defined(_MSC_VER) +# define KRML_HOST_EPRINTF(...) fprintf(stderr, __VA_ARGS__) +#endif + +#ifndef KRML_HOST_EXIT +# define KRML_HOST_EXIT exit +#endif + #ifndef KRML_HOST_MALLOC # define KRML_HOST_MALLOC malloc #endif @@ -35,6 +57,28 @@ # define KRML_HOST_IGNORE(x) (void)(x) #endif +/* In FStar.Buffer.fst, the size of arrays is uint32_t, but it's a number of + * *elements*. Do an ugly, run-time check (some of which KaRaMeL can eliminate). + */ +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 4)) +# define _KRML_CHECK_SIZE_PRAGMA \ + _Pragma("GCC diagnostic ignored \"-Wtype-limits\"") +#else +# define _KRML_CHECK_SIZE_PRAGMA +#endif + +#define KRML_CHECK_SIZE(size_elt, sz) \ + do { \ + _KRML_CHECK_SIZE_PRAGMA \ + if (((size_t)(sz)) > ((size_t)(SIZE_MAX / (size_elt)))) { \ + KRML_HOST_PRINTF( \ + "Maximum allocatable size exceeded, aborting before overflow at " \ + "%s:%d\n", \ + __FILE__, __LINE__); \ + KRML_HOST_EXIT(253); \ + } \ + } while (0) + /* Macros for prettier unrolling of loops */ #define KRML_LOOP1(i, n, x) { \ x \ diff --git a/Modules/_hacl/include/krml/lowstar_endianness.h b/Modules/_hacl/include/krml/lowstar_endianness.h index 32a7391e817ebb..1aa2ccd644c06f 100644 --- a/Modules/_hacl/include/krml/lowstar_endianness.h +++ b/Modules/_hacl/include/krml/lowstar_endianness.h @@ -77,7 +77,7 @@ # define le64toh(x) (x) /* ... for Windows (GCC-like, e.g. mingw or clang) */ -#elif (defined(_WIN32) || defined(_WIN64)) && \ +#elif (defined(_WIN32) || defined(_WIN64) || defined(__EMSCRIPTEN__)) && \ (defined(__GNUC__) || defined(__clang__)) # define htobe16(x) __builtin_bswap16(x) @@ -96,7 +96,8 @@ # define le64toh(x) (x) /* ... generic big-endian fallback code */ -#elif defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +/* ... AIX doesn't have __BYTE_ORDER__ (with XLC compiler) & is always big-endian */ +#elif (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) || defined(_AIX) /* byte swapping code inspired by: * https://github.com/rweather/arduinolibs/blob/master/libraries/Crypto/utility/EndianUtil.h diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA3.h b/Modules/_hacl/internal/Hacl_Hash_SHA3.h new file mode 100644 index 00000000000000..1c9808b8dd497c --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Hash_SHA3.h @@ -0,0 +1,65 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Hash_SHA3_H +#define __internal_Hacl_Hash_SHA3_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "../Hacl_Hash_SHA3.h" + +void +Hacl_Hash_SHA3_update_multi_sha3( + Spec_Hash_Definitions_hash_alg a, + uint64_t *s, + uint8_t *blocks, + uint32_t n_blocks +); + +void +Hacl_Hash_SHA3_update_last_sha3( + Spec_Hash_Definitions_hash_alg a, + uint64_t *s, + uint8_t *input, + uint32_t input_len +); + +void Hacl_Impl_SHA3_state_permute(uint64_t *s); + +void Hacl_Impl_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s); + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Hash_SHA3_H_DEFINED +#endif diff --git a/Modules/_hacl/python_hacl_namespaces.h b/Modules/_hacl/python_hacl_namespaces.h index ee28f244266b85..0df236282ac509 100644 --- a/Modules/_hacl/python_hacl_namespaces.h +++ b/Modules/_hacl/python_hacl_namespaces.h @@ -59,5 +59,28 @@ #define Hacl_Streaming_SHA1_legacy_copy python_hashlib_Hacl_Streaming_SHA1_legacy_copy #define Hacl_Streaming_SHA1_legacy_hash python_hashlib_Hacl_Streaming_SHA1_legacy_hash +#define Hacl_Hash_SHA3_update_last_sha3 python_hashlib_Hacl_Hash_SHA3_update_last_sha3 +#define Hacl_Hash_SHA3_update_multi_sha3 python_hashlib_Hacl_Hash_SHA3_update_multi_sha3 +#define Hacl_Impl_SHA3_absorb_inner python_hashlib_Hacl_Impl_SHA3_absorb_inner +#define Hacl_Impl_SHA3_keccak python_hashlib_Hacl_Impl_SHA3_keccak +#define Hacl_Impl_SHA3_loadState python_hashlib_Hacl_Impl_SHA3_loadState +#define Hacl_Impl_SHA3_squeeze python_hashlib_Hacl_Impl_SHA3_squeeze +#define Hacl_Impl_SHA3_state_permute python_hashlib_Hacl_Impl_SHA3_state_permute +#define Hacl_SHA3_sha3_224 python_hashlib_Hacl_SHA3_sha3_224 +#define Hacl_SHA3_sha3_256 python_hashlib_Hacl_SHA3_sha3_256 +#define Hacl_SHA3_sha3_384 python_hashlib_Hacl_SHA3_sha3_384 +#define Hacl_SHA3_sha3_512 python_hashlib_Hacl_SHA3_sha3_512 +#define Hacl_SHA3_shake128_hacl python_hashlib_Hacl_SHA3_shake128_hacl +#define Hacl_SHA3_shake256_hacl python_hashlib_Hacl_SHA3_shake256_hacl +#define Hacl_Streaming_Keccak_block_len python_hashlib_Hacl_Streaming_Keccak_block_len +#define Hacl_Streaming_Keccak_copy python_hashlib_Hacl_Streaming_Keccak_copy +#define Hacl_Streaming_Keccak_finish python_hashlib_Hacl_Streaming_Keccak_finish +#define Hacl_Streaming_Keccak_free python_hashlib_Hacl_Streaming_Keccak_free +#define Hacl_Streaming_Keccak_get_alg python_hashlib_Hacl_Streaming_Keccak_get_alg +#define Hacl_Streaming_Keccak_hash_len python_hashlib_Hacl_Streaming_Keccak_hash_len +#define Hacl_Streaming_Keccak_is_shake python_hashlib_Hacl_Streaming_Keccak_is_shake +#define Hacl_Streaming_Keccak_malloc python_hashlib_Hacl_Streaming_Keccak_malloc +#define Hacl_Streaming_Keccak_reset python_hashlib_Hacl_Streaming_Keccak_reset +#define Hacl_Streaming_Keccak_update python_hashlib_Hacl_Streaming_Keccak_update #endif // _PYTHON_HACL_NAMESPACES_H diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index 76b92ec4599102..220ebbe5561341 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=13e0c6721ac9206c4249ecc1dc04ed617ad1e262 +expected_hacl_star_rev=363eae2c2eb60e46f182ddd4bd1cd3f1d00b35c9 hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" @@ -45,11 +45,14 @@ dist_files=( Hacl_Hash_SHA1.h internal/Hacl_Hash_SHA1.h Hacl_Hash_MD5.h + Hacl_Hash_SHA3.h internal/Hacl_Hash_MD5.h + internal/Hacl_Hash_SHA3.h internal/Hacl_SHA2_Generic.h Hacl_Streaming_SHA2.c Hacl_Hash_SHA1.c Hacl_Hash_MD5.c + Hacl_Hash_SHA3.c ) declare -a include_files @@ -134,9 +137,9 @@ $sed -i -z 's!#include \n!#include \n#include "python_hacl_n # Finally, we remove a bunch of ifdefs from target.h that are, again, useful in # the general case, but not exercised by the subset of HACL* that we vendor. -$sed -z -i 's!#ifndef KRML_\(HOST_PRINTF\|HOST_EXIT\|PRE_ALIGN\|POST_ALIGN\|ALIGNED_MALLOC\|ALIGNED_FREE\|HOST_TIME\)\n\(\n\|# [^\n]*\n\|[^#][^\n]*\n\)*#endif\n\n!!g' include/krml/internal/target.h -$sed -z -i 's!\n\n\([^#][^\n]*\n\)*#define KRML_\(EABORT\|EXIT\|CHECK_SIZE\)[^\n]*\(\n [^\n]*\)*!!g' include/krml/internal/target.h +$sed -z -i 's!#ifndef KRML_\(PRE_ALIGN\|POST_ALIGN\|ALIGNED_MALLOC\|ALIGNED_FREE\|HOST_TIME\)\n\(\n\|# [^\n]*\n\|[^#][^\n]*\n\)*#endif\n\n!!g' include/krml/internal/target.h +$sed -z -i 's!\n\n\([^#][^\n]*\n\)*#define KRML_\(EABORT\|EXIT\)[^\n]*\(\n [^\n]*\)*!!g' include/krml/internal/target.h $sed -z -i 's!\n\n\([^#][^\n]*\n\)*#if [^\n]*\n\( [^\n]*\n\)*#define KRML_\(EABORT\|EXIT\|CHECK_SIZE\)[^\n]*\(\n [^\n]*\)*!!g' include/krml/internal/target.h -$sed -z -i 's!\n\n\([^#][^\n]*\n\)*#if [^\n]*\n\( [^\n]*\n\)*# define _\?KRML_\(DEPRECATED\|CHECK_SIZE_PRAGMA\|HOST_EPRINTF\|HOST_SNPRINTF\)[^\n]*\n\([^#][^\n]*\n\|#el[^\n]*\n\|# [^\n]*\n\)*#endif!!g' include/krml/internal/target.h +$sed -z -i 's!\n\n\([^#][^\n]*\n\)*#if [^\n]*\n\( [^\n]*\n\)*# define _\?KRML_\(DEPRECATED\|HOST_SNPRINTF\)[^\n]*\n\([^#][^\n]*\n\|#el[^\n]*\n\|# [^\n]*\n\)*#endif!!g' include/krml/internal/target.h echo "Updated; verify all is okay using git diff and git status." diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 7476e5dc7dd61e..99d0b72819137e 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -2260,6 +2260,7 @@ static PyModuleDef_Slot hashlib_slots[] = { {Py_mod_exec, hashlib_md_meth_names}, {Py_mod_exec, hashlib_init_constructors}, {Py_mod_exec, hashlib_exception}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 07ddc7b0851241..00285ae01f8574 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -682,6 +682,7 @@ heapq_exec(PyObject *m) static struct PyModuleDef_Slot heapq_slots[] = { {Py_mod_exec, heapq_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 8ec3a6081c98d9..c05407b5d61815 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -321,7 +321,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, #ifdef HAVE_WINDOWS_CONSOLE_IO const PyConfig *config = _Py_GetConfig(); if (!config->legacy_windows_stdio && _PyIO_get_console_type(path_or_fd) != '\0') { - RawIO_class = (PyObject *)&PyWindowsConsoleIO_Type; + RawIO_class = (PyObject *)state->PyWindowsConsoleIO_Type; encoding = "utf-8"; } #endif @@ -580,17 +580,24 @@ iomodule_traverse(PyObject *mod, visitproc visit, void *arg) { _PyIO_State *state = get_io_state(mod); if (!state->initialized) return 0; - Py_VISIT(state->locale_module); Py_VISIT(state->unsupported_operation); + Py_VISIT(state->PyIncrementalNewlineDecoder_Type); + Py_VISIT(state->PyRawIOBase_Type); + Py_VISIT(state->PyBufferedIOBase_Type); Py_VISIT(state->PyBufferedRWPair_Type); Py_VISIT(state->PyBufferedRandom_Type); Py_VISIT(state->PyBufferedReader_Type); Py_VISIT(state->PyBufferedWriter_Type); + Py_VISIT(state->PyBytesIOBuffer_Type); Py_VISIT(state->PyBytesIO_Type); Py_VISIT(state->PyFileIO_Type); Py_VISIT(state->PyStringIO_Type); + Py_VISIT(state->PyTextIOBase_Type); Py_VISIT(state->PyTextIOWrapper_Type); +#ifdef HAVE_WINDOWS_CONSOLE_IO + Py_VISIT(state->PyWindowsConsoleIO_Type); +#endif return 0; } @@ -600,18 +607,24 @@ iomodule_clear(PyObject *mod) { _PyIO_State *state = get_io_state(mod); if (!state->initialized) return 0; - if (state->locale_module != NULL) - Py_CLEAR(state->locale_module); Py_CLEAR(state->unsupported_operation); + Py_CLEAR(state->PyIncrementalNewlineDecoder_Type); + Py_CLEAR(state->PyRawIOBase_Type); + Py_CLEAR(state->PyBufferedIOBase_Type); Py_CLEAR(state->PyBufferedRWPair_Type); Py_CLEAR(state->PyBufferedRandom_Type); Py_CLEAR(state->PyBufferedReader_Type); Py_CLEAR(state->PyBufferedWriter_Type); + Py_CLEAR(state->PyBytesIOBuffer_Type); Py_CLEAR(state->PyBytesIO_Type); Py_CLEAR(state->PyFileIO_Type); Py_CLEAR(state->PyStringIO_Type); + Py_CLEAR(state->PyTextIOBase_Type); Py_CLEAR(state->PyTextIOWrapper_Type); +#ifdef HAVE_WINDOWS_CONSOLE_IO + Py_CLEAR(state->PyWindowsConsoleIO_Type); +#endif return 0; } @@ -653,31 +666,17 @@ struct PyModuleDef _PyIO_Module = { static PyTypeObject* static_types[] = { // Base classes &PyIOBase_Type, - &PyIncrementalNewlineDecoder_Type, // PyIOBase_Type subclasses &PyBufferedIOBase_Type, &PyRawIOBase_Type, &PyTextIOBase_Type, - - // PyRawIOBase_Type(PyIOBase_Type) subclasses - &_PyBytesIOBuffer_Type, -#ifdef HAVE_WINDOWS_CONSOLE_IO - &PyWindowsConsoleIO_Type, -#endif }; PyStatus _PyIO_InitTypes(PyInterpreterState *interp) { -#ifdef HAVE_WINDOWS_CONSOLE_IO - if (_Py_IsMainInterpreter(interp)) { - // Set type base classes - PyWindowsConsoleIO_Type.tp_base = &PyRawIOBase_Type; - } -#endif - for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) { PyTypeObject *type = static_types[i]; if (_PyStaticType_InitBuiltin(interp, type) < 0) { @@ -749,24 +748,37 @@ PyInit__io(void) } } + // Base classes + ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &nldecoder_spec, NULL); + ADD_TYPE(m, state->PyBytesIOBuffer_Type, &bytesiobuf_spec, NULL); + + // PyIOBase_Type subclasses + state->PyRawIOBase_Type = (PyTypeObject *)Py_NewRef(&PyRawIOBase_Type); + state->PyBufferedIOBase_Type = (PyTypeObject *)Py_NewRef(&PyBufferedIOBase_Type); + state->PyTextIOBase_Type = (PyTypeObject *)Py_NewRef(&PyTextIOBase_Type); + // PyBufferedIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyBytesIO_Type, &bytesio_spec, &PyBufferedIOBase_Type); + ADD_TYPE(m, state->PyBytesIO_Type, &bytesio_spec, state->PyBufferedIOBase_Type); ADD_TYPE(m, state->PyBufferedWriter_Type, &bufferedwriter_spec, - &PyBufferedIOBase_Type); + state->PyBufferedIOBase_Type); ADD_TYPE(m, state->PyBufferedReader_Type, &bufferedreader_spec, - &PyBufferedIOBase_Type); + state->PyBufferedIOBase_Type); ADD_TYPE(m, state->PyBufferedRWPair_Type, &bufferedrwpair_spec, - &PyBufferedIOBase_Type); + state->PyBufferedIOBase_Type); ADD_TYPE(m, state->PyBufferedRandom_Type, &bufferedrandom_spec, - &PyBufferedIOBase_Type); + state->PyBufferedIOBase_Type); // PyRawIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyFileIO_Type, &fileio_spec, &PyRawIOBase_Type); + ADD_TYPE(m, state->PyFileIO_Type, &fileio_spec, state->PyRawIOBase_Type); +#ifdef MS_WINDOWS + ADD_TYPE(m, state->PyWindowsConsoleIO_Type, &winconsoleio_spec, + state->PyRawIOBase_Type); +#endif // PyTextIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyStringIO_Type, &stringio_spec, &PyTextIOBase_Type); + ADD_TYPE(m, state->PyStringIO_Type, &stringio_spec, state->PyTextIOBase_Type); ADD_TYPE(m, state->PyTextIOWrapper_Type, &textiowrapper_spec, - &PyTextIOBase_Type); + state->PyTextIOBase_Type); state->initialized = 1; diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index d7224e56f9a722..1bf301c9cf0a94 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -5,6 +5,7 @@ #include "exports.h" #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_typeobject.h" // _PyType_GetModuleState() #include "structmember.h" /* ABCs */ @@ -13,22 +14,21 @@ extern PyTypeObject PyRawIOBase_Type; extern PyTypeObject PyBufferedIOBase_Type; extern PyTypeObject PyTextIOBase_Type; -/* Concrete classes */ -extern PyTypeObject PyIncrementalNewlineDecoder_Type; - /* Type specs */ extern PyType_Spec bufferedrandom_spec; extern PyType_Spec bufferedreader_spec; extern PyType_Spec bufferedrwpair_spec; extern PyType_Spec bufferedwriter_spec; extern PyType_Spec bytesio_spec; +extern PyType_Spec bytesiobuf_spec; extern PyType_Spec fileio_spec; +extern PyType_Spec nldecoder_spec; extern PyType_Spec stringio_spec; extern PyType_Spec textiowrapper_spec; #ifdef HAVE_WINDOWS_CONSOLE_IO -extern PyTypeObject PyWindowsConsoleIO_Type; -#endif /* HAVE_WINDOWS_CONSOLE_IO */ +extern PyType_Spec winconsoleio_spec; +#endif /* These functions are used as METH_NOARGS methods, are normally called * with args=NULL, and return a new reference. @@ -142,19 +142,25 @@ extern PyModuleDef _PyIO_Module; typedef struct { int initialized; - PyObject *locale_module; - PyObject *unsupported_operation; /* Types */ + PyTypeObject *PyIncrementalNewlineDecoder_Type; + PyTypeObject *PyRawIOBase_Type; + PyTypeObject *PyBufferedIOBase_Type; PyTypeObject *PyBufferedRWPair_Type; PyTypeObject *PyBufferedRandom_Type; PyTypeObject *PyBufferedReader_Type; PyTypeObject *PyBufferedWriter_Type; + PyTypeObject *PyBytesIOBuffer_Type; PyTypeObject *PyBytesIO_Type; PyTypeObject *PyFileIO_Type; PyTypeObject *PyStringIO_Type; + PyTypeObject *PyTextIOBase_Type; PyTypeObject *PyTextIOWrapper_Type; +#ifdef MS_WINDOWS + PyTypeObject *PyWindowsConsoleIO_Type; +#endif } _PyIO_State; #define IO_MOD_STATE(mod) ((_PyIO_State *)PyModule_GetState(mod)) @@ -168,6 +174,14 @@ get_io_state(PyObject *module) return (_PyIO_State *)state; } +static inline _PyIO_State * +get_io_state_by_cls(PyTypeObject *cls) +{ + void *state = _PyType_GetModuleState(cls); + assert(state != NULL); + return (_PyIO_State *)state; +} + static inline _PyIO_State * find_io_state_by_def(PyTypeObject *type) { @@ -181,5 +195,3 @@ extern _PyIO_State *_PyIO_get_module_state(void); #ifdef HAVE_WINDOWS_CONSOLE_IO extern char _PyIO_get_console_type(PyObject *); #endif - -extern Py_EXPORTED_SYMBOL PyTypeObject _PyBytesIOBuffer_Type; diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 2c71768be97870..723d16b47fef9f 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -2231,7 +2231,7 @@ bufferedrwpair_close(rwpair *self, PyObject *Py_UNUSED(ignored)) } else { Py_DECREF(ret); - } + } ret = _forward_call(self->reader, &_Py_ID(close), NULL); if (exc != NULL) { _PyErr_ChainExceptions1(exc); diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 7e9d28b3b9655c..3fddfc2ed0bc9c 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -308,14 +308,18 @@ _io_BytesIO_flush_impl(bytesio *self) /*[clinic input] _io.BytesIO.getbuffer + cls: defining_class + / + Get a read-write view over the contents of the BytesIO object. [clinic start generated code]*/ static PyObject * -_io_BytesIO_getbuffer_impl(bytesio *self) -/*[clinic end generated code: output=72cd7c6e13aa09ed input=8f738ef615865176]*/ +_io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls) +/*[clinic end generated code: output=045091d7ce87fe4e input=0668fbb48f95dffa]*/ { - PyTypeObject *type = &_PyBytesIOBuffer_Type; + _PyIO_State *state = get_io_state_by_cls(cls); + PyTypeObject *type = state->PyBytesIOBuffer_Type; bytesiobuf *buf; PyObject *view; @@ -1086,9 +1090,17 @@ bytesiobuf_releasebuffer(bytesiobuf *obj, Py_buffer *view) b->exports--; } +static int +bytesiobuf_clear(bytesiobuf *self) +{ + Py_CLEAR(self->source); + return 0; +} + static int bytesiobuf_traverse(bytesiobuf *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->source); return 0; } @@ -1096,54 +1108,29 @@ bytesiobuf_traverse(bytesiobuf *self, visitproc visit, void *arg) static void bytesiobuf_dealloc(bytesiobuf *self) { + PyTypeObject *tp = Py_TYPE(self); /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); - Py_CLEAR(self->source); - Py_TYPE(self)->tp_free(self); + (void)bytesiobuf_clear(self); + tp->tp_free(self); + Py_DECREF(tp); } -static PyBufferProcs bytesiobuf_as_buffer = { - (getbufferproc) bytesiobuf_getbuffer, - (releasebufferproc) bytesiobuf_releasebuffer, +static PyType_Slot bytesiobuf_slots[] = { + {Py_tp_dealloc, bytesiobuf_dealloc}, + {Py_tp_traverse, bytesiobuf_traverse}, + {Py_tp_clear, bytesiobuf_clear}, + + // Buffer protocol + {Py_bf_getbuffer, bytesiobuf_getbuffer}, + {Py_bf_releasebuffer, bytesiobuf_releasebuffer}, + {0, NULL}, }; -Py_EXPORTED_SYMBOL PyTypeObject _PyBytesIOBuffer_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io._BytesIOBuffer", /*tp_name*/ - sizeof(bytesiobuf), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)bytesiobuf_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - &bytesiobuf_as_buffer, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - 0, /*tp_doc*/ - (traverseproc)bytesiobuf_traverse, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - 0, /*tp_new*/ +PyType_Spec bytesiobuf_spec = { + .name = "_io._BytesIOBuffer", + .basicsize = sizeof(bytesiobuf), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), + .slots = bytesiobuf_slots, }; diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index 84b58db6c7a702..9550c8728c251e 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -87,15 +87,19 @@ PyDoc_STRVAR(_io_BytesIO_getbuffer__doc__, "Get a read-write view over the contents of the BytesIO object."); #define _IO_BYTESIO_GETBUFFER_METHODDEF \ - {"getbuffer", (PyCFunction)_io_BytesIO_getbuffer, METH_NOARGS, _io_BytesIO_getbuffer__doc__}, + {"getbuffer", _PyCFunction_CAST(_io_BytesIO_getbuffer), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_BytesIO_getbuffer__doc__}, static PyObject * -_io_BytesIO_getbuffer_impl(bytesio *self); +_io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls); static PyObject * -_io_BytesIO_getbuffer(bytesio *self, PyObject *Py_UNUSED(ignored)) +_io_BytesIO_getbuffer(bytesio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _io_BytesIO_getbuffer_impl(self); + if (nargs) { + PyErr_SetString(PyExc_TypeError, "getbuffer() takes no arguments"); + return NULL; + } + return _io_BytesIO_getbuffer_impl(self, cls); } PyDoc_STRVAR(_io_BytesIO_getvalue__doc__, @@ -534,4 +538,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=a44770efbaeb80dd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=098584d485420b65 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index b6e9bd5b65a029..dfad8a58c4723e 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -18,15 +18,19 @@ PyDoc_STRVAR(_io_FileIO_close__doc__, "called more than once without error."); #define _IO_FILEIO_CLOSE_METHODDEF \ - {"close", (PyCFunction)_io_FileIO_close, METH_NOARGS, _io_FileIO_close__doc__}, + {"close", _PyCFunction_CAST(_io_FileIO_close), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_close__doc__}, static PyObject * -_io_FileIO_close_impl(fileio *self); +_io_FileIO_close_impl(fileio *self, PyTypeObject *cls); static PyObject * -_io_FileIO_close(fileio *self, PyObject *Py_UNUSED(ignored)) +_io_FileIO_close(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _io_FileIO_close_impl(self); + if (nargs) { + PyErr_SetString(PyExc_TypeError, "close() takes no arguments"); + return NULL; + } + return _io_FileIO_close_impl(self, cls); } PyDoc_STRVAR(_io_FileIO___init____doc__, @@ -466,4 +470,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=27f883807a6c29ae input=a9049054013a1b77]*/ +/*[clinic end generated code: output=29ed2ae6c451c139 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 1118d86e6c9a10..cc0e7307b9da77 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -130,6 +130,9 @@ internal_close(fileio *self) /*[clinic input] _io.FileIO.close + cls: defining_class + / + Close the file. A closed file cannot be used for further I/O operations. close() may be @@ -137,18 +140,20 @@ called more than once without error. [clinic start generated code]*/ static PyObject * -_io_FileIO_close_impl(fileio *self) -/*[clinic end generated code: output=7737a319ef3bad0b input=f35231760d54a522]*/ +_io_FileIO_close_impl(fileio *self, PyTypeObject *cls) +/*[clinic end generated code: output=c30cbe9d1f23ca58 input=70da49e63db7c64d]*/ { PyObject *res; - PyObject *exc; int rc; - res = PyObject_CallMethodOneArg((PyObject*)&PyRawIOBase_Type, + _PyIO_State *state = get_io_state_by_cls(cls); + res = PyObject_CallMethodOneArg((PyObject*)state->PyRawIOBase_Type, &_Py_ID(close), (PyObject *)self); if (!self->closefd) { self->fd = -1; return res; } + + PyObject *exc; if (res == NULL) { exc = PyErr_GetRaisedException(); } diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 54c050f0be4688..13d3b870b39a81 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -716,9 +716,10 @@ _io_StringIO___init___impl(stringio *self, PyObject *value, self->writenl = Py_NewRef(self->readnl); } + _PyIO_State *module_state = find_io_state_by_def(Py_TYPE(self)); if (self->readuniversal) { self->decoder = PyObject_CallFunctionObjArgs( - (PyObject *)&PyIncrementalNewlineDecoder_Type, + (PyObject *)module_state->PyIncrementalNewlineDecoder_Type, Py_None, self->readtranslate ? Py_True : Py_False, NULL); if (self->decoder == NULL) return -1; @@ -750,7 +751,7 @@ _io_StringIO___init___impl(stringio *self, PyObject *value, self->state = STATE_ACCUMULATING; } self->pos = 0; - self->module_state = find_io_state_by_def(Py_TYPE(self)); + self->module_state = module_state; self->closed = 0; self->ok = 1; return 0; diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 3ff84cb623af74..070687a83d1bc2 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -18,10 +18,10 @@ /*[clinic input] module _io -class _io.IncrementalNewlineDecoder "nldecoder_object *" "&PyIncrementalNewlineDecoder_Type" +class _io.IncrementalNewlineDecoder "nldecoder_object *" "clinic_state()->PyIncrementalNewlineDecoder_Type" class _io.TextIOWrapper "textio *" "clinic_state()->TextIOWrapper_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d3f032e90f74c8f2]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=81f67cf54eaa6001]*/ /* TextIOBase */ @@ -248,12 +248,32 @@ _io_IncrementalNewlineDecoder___init___impl(nldecoder_object *self, return 0; } -static void -incrementalnewlinedecoder_dealloc(nldecoder_object *self) +static int +incrementalnewlinedecoder_traverse(nldecoder_object *self, visitproc visit, + void *arg) +{ + Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->decoder); + Py_VISIT(self->errors); + return 0; +} + +static int +incrementalnewlinedecoder_clear(nldecoder_object *self) { Py_CLEAR(self->decoder); Py_CLEAR(self->errors); - Py_TYPE(self)->tp_free((PyObject *)self); + return 0; +} + +static void +incrementalnewlinedecoder_dealloc(nldecoder_object *self) +{ + PyTypeObject *tp = Py_TYPE(self); + _PyObject_GC_UNTRACK(self); + (void)incrementalnewlinedecoder_clear(self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); } static int @@ -872,8 +892,9 @@ _textiowrapper_set_decoder(textio *self, PyObject *codec_info, return -1; if (self->readuniversal) { + _PyIO_State *state = self->state; PyObject *incrementalDecoder = PyObject_CallFunctionObjArgs( - (PyObject *)&PyIncrementalNewlineDecoder_Type, + (PyObject *)state->PyIncrementalNewlineDecoder_Type, self->decoder, self->readtranslate ? Py_True : Py_False, NULL); if (incrementalDecoder == NULL) return -1; @@ -884,11 +905,12 @@ _textiowrapper_set_decoder(textio *self, PyObject *codec_info, } static PyObject* -_textiowrapper_decode(PyObject *decoder, PyObject *bytes, int eof) +_textiowrapper_decode(_PyIO_State *state, PyObject *decoder, PyObject *bytes, + int eof) { PyObject *chars; - if (Py_IS_TYPE(decoder, &PyIncrementalNewlineDecoder_Type)) + if (Py_IS_TYPE(decoder, state->PyIncrementalNewlineDecoder_Type)) chars = _PyIncrementalNewlineDecoder_decode(decoder, bytes, eof); else chars = PyObject_CallMethodObjArgs(decoder, &_Py_ID(decode), bytes, @@ -1167,6 +1189,8 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, self->buffer = Py_NewRef(buffer); /* Build the decoder object */ + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); + self->state = state; if (_textiowrapper_set_decoder(self, codec_info, PyUnicode_AsUTF8(errors)) != 0) goto error; @@ -1177,7 +1201,6 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, /* Finished sorting out the codec details */ Py_CLEAR(codec_info); - _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); if (Py_IS_TYPE(buffer, state->PyBufferedReader_Type) || Py_IS_TYPE(buffer, state->PyBufferedWriter_Type) || Py_IS_TYPE(buffer, state->PyBufferedRandom_Type)) @@ -1214,7 +1237,6 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, goto error; } - self->state = state; self->ok = 1; return 0; @@ -1843,7 +1865,8 @@ textiowrapper_read_chunk(textio *self, Py_ssize_t size_hint) nbytes = input_chunk_buf.len; eof = (nbytes == 0); - decoded_chars = _textiowrapper_decode(self->decoder, input_chunk, eof); + decoded_chars = _textiowrapper_decode(self->state, self->decoder, + input_chunk, eof); PyBuffer_Release(&input_chunk_buf); if (decoded_chars == NULL) goto fail; @@ -1913,7 +1936,8 @@ _io_TextIOWrapper_read_impl(textio *self, Py_ssize_t n) if (bytes == NULL) goto fail; - if (Py_IS_TYPE(self->decoder, &PyIncrementalNewlineDecoder_Type)) + _PyIO_State *state = self->state; + if (Py_IS_TYPE(self->decoder, state->PyIncrementalNewlineDecoder_Type)) decoded = _PyIncrementalNewlineDecoder_decode(self->decoder, bytes, 1); else @@ -3172,45 +3196,23 @@ static PyGetSetDef incrementalnewlinedecoder_getset[] = { {NULL} }; -PyTypeObject PyIncrementalNewlineDecoder_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.IncrementalNewlineDecoder", /*tp_name*/ - sizeof(nldecoder_object), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)incrementalnewlinedecoder_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - _io_IncrementalNewlineDecoder___init____doc__, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /*tp_weaklistoffset*/ - 0, /* tp_iter */ - 0, /* tp_iternext */ - incrementalnewlinedecoder_methods, /* tp_methods */ - 0, /* tp_members */ - incrementalnewlinedecoder_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - _io_IncrementalNewlineDecoder___init__, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ +static PyType_Slot nldecoder_slots[] = { + {Py_tp_dealloc, incrementalnewlinedecoder_dealloc}, + {Py_tp_doc, (void *)_io_IncrementalNewlineDecoder___init____doc__}, + {Py_tp_methods, incrementalnewlinedecoder_methods}, + {Py_tp_getset, incrementalnewlinedecoder_getset}, + {Py_tp_traverse, incrementalnewlinedecoder_traverse}, + {Py_tp_clear, incrementalnewlinedecoder_clear}, + {Py_tp_init, _io_IncrementalNewlineDecoder___init__}, + {0, NULL}, +}; + +PyType_Spec nldecoder_spec = { + .name = "_io.IncrementalNewlineDecoder", + .basicsize = sizeof(nldecoder_object), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = nldecoder_slots, }; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index f836e230243020..fdb57cff7c04d6 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -137,9 +137,9 @@ char _PyIO_get_console_type(PyObject *path_or_fd) { /*[clinic input] module _io -class _io._WindowsConsoleIO "winconsoleio *" "&PyWindowsConsoleIO_Type" +class _io._WindowsConsoleIO "winconsoleio *" "clinic_state()->PyWindowsConsoleIO_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e897fdc1fba4e131]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=05526e723011ab36]*/ typedef struct { PyObject_HEAD @@ -156,8 +156,6 @@ typedef struct { wchar_t wbuf; } winconsoleio; -PyTypeObject PyWindowsConsoleIO_Type; - int _PyWindowsConsoleIO_closed(PyObject *self) { @@ -265,7 +263,10 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, int fd_is_own = 0; HANDLE handle = NULL; - assert(PyObject_TypeCheck(self, (PyTypeObject *)&PyWindowsConsoleIO_Type)); +#ifdef Py_DEBUG + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); + assert(PyObject_TypeCheck(self, state->PyWindowsConsoleIO_Type)); +#endif if (self->fd >= 0) { if (self->closefd) { /* Have to close the existing file first. */ @@ -417,6 +418,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, static int winconsoleio_traverse(winconsoleio *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); return 0; } @@ -431,6 +433,7 @@ winconsoleio_clear(winconsoleio *self) static void winconsoleio_dealloc(winconsoleio *self) { + PyTypeObject *tp = Py_TYPE(self); self->finalizing = 1; if (_PyIOBase_finalize((PyObject *) self) < 0) return; @@ -438,7 +441,8 @@ winconsoleio_dealloc(winconsoleio *self) if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); Py_CLEAR(self->dict); - Py_TYPE(self)->tp_free((PyObject *)self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); } static PyObject * @@ -1078,7 +1082,9 @@ _io__WindowsConsoleIO_isatty_impl(winconsoleio *self) Py_RETURN_TRUE; } +#define clinic_state() (IO_STATE()) #include "clinic/winconsoleio.c.h" +#undef clinic_state static PyMethodDef winconsoleio_methods[] = { _IO__WINDOWSCONSOLEIO_READ_METHODDEF @@ -1124,59 +1130,32 @@ static PyGetSetDef winconsoleio_getsetlist[] = { static PyMemberDef winconsoleio_members[] = { {"_blksize", T_UINT, offsetof(winconsoleio, blksize), 0}, {"_finalizing", T_BOOL, offsetof(winconsoleio, finalizing), 0}, + {"__weaklistoffset__", T_PYSSIZET, offsetof(winconsoleio, weakreflist), READONLY}, + {"__dictoffset__", T_PYSSIZET, offsetof(winconsoleio, dict), READONLY}, {NULL} }; -PyTypeObject PyWindowsConsoleIO_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io._WindowsConsoleIO", - sizeof(winconsoleio), - 0, - (destructor)winconsoleio_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)winconsoleio_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - _io__WindowsConsoleIO___init____doc__, /* tp_doc */ - (traverseproc)winconsoleio_traverse, /* tp_traverse */ - (inquiry)winconsoleio_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(winconsoleio, weakreflist), /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - winconsoleio_methods, /* tp_methods */ - winconsoleio_members, /* tp_members */ - winconsoleio_getsetlist, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(winconsoleio, dict), /* tp_dictoffset */ - _io__WindowsConsoleIO___init__, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ - winconsoleio_new, /* tp_new */ - PyObject_GC_Del, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ +static PyType_Slot winconsoleio_slots[] = { + {Py_tp_dealloc, winconsoleio_dealloc}, + {Py_tp_repr, winconsoleio_repr}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_doc, (void *)_io__WindowsConsoleIO___init____doc__}, + {Py_tp_traverse, winconsoleio_traverse}, + {Py_tp_clear, winconsoleio_clear}, + {Py_tp_methods, winconsoleio_methods}, + {Py_tp_members, winconsoleio_members}, + {Py_tp_getset, winconsoleio_getsetlist}, + {Py_tp_init, _io__WindowsConsoleIO___init__}, + {Py_tp_new, winconsoleio_new}, + {0, NULL}, +}; + +PyType_Spec winconsoleio_spec = { + .name = "_io._WindowsConsoleIO", + .basicsize = sizeof(winconsoleio), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = winconsoleio_slots, }; #endif /* HAVE_WINDOWS_CONSOLE_IO */ diff --git a/Modules/_json.c b/Modules/_json.c index fa8e2a936d2c33..c90de05b046b00 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1801,6 +1801,7 @@ _json_exec(PyObject *module) static PyModuleDef_Slot _json_slots[] = { {Py_mod_exec, _json_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 96675cdfb661ad..1ada7305117bb7 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -874,6 +874,7 @@ _locale_exec(PyObject *module) static struct PyModuleDef_Slot _locale_slots[] = { {Py_mod_exec, _locale_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 83d034ae7eed78..1c84f66ee6f579 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -49,6 +49,8 @@ typedef struct { int flags; PyObject *externalTimer; double externalTimerUnit; + int tool_id; + PyObject* missing; } ProfilerObject; #define POF_ENABLED 0x001 @@ -399,64 +401,6 @@ ptrace_leave_call(PyObject *self, void *key) pObj->freelistProfilerContext = pContext; } -static int -profiler_callback(PyObject *self, PyFrameObject *frame, int what, - PyObject *arg) -{ - switch (what) { - - /* the 'frame' of a called function is about to start its execution */ - case PyTrace_CALL: - { - PyCodeObject *code = PyFrame_GetCode(frame); - ptrace_enter_call(self, (void *)code, (PyObject *)code); - Py_DECREF(code); - break; - } - - /* the 'frame' of a called function is about to finish - (either normally or with an exception) */ - case PyTrace_RETURN: - { - PyCodeObject *code = PyFrame_GetCode(frame); - ptrace_leave_call(self, (void *)code); - Py_DECREF(code); - break; - } - - /* case PyTrace_EXCEPTION: - If the exception results in the function exiting, a - PyTrace_RETURN event will be generated, so we don't need to - handle it. */ - - /* the Python function 'frame' is issuing a call to the built-in - function 'arg' */ - case PyTrace_C_CALL: - if ((((ProfilerObject *)self)->flags & POF_BUILTINS) - && PyCFunction_Check(arg)) { - ptrace_enter_call(self, - ((PyCFunctionObject *)arg)->m_ml, - arg); - } - break; - - /* the call to the built-in function 'arg' is returning into its - caller 'frame' */ - case PyTrace_C_RETURN: /* ...normally */ - case PyTrace_C_EXCEPTION: /* ...with an exception set */ - if ((((ProfilerObject *)self)->flags & POF_BUILTINS) - && PyCFunction_Check(arg)) { - ptrace_leave_call(self, - ((PyCFunctionObject *)arg)->m_ml); - } - break; - - default: - break; - } - return 0; -} - static int pending_exception(ProfilerObject *pObj) { @@ -650,6 +594,99 @@ setBuiltins(ProfilerObject *pObj, int nvalue) return 0; } +PyObject* pystart_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +{ + PyObject* code = args[0]; + ptrace_enter_call((PyObject*)self, (void *)code, (PyObject *)code); + + Py_RETURN_NONE; +} + +PyObject* pyreturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +{ + PyObject* code = args[0]; + ptrace_leave_call((PyObject*)self, (void *)code); + + Py_RETURN_NONE; +} + +PyObject* get_cfunc_from_callable(PyObject* callable, PyObject* self_arg, PyObject* missing) +{ + // return a new reference + if (PyCFunction_Check(callable)) { + Py_INCREF(callable); + return (PyObject*)((PyCFunctionObject *)callable); + } + if (Py_TYPE(callable) == &PyMethodDescr_Type) { + /* For backwards compatibility need to + * convert to builtin method */ + + /* If no arg, skip */ + if (self_arg == missing) { + return NULL; + } + PyObject *meth = Py_TYPE(callable)->tp_descr_get( + callable, self_arg, (PyObject*)Py_TYPE(self_arg)); + if (meth == NULL) { + return NULL; + } + if (PyCFunction_Check(meth)) { + return (PyObject*)((PyCFunctionObject *)meth); + } + } + return NULL; +} + +PyObject* ccall_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +{ + if (self->flags & POF_BUILTINS) { + PyObject* callable = args[2]; + PyObject* self_arg = args[3]; + + PyObject* cfunc = get_cfunc_from_callable(callable, self_arg, self->missing); + + if (cfunc) { + ptrace_enter_call((PyObject*)self, + ((PyCFunctionObject *)cfunc)->m_ml, + cfunc); + Py_DECREF(cfunc); + } + } + Py_RETURN_NONE; +} + +PyObject* creturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +{ + if (self->flags & POF_BUILTINS) { + PyObject* callable = args[2]; + PyObject* self_arg = args[3]; + + PyObject* cfunc = get_cfunc_from_callable(callable, self_arg, self->missing); + + if (cfunc) { + ptrace_leave_call((PyObject*)self, + ((PyCFunctionObject *)cfunc)->m_ml); + Py_DECREF(cfunc); + } + } + Py_RETURN_NONE; +} + +static const struct { + int event; + const char* callback_method; +} callback_table[] = { + {PY_MONITORING_EVENT_PY_START, "_pystart_callback"}, + {PY_MONITORING_EVENT_PY_RESUME, "_pystart_callback"}, + {PY_MONITORING_EVENT_PY_RETURN, "_pyreturn_callback"}, + {PY_MONITORING_EVENT_PY_YIELD, "_pyreturn_callback"}, + {PY_MONITORING_EVENT_PY_UNWIND, "_pyreturn_callback"}, + {PY_MONITORING_EVENT_CALL, "_ccall_callback"}, + {PY_MONITORING_EVENT_C_RETURN, "_creturn_callback"}, + {PY_MONITORING_EVENT_C_RAISE, "_creturn_callback"}, + {0, NULL} +}; + PyDoc_STRVAR(enable_doc, "\ enable(subcalls=True, builtins=True)\n\ \n\ @@ -666,6 +703,8 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) int subcalls = -1; int builtins = -1; static char *kwlist[] = {"subcalls", "builtins", 0}; + int all_events = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|pp:enable", kwlist, &subcalls, &builtins)) return NULL; @@ -673,11 +712,37 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetProfile(tstate, profiler_callback, (PyObject*)self) < 0) { + PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); + if (!monitoring) { + return NULL; + } + + if (PyObject_CallMethod(monitoring, "use_tool_id", "is", self->tool_id, "cProfile") == NULL) { + PyErr_Format(PyExc_ValueError, "Another profiling tool is already active"); + Py_DECREF(monitoring); + return NULL; + } + + for (int i = 0; callback_table[i].callback_method; i++) { + PyObject* callback = PyObject_GetAttrString((PyObject*)self, callback_table[i].callback_method); + if (!callback) { + Py_DECREF(monitoring); + return NULL; + } + Py_XDECREF(PyObject_CallMethod(monitoring, "register_callback", "iiO", self->tool_id, + (1 << callback_table[i].event), + callback)); + Py_DECREF(callback); + all_events |= (1 << callback_table[i].event); + } + + if (!PyObject_CallMethod(monitoring, "set_events", "ii", self->tool_id, all_events)) { + Py_DECREF(monitoring); return NULL; } + Py_DECREF(monitoring); + self->flags |= POF_ENABLED; Py_RETURN_NONE; } @@ -707,13 +772,44 @@ Stop collecting profiling information.\n\ static PyObject* profiler_disable(ProfilerObject *self, PyObject* noarg) { - PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetProfile(tstate, NULL, NULL) < 0) { - return NULL; + if (self->flags & POF_ENABLED) { + PyObject* result = NULL; + PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); + + if (!monitoring) { + return NULL; + } + + for (int i = 0; callback_table[i].callback_method; i++) { + result = PyObject_CallMethod(monitoring, "register_callback", "iiO", self->tool_id, + (1 << callback_table[i].event), Py_None); + if (!result) { + Py_DECREF(monitoring); + return NULL; + } + Py_DECREF(result); + } + + result = PyObject_CallMethod(monitoring, "set_events", "ii", self->tool_id, 0); + if (!result) { + Py_DECREF(monitoring); + return NULL; + } + Py_DECREF(result); + + result = PyObject_CallMethod(monitoring, "free_tool_id", "i", self->tool_id); + if (!result) { + Py_DECREF(monitoring); + return NULL; + } + Py_DECREF(result); + + Py_DECREF(monitoring); + + self->flags &= ~POF_ENABLED; + flush_unmatched(self); } - self->flags &= ~POF_ENABLED; - flush_unmatched(self); if (pending_exception(self)) { return NULL; } @@ -778,17 +874,37 @@ profiler_init(ProfilerObject *pObj, PyObject *args, PyObject *kw) return -1; pObj->externalTimerUnit = timeunit; Py_XSETREF(pObj->externalTimer, Py_XNewRef(timer)); + pObj->tool_id = PY_MONITORING_PROFILER_ID; + + PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); + if (!monitoring) { + return -1; + } + pObj->missing = PyObject_GetAttrString(monitoring, "MISSING"); + if (!pObj->missing) { + Py_DECREF(monitoring); + return -1; + } + Py_DECREF(monitoring); return 0; } static PyMethodDef profiler_methods[] = { _LSPROF_PROFILER_GETSTATS_METHODDEF - {"enable", _PyCFunction_CAST(profiler_enable), + {"enable", _PyCFunction_CAST(profiler_enable), METH_VARARGS | METH_KEYWORDS, enable_doc}, - {"disable", (PyCFunction)profiler_disable, + {"disable", (PyCFunction)profiler_disable, METH_NOARGS, disable_doc}, - {"clear", (PyCFunction)profiler_clear, + {"clear", (PyCFunction)profiler_clear, METH_NOARGS, clear_doc}, + {"_pystart_callback", _PyCFunction_CAST(pystart_callback), + METH_FASTCALL, NULL}, + {"_pyreturn_callback", _PyCFunction_CAST(pyreturn_callback), + METH_FASTCALL, NULL}, + {"_ccall_callback", _PyCFunction_CAST(ccall_callback), + METH_FASTCALL, NULL}, + {"_creturn_callback", _PyCFunction_CAST(creturn_callback), + METH_FASTCALL, NULL}, {NULL, NULL} }; @@ -885,6 +1001,9 @@ _lsprof_exec(PyObject *module) static PyModuleDef_Slot _lsprofslots[] = { {Py_mod_exec, _lsprof_exec}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index bccab8639159e7..e34fbad230d51a 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -1611,6 +1611,7 @@ static PyMethodDef lzma_methods[] = { static PyModuleDef_Slot lzma_slots[] = { {Py_mod_exec, lzma_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 2463e1e1a8bf7e..8f9daa5c3de0cc 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -276,6 +276,7 @@ multiprocessing_exec(PyObject *module) static PyModuleDef_Slot multiprocessing_slots[] = { {Py_mod_exec, multiprocessing_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index d64ded4168228f..88c93fe313785c 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -110,12 +110,19 @@ static PyMethodDef module_methods[ ] = { }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL} +}; + + static struct PyModuleDef _posixshmemmodule = { PyModuleDef_HEAD_INIT, .m_name = "_posixshmem", .m_doc = "POSIX shared memory module", .m_size = 0, .m_methods = module_methods, + .m_slots = module_slots, }; /* Module init function */ diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 99be977417743e..b70d426fa29bc0 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -94,12 +94,18 @@ opcode_functions[] = { {NULL, NULL, 0, NULL} }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL} +}; + static struct PyModuleDef opcodemodule = { PyModuleDef_HEAD_INIT, .m_name = "_opcode", .m_doc = "Opcode support module.", .m_size = 0, - .m_methods = opcode_functions + .m_methods = opcode_functions, + .m_slots = module_slots, }; PyMODINIT_FUNC diff --git a/Modules/_operator.c b/Modules/_operator.c index 38335b6995016c..68ccc90562d38d 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1828,6 +1828,7 @@ operator_exec(PyObject *module) static struct PyModuleDef_Slot operator_slots[] = { {Py_mod_exec, operator_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 360c7910f67187..bf7ecae0cc0e50 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -7912,6 +7912,7 @@ _pickle_exec(PyObject *m) static PyModuleDef_Slot pickle_slots[] = { {Py_mod_exec, _pickle_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index f5bce8cd7628ad..2bf83db0e228fb 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -1140,6 +1140,7 @@ static PyMethodDef module_methods[] = { }; static PyModuleDef_Slot _posixsubprocess_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index af19dd6c198b67..d36a911a57c02c 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -431,6 +431,7 @@ queuemodule_exec(PyObject *module) static PyModuleDef_Slot queuemodule_slots[] = { {Py_mod_exec, queuemodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 6e22053239305a..fda5ef267fb470 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -624,6 +624,7 @@ _random_exec(PyObject *module) static PyModuleDef_Slot _random_slots[] = { {Py_mod_exec, _random_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index 344b66f9aad522..e66918016b8da6 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -232,6 +232,7 @@ static PyMethodDef mod_methods[] = { }; static PyModuleDef_Slot _scproxy_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_sha3/LICENSE b/Modules/_sha3/LICENSE deleted file mode 100644 index d2d484d8820dcf..00000000000000 --- a/Modules/_sha3/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 Markku-Juhani O. Saarinen - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/Modules/_sha3/README.txt b/Modules/_sha3/README.txt deleted file mode 100644 index b35919b01677d3..00000000000000 --- a/Modules/_sha3/README.txt +++ /dev/null @@ -1,8 +0,0 @@ -tiny_sha3 -========= - -https://github.com/mjosaarinen/tiny_sha3 -commit dcbb3192047c2a721f5f851db591871d428036a9 - -- All functions have been converted to static functions. -- sha3() function is commented out. diff --git a/Modules/_sha3/sha3.c b/Modules/_sha3/sha3.c deleted file mode 100644 index e2d3fd7b8ad855..00000000000000 --- a/Modules/_sha3/sha3.c +++ /dev/null @@ -1,193 +0,0 @@ -// sha3.c -// 19-Nov-11 Markku-Juhani O. Saarinen - -// Revised 07-Aug-15 to match with official release of FIPS PUB 202 "SHA3" -// Revised 03-Sep-15 for portability + OpenSSL - style API - -#include "sha3.h" - -// update the state with given number of rounds - -static void sha3_keccakf(uint64_t st[25]) -{ - // constants - const uint64_t keccakf_rndc[24] = { - 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, - 0x8000000080008000, 0x000000000000808b, 0x0000000080000001, - 0x8000000080008081, 0x8000000000008009, 0x000000000000008a, - 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, - 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, - 0x8000000000008003, 0x8000000000008002, 0x8000000000000080, - 0x000000000000800a, 0x800000008000000a, 0x8000000080008081, - 0x8000000000008080, 0x0000000080000001, 0x8000000080008008 - }; - const int keccakf_rotc[24] = { - 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, - 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44 - }; - const int keccakf_piln[24] = { - 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, - 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1 - }; - - // variables - int i, j, r; - uint64_t t, bc[5]; - -#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ - uint8_t *v; - - // endianess conversion. this is redundant on little-endian targets - for (i = 0; i < 25; i++) { - v = (uint8_t *) &st[i]; - st[i] = ((uint64_t) v[0]) | (((uint64_t) v[1]) << 8) | - (((uint64_t) v[2]) << 16) | (((uint64_t) v[3]) << 24) | - (((uint64_t) v[4]) << 32) | (((uint64_t) v[5]) << 40) | - (((uint64_t) v[6]) << 48) | (((uint64_t) v[7]) << 56); - } -#endif - - // actual iteration - for (r = 0; r < KECCAKF_ROUNDS; r++) { - - // Theta - for (i = 0; i < 5; i++) - bc[i] = st[i] ^ st[i + 5] ^ st[i + 10] ^ st[i + 15] ^ st[i + 20]; - - for (i = 0; i < 5; i++) { - t = bc[(i + 4) % 5] ^ ROTL64(bc[(i + 1) % 5], 1); - for (j = 0; j < 25; j += 5) - st[j + i] ^= t; - } - - // Rho Pi - t = st[1]; - for (i = 0; i < 24; i++) { - j = keccakf_piln[i]; - bc[0] = st[j]; - st[j] = ROTL64(t, keccakf_rotc[i]); - t = bc[0]; - } - - // Chi - for (j = 0; j < 25; j += 5) { - for (i = 0; i < 5; i++) - bc[i] = st[j + i]; - for (i = 0; i < 5; i++) - st[j + i] ^= (~bc[(i + 1) % 5]) & bc[(i + 2) % 5]; - } - - // Iota - st[0] ^= keccakf_rndc[r]; - } - -#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ - // endianess conversion. this is redundant on little-endian targets - for (i = 0; i < 25; i++) { - v = (uint8_t *) &st[i]; - t = st[i]; - v[0] = t & 0xFF; - v[1] = (t >> 8) & 0xFF; - v[2] = (t >> 16) & 0xFF; - v[3] = (t >> 24) & 0xFF; - v[4] = (t >> 32) & 0xFF; - v[5] = (t >> 40) & 0xFF; - v[6] = (t >> 48) & 0xFF; - v[7] = (t >> 56) & 0xFF; - } -#endif -} - -// Initialize the context for SHA3 - -static int sha3_init(sha3_ctx_t *c, int mdlen) -{ - int i; - - for (i = 0; i < 25; i++) - c->st.q[i] = 0; - c->mdlen = mdlen; - c->rsiz = 200 - 2 * mdlen; - c->pt = 0; - - return 1; -} - -// update state with more data - -static int sha3_update(sha3_ctx_t *c, const void *data, size_t len) -{ - size_t i; - int j; - - j = c->pt; - for (i = 0; i < len; i++) { - c->st.b[j++] ^= ((const uint8_t *) data)[i]; - if (j >= c->rsiz) { - sha3_keccakf(c->st.q); - j = 0; - } - } - c->pt = j; - - return 1; -} - -// finalize and output a hash - -static int sha3_final(void *md, sha3_ctx_t *c) -{ - int i; - - c->st.b[c->pt] ^= 0x06; - c->st.b[c->rsiz - 1] ^= 0x80; - sha3_keccakf(c->st.q); - - for (i = 0; i < c->mdlen; i++) { - ((uint8_t *) md)[i] = c->st.b[i]; - } - - return 1; -} - -#if 0 -// compute a SHA-3 hash (md) of given byte length from "in" - -void *sha3(const void *in, size_t inlen, void *md, int mdlen) -{ - sha3_ctx_t sha3; - - sha3_init(&sha3, mdlen); - sha3_update(&sha3, in, inlen); - sha3_final(md, &sha3); - - return md; -} -#endif - -// SHAKE128 and SHAKE256 extensible-output functionality - -static void shake_xof(sha3_ctx_t *c) -{ - c->st.b[c->pt] ^= 0x1F; - c->st.b[c->rsiz - 1] ^= 0x80; - sha3_keccakf(c->st.q); - c->pt = 0; -} - -static void shake_out(sha3_ctx_t *c, void *out, size_t len) -{ - size_t i; - int j; - - j = c->pt; - for (i = 0; i < len; i++) { - if (j >= c->rsiz) { - sha3_keccakf(c->st.q); - j = 0; - } - ((uint8_t *) out)[i] = c->st.b[j++]; - } - c->pt = j; -} - diff --git a/Modules/_sha3/sha3.h b/Modules/_sha3/sha3.h deleted file mode 100644 index f973d6733ec2cc..00000000000000 --- a/Modules/_sha3/sha3.h +++ /dev/null @@ -1,49 +0,0 @@ -// sha3.h -// 19-Nov-11 Markku-Juhani O. Saarinen - -#ifndef SHA3_H -#define SHA3_H - -#include -#include - -#ifndef KECCAKF_ROUNDS -#define KECCAKF_ROUNDS 24 -#endif - -#ifndef ROTL64 -#define ROTL64(x, y) (((x) << (y)) | ((x) >> (64 - (y)))) -#endif - -// state context -typedef struct { - union { // state: - uint8_t b[200]; // 8-bit bytes - uint64_t q[25]; // 64-bit words - } st; - int pt, rsiz, mdlen; // these don't overflow -} sha3_ctx_t; - -// Compression function. -static void sha3_keccakf(uint64_t st[25]); - -// OpenSSL - like interfece -static int sha3_init(sha3_ctx_t *c, int mdlen); // mdlen = hash output in bytes -static int sha3_update(sha3_ctx_t *c, const void *data, size_t len); -static int sha3_final(void *md, sha3_ctx_t *c); // digest goes to md - -// compute a sha3 hash (md) of given byte length from "in" -#if 0 -static void *sha3(const void *in, size_t inlen, void *md, int mdlen); -#endif - -// SHAKE128 and SHAKE256 extensible-output functions -#define shake128_init(c) sha3_init(c, 16) -#define shake256_init(c) sha3_init(c, 32) -#define shake_update sha3_update - -static void shake_xof(sha3_ctx_t *c); -static void shake_out(sha3_ctx_t *c, void *out, size_t len); - -#endif - diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index 182754cca36d61..417abcc4626170 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -228,7 +228,7 @@ PyDoc_STRVAR(blobopen__doc__, static PyObject * blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, - int row, int readonly, const char *name); + sqlite3_int64 row, int readonly, const char *name); static PyObject * blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -263,7 +263,7 @@ blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyO Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; const char *table; const char *col; - int row; + sqlite3_int64 row; int readonly = 0; const char *name = "main"; @@ -297,8 +297,7 @@ blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyO PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - row = _PyLong_AsInt(args[2]); - if (row == -1 && PyErr_Occurred()) { + if (!sqlite3_int64_converter(args[2], &row)) { goto exit; } if (!noptargs) { @@ -1666,4 +1665,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=8b03149c115ee6da input=a9049054013a1b77]*/ +/*[clinic end generated code: output=834a99827555bf1a input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index aec3aa8bbf4ed8..7bbb462ed54dfa 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -118,6 +118,20 @@ autocommit_converter(PyObject *val, enum autocommit_mode *result) return 0; } +static int +sqlite3_int64_converter(PyObject *obj, sqlite3_int64 *result) +{ + if (!PyLong_Check(obj)) { + PyErr_SetString(PyExc_TypeError, "expected 'int'"); + return 0; + } + *result = _pysqlite_long_as_int64(obj); + if (PyErr_Occurred()) { + return 0; + } + return 1; +} + #define clinic_state() (pysqlite_get_state_by_type(Py_TYPE(self))) #include "clinic/connection.c.h" #undef clinic_state @@ -188,8 +202,12 @@ class Autocommit_converter(CConverter): type = "enum autocommit_mode" converter = "autocommit_converter" +class sqlite3_int64_converter(CConverter): + type = "sqlite3_int64" + converter = "sqlite3_int64_converter" + [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=bc2aa6c7ba0c5f8f]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=dff8760fb1eba6a1]*/ // NB: This needs to be in sync with the sqlite3.connect docstring /*[clinic input] @@ -483,7 +501,7 @@ _sqlite3.Connection.blobopen as blobopen Table name. column as col: str Column name. - row: int + row: sqlite3_int64 Row index. / * @@ -497,8 +515,8 @@ Open and return a BLOB object. static PyObject * blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, - int row, int readonly, const char *name) -/*[clinic end generated code: output=0c8e2e58516d0b5c input=fa73c83aa7a7ddee]*/ + sqlite3_int64 row, int readonly, const char *name) +/*[clinic end generated code: output=6a02d43efb885d1c input=23576bd1108d8774]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 9c42faa232c70d..27bd42f4595e1c 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -785,6 +785,7 @@ module_exec(PyObject *module) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 4b6290a5967932..f8a1a05a318889 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -3221,6 +3221,7 @@ sre_exec(PyObject *m) static PyModuleDef_Slot sre_slots[] = { {Py_mod_exec, sre_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index c9e2f24d66cc00..016a5a5cbca548 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -6161,6 +6161,9 @@ static PyModuleDef_Slot sslmodule_slots[] = { {Py_mod_exec, sslmodule_init_constants}, {Py_mod_exec, sslmodule_init_versioninfo}, {Py_mod_exec, sslmodule_init_strings}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_stat.c b/Modules/_stat.c index 546e6a5f94ca15..4218799103b59d 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -612,6 +612,7 @@ stat_exec(PyObject *module) static PyModuleDef_Slot stat_slots[] = { {Py_mod_exec, stat_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_statisticsmodule.c b/Modules/_statisticsmodule.c index b9d1e4f1616036..1d5465fbe6d04e 100644 --- a/Modules/_statisticsmodule.c +++ b/Modules/_statisticsmodule.c @@ -129,6 +129,7 @@ PyDoc_STRVAR(statistics_doc, "Accelerators for the statistics module.\n"); static struct PyModuleDef_Slot _statisticsmodule_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_struct.c b/Modules/_struct.c index 3db7b991acd0a1..26434f714de5cc 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2572,6 +2572,7 @@ _structmodule_exec(PyObject *m) static PyModuleDef_Slot _structmodule_slots[] = { {Py_mod_exec, _structmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_testcapi/immortal.c b/Modules/_testcapi/immortal.c index 10e1733d08a9ea..9f81389811c645 100644 --- a/Modules/_testcapi/immortal.c +++ b/Modules/_testcapi/immortal.c @@ -1,6 +1,6 @@ #include "parts.h" -int verify_immortality(PyObject *object) +int verify_immortality(PyObject *object) { assert(_Py_IsImmortal(object)); Py_ssize_t old_count = Py_REFCNT(object); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index ae70a33eb4d1ed..79ab7d3f5555c2 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1488,6 +1488,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) int allow_threads = -1; int allow_daemon_threads = -1; int check_multi_interp_extensions = -1; + int own_gil = -1; int r; PyThreadState *substate, *mainstate; /* only initialise 'cflags.cf_flags' to test backwards compatibility */ @@ -1500,13 +1501,15 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) "allow_threads", "allow_daemon_threads", "check_multi_interp_extensions", + "own_gil", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "s$pppppp:run_in_subinterp_with_config", kwlist, + "s$ppppppp:run_in_subinterp_with_config", kwlist, &code, &use_main_obmalloc, &allow_fork, &allow_exec, &allow_threads, &allow_daemon_threads, - &check_multi_interp_extensions)) { + &check_multi_interp_extensions, + &own_gil)) { return NULL; } if (use_main_obmalloc < 0) { @@ -1525,6 +1528,10 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_ValueError, "missing allow_threads"); return NULL; } + if (own_gil < 0) { + PyErr_SetString(PyExc_ValueError, "missing own_gil"); + return NULL; + } if (allow_daemon_threads < 0) { PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads"); return NULL; @@ -1545,6 +1552,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) .allow_threads = allow_threads, .allow_daemon_threads = allow_daemon_threads, .check_multi_interp_extensions = check_multi_interp_extensions, + .own_gil = own_gil, }; PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); if (PyStatus_Exception(status)) { @@ -4221,7 +4229,7 @@ PyInit__testcapi(void) return NULL; } int ret = PyModule_AddType(m, (PyTypeObject*)ObjExtraData_Type); - Py_DECREF(&ObjExtraData_Type); + Py_DECREF(ObjExtraData_Type); if (ret < 0) { return NULL; } diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 91fdee24d328d9..6ff55a2755cf5a 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -9,6 +9,19 @@ #include "Python.h" + +// Used for clone_with_conv_f1 and clone_with_conv_v2 +typedef struct { + const char *name; +} custom_t; + +static int +custom_converter(PyObject *obj, custom_t *val) +{ + return 1; +} + + #include "clinic/_testclinic.c.h" @@ -1117,6 +1130,70 @@ gh_99240_double_free_impl(PyObject *module, char *a, char *b) } +/*[clinic input] +_testclinic.clone_f1 as clone_f1 + path: str +[clinic start generated code]*/ + +static PyObject * +clone_f1_impl(PyObject *module, const char *path) +/*[clinic end generated code: output=8c30b5620ba86715 input=9c614b7f025ebf70]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +_testclinic.clone_f2 as clone_f2 = _testclinic.clone_f1 +[clinic start generated code]*/ + +static PyObject * +clone_f2_impl(PyObject *module, const char *path) +/*[clinic end generated code: output=6aa1c39bec3f5d9b input=1aaaf47d6ed2324a]*/ +{ + Py_RETURN_NONE; +} + + +/*[python input] +class custom_t_converter(CConverter): + type = 'custom_t' + converter = 'custom_converter' + + def pre_render(self): + self.c_default = f'''{{ + .name = "{self.function.name}", + }}''' + +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=b2fb801e99a06bf6]*/ + + +/*[clinic input] +_testclinic.clone_with_conv_f1 as clone_with_conv_f1 + path: custom_t = None +[clinic start generated code]*/ + +static PyObject * +clone_with_conv_f1_impl(PyObject *module, custom_t path) +/*[clinic end generated code: output=f7e030ffd5439cb0 input=bc77bc80dec3f46d]*/ +{ + return PyUnicode_FromString(path.name); +} + + +/*[clinic input] +_testclinic.clone_with_conv_f2 as clone_with_conv_f2 = _testclinic.clone_with_conv_f1 +[clinic start generated code]*/ + +static PyObject * +clone_with_conv_f2_impl(PyObject *module, custom_t path) +/*[clinic end generated code: output=9d7fdd6a75eecee4 input=cff459a205fa83bb]*/ +{ + return PyUnicode_FromString(path.name); +} + + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF OBJECTS_CONVERTER_METHODDEF @@ -1168,6 +1245,10 @@ static PyMethodDef tester_methods[] = { GH_32092_KW_PASS_METHODDEF GH_99233_REFCOUNT_METHODDEF GH_99240_DOUBLE_FREE_METHODDEF + CLONE_F1_METHODDEF + CLONE_F2_METHODDEF + CLONE_WITH_CONV_F1_METHODDEF + CLONE_WITH_CONV_F2_METHODDEF {NULL, NULL} }; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 1e38f1aa63499b..40ad6f88868daf 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -593,17 +593,19 @@ _testinternalcapi.compiler_codegen -> object ast: object filename: object optimize: int + compile_mode: int = 0 Apply compiler code generation to an AST. [clinic start generated code]*/ static PyObject * _testinternalcapi_compiler_codegen_impl(PyObject *module, PyObject *ast, - PyObject *filename, int optimize) -/*[clinic end generated code: output=fbbbbfb34700c804 input=e9fbe6562f7f75e4]*/ + PyObject *filename, int optimize, + int compile_mode) +/*[clinic end generated code: output=40a68f6e13951cc8 input=a0e00784f1517cd7]*/ { PyCompilerFlags *flags = NULL; - return _PyCompile_CodeGen(ast, filename, flags, optimize); + return _PyCompile_CodeGen(ast, filename, flags, optimize, compile_mode); } @@ -729,6 +731,13 @@ get_interp_settings(PyObject *self, PyObject *args) return NULL; } + /* "own GIL" */ + PyObject *own_gil = interp->ceval.own_gil ? Py_True : Py_False; + if (PyDict_SetItemString(settings, "own_gil", own_gil) != 0) { + Py_DECREF(settings); + return NULL; + } + return settings; } @@ -789,6 +798,7 @@ module_exec(PyObject *module) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index cf8990a2df0a9b..ca71b6156b005d 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -441,6 +441,7 @@ static int execfunc(PyObject *m) static PyModuleDef_Slot main_slots[] = { {Py_mod_exec, execfunc}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; @@ -680,6 +681,27 @@ PyInit__testmultiphase_export_unreported_exception(void) return PyModuleDef_Init(&main_def); } +static PyObject* +createfunc_noop(PyObject *spec, PyModuleDef *def) +{ + return PyModule_New("spam"); +} + +static PyModuleDef_Slot slots_multiple_create_slots[] = { + {Py_mod_create, createfunc_noop}, + {Py_mod_create, createfunc_noop}, + {0, NULL}, +}; + +static PyModuleDef def_multiple_create_slots = TEST_MODULE_DEF( + "_testmultiphase_multiple_create_slots", slots_multiple_create_slots, NULL); + +PyMODINIT_FUNC +PyInit__testmultiphase_multiple_create_slots(void) +{ + return PyModuleDef_Init(&def_multiple_create_slots); +} + static PyObject* createfunc_null(PyObject *spec, PyModuleDef *def) { @@ -745,6 +767,7 @@ PyInit__testmultiphase_create_unreported_exception(void) static PyModuleDef_Slot slots_nonmodule_with_exec_slots[] = { {Py_mod_create, createfunc_nonmodule}, {Py_mod_exec, execfunc}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; @@ -765,6 +788,7 @@ execfunc_err(PyObject *mod) static PyModuleDef_Slot slots_exec_err[] = { {Py_mod_exec, execfunc_err}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; @@ -786,6 +810,7 @@ execfunc_raise(PyObject *spec) static PyModuleDef_Slot slots_exec_raise[] = { {Py_mod_exec, execfunc_raise}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; @@ -807,6 +832,7 @@ execfunc_unreported_exception(PyObject *mod) static PyModuleDef_Slot slots_exec_unreported_exception[] = { {Py_mod_exec, execfunc_unreported_exception}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; @@ -845,6 +871,7 @@ meth_state_access_exec(PyObject *m) static PyModuleDef_Slot meth_state_access_slots[] = { {Py_mod_exec, meth_state_access_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; @@ -884,3 +911,59 @@ PyInit__test_module_state_shared(void) } return module; } + + +/* multiple interpreters support */ + +static PyModuleDef_Slot slots_multiple_multiple_interpreters_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL}, +}; + +static PyModuleDef def_multiple_multiple_interpreters_slots = TEST_MODULE_DEF( + "_testmultiphase_multiple_multiple_interpreters_slots", + slots_multiple_multiple_interpreters_slots, + NULL); + +PyMODINIT_FUNC +PyInit__testmultiphase_multiple_multiple_interpreters_slots(void) +{ + return PyModuleDef_Init(&def_multiple_multiple_interpreters_slots); +} + +static PyModuleDef_Slot non_isolated_slots[] = { + {Py_mod_exec, execfunc}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL}, +}; + +static PyModuleDef non_isolated_def = TEST_MODULE_DEF("_test_non_isolated", + non_isolated_slots, + testexport_methods); + +PyMODINIT_FUNC +PyInit__test_non_isolated(void) +{ + return PyModuleDef_Init(&non_isolated_def); +} + + +static PyModuleDef_Slot shared_gil_only_slots[] = { + {Py_mod_exec, execfunc}, + /* Note that Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED is the default. + We put it here explicitly to draw attention to the contrast + with Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. */ + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED}, + {0, NULL}, +}; + +static PyModuleDef shared_gil_only_def = TEST_MODULE_DEF("_test_shared_gil_only", + shared_gil_only_slots, + testexport_methods); + +PyMODINIT_FUNC +PyInit__test_shared_gil_only(void) +{ + return PyModuleDef_Init(&shared_gil_only_def); +} diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index fd2fd9ab25f113..5d753b4a0ebc5e 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1710,6 +1710,7 @@ The 'threading' module provides a more convenient interface."); static PyModuleDef_Slot thread_module_slots[] = { {Py_mod_exec, thread_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index d69c5636486da9..c5714d5e7d5a0f 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -7,6 +7,7 @@ #include "pycore_runtime.h" // _Py_ID() #include "pycore_traceback.h" #include +#include "frameobject.h" // _PyInterpreterFrame_GetLine #include // malloc() @@ -257,7 +258,7 @@ static void tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) { frame->filename = &_Py_STR(anon_unknown); - int lineno = _PyInterpreterFrame_GetLine(pyframe); + int lineno = PyUnstable_InterpreterFrame_GetLine(pyframe); if (lineno < 0) { lineno = 0; } diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 262dddb63fd5fe..64286375636aff 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -36,6 +36,7 @@ PyDoc_STRVAR(typing_doc, "Accelerators for the typing module.\n"); static struct PyModuleDef_Slot _typingmodule_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index eae38f5c98cc7f..ed3b2fedfd4d88 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -106,6 +106,7 @@ static PyMethodDef uuid_methods[] = { static PyModuleDef_Slot uuid_slots[] = { {Py_mod_exec, uuid_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_weakref.c b/Modules/_weakref.c index 157a852ae9a378..387b8fa9d0a6f1 100644 --- a/Modules/_weakref.c +++ b/Modules/_weakref.c @@ -174,6 +174,7 @@ weakref_exec(PyObject *module) static struct PyModuleDef_Slot weakref_slots[] = { {Py_mod_exec, weakref_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_winapi.c b/Modules/_winapi.c index fa380b8b798405..473bcb4736e925 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -2259,6 +2259,7 @@ static int winapi_exec(PyObject *m) static PyModuleDef_Slot winapi_slots[] = { {Py_mod_exec, winapi_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 13b005eaef9866..616dd577688116 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -2418,6 +2418,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 95273ab278d996..d7daae254638ec 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -513,6 +513,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) // Create and initialize the new interpreter. PyThreadState *save_tstate = _PyThreadState_GET(); + assert(save_tstate != NULL); const PyInterpreterConfig config = isolated ? (PyInterpreterConfig)_PyInterpreterConfig_INIT : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; @@ -821,6 +822,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index fb0c191d2c494d..37d402824853f0 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -526,13 +526,20 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { #if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_sre_compile) static int SRE_COMPILE_INITIALIZED = 0; if (!SRE_COMPILE_INITIALIZED && !init_sre_compile()) { - PyErr_Print(); - abort(); + if (!PyErr_ExceptionMatches(PyExc_DeprecationWarning)) { + PyErr_Print(); + abort(); + } + else { + PyErr_Clear(); + } } else { SRE_COMPILE_INITIALIZED = 1; } - rv |= _run_fuzz(data, size, fuzz_sre_compile); + if (SRE_COMPILE_INITIALIZED) { + rv |= _run_fuzz(data, size, fuzz_sre_compile); + } #endif #if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_sre_match) static int SRE_MATCH_INITIALIZED = 0; diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index c215a75b804fdb..3b2d282d65cab9 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -2822,7 +2822,10 @@ zoneinfomodule_exec(PyObject *m) } static PyModuleDef_Slot zoneinfomodule_slots[] = { - {Py_mod_exec, zoneinfomodule_exec}, {0, NULL}}; + {Py_mod_exec, zoneinfomodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL}, +}; static struct PyModuleDef zoneinfomodule = { .m_base = PyModuleDef_HEAD_INIT, diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 798a7629257966..f94bbec8e0bb3c 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -3111,6 +3111,7 @@ array_modexec(PyObject *m) static PyModuleDef_Slot arrayslots[] = { {Py_mod_exec, array_modexec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index 47afd7f0751039..5882d405636400 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -314,12 +314,18 @@ upon normal program termination.\n\ Two public functions, register and unregister, are defined.\n\ "); +static PyModuleDef_Slot atexitmodule_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL} +}; + static struct PyModuleDef atexitmodule = { PyModuleDef_HEAD_INIT, .m_name = "atexit", .m_doc = atexit__doc__, .m_size = 0, .m_methods = atexit_methods, + .m_slots = atexitmodule_slots, }; PyMODINIT_FUNC diff --git a/Modules/audioop.c b/Modules/audioop.c index 9325f82f9a17e0..604306d449265c 100644 --- a/Modules/audioop.c +++ b/Modules/audioop.c @@ -1975,6 +1975,7 @@ audioop_exec(PyObject* module) static PyModuleDef_Slot audioop_slots[] = { {Py_mod_exec, audioop_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/binascii.c b/Modules/binascii.c index 95ddb26988d6c9..4ecff4793be9a0 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -1291,6 +1291,7 @@ binascii_exec(PyObject *module) { static PyModuleDef_Slot binascii_slots[] = { {Py_mod_exec, binascii_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h index e553ff3e17b898..36bc7024df9acc 100644 --- a/Modules/cjkcodecs/cjkcodecs.h +++ b/Modules/cjkcodecs/cjkcodecs.h @@ -502,6 +502,7 @@ static struct PyMethodDef _cjk_methods[] = { static PyModuleDef_Slot _cjk_slots[] = { {Py_mod_exec, _cjk_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 233fc3020fd6a8..b501e4fb923232 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -2062,6 +2062,7 @@ static struct PyMethodDef _multibytecodec_methods[] = { static PyModuleDef_Slot _multibytecodec_slots[] = { {Py_mod_exec, _multibytecodec_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 831f58ca650aab..cc69f5c3d2fe9f 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -2817,4 +2817,262 @@ gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=e8211606b03d733a input=a9049054013a1b77]*/ + +PyDoc_STRVAR(clone_f1__doc__, +"clone_f1($module, /, path)\n" +"--\n" +"\n"); + +#define CLONE_F1_METHODDEF \ + {"clone_f1", _PyCFunction_CAST(clone_f1), METH_FASTCALL|METH_KEYWORDS, clone_f1__doc__}, + +static PyObject * +clone_f1_impl(PyObject *module, const char *path); + +static PyObject * +clone_f1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "clone_f1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + const char *path; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("clone_f1", "argument 'path'", "str", args[0]); + goto exit; + } + Py_ssize_t path_length; + path = PyUnicode_AsUTF8AndSize(args[0], &path_length); + if (path == NULL) { + goto exit; + } + if (strlen(path) != (size_t)path_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = clone_f1_impl(module, path); + +exit: + return return_value; +} + +PyDoc_STRVAR(clone_f2__doc__, +"clone_f2($module, /, path)\n" +"--\n" +"\n"); + +#define CLONE_F2_METHODDEF \ + {"clone_f2", _PyCFunction_CAST(clone_f2), METH_FASTCALL|METH_KEYWORDS, clone_f2__doc__}, + +static PyObject * +clone_f2_impl(PyObject *module, const char *path); + +static PyObject * +clone_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "clone_f2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + const char *path; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("clone_f2", "argument 'path'", "str", args[0]); + goto exit; + } + Py_ssize_t path_length; + path = PyUnicode_AsUTF8AndSize(args[0], &path_length); + if (path == NULL) { + goto exit; + } + if (strlen(path) != (size_t)path_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = clone_f2_impl(module, path); + +exit: + return return_value; +} + +PyDoc_STRVAR(clone_with_conv_f1__doc__, +"clone_with_conv_f1($module, /, path=None)\n" +"--\n" +"\n"); + +#define CLONE_WITH_CONV_F1_METHODDEF \ + {"clone_with_conv_f1", _PyCFunction_CAST(clone_with_conv_f1), METH_FASTCALL|METH_KEYWORDS, clone_with_conv_f1__doc__}, + +static PyObject * +clone_with_conv_f1_impl(PyObject *module, custom_t path); + +static PyObject * +clone_with_conv_f1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "clone_with_conv_f1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + custom_t path = { + .name = "clone_with_conv_f1", + }; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (!custom_converter(args[0], &path)) { + goto exit; + } +skip_optional_pos: + return_value = clone_with_conv_f1_impl(module, path); + +exit: + return return_value; +} + +PyDoc_STRVAR(clone_with_conv_f2__doc__, +"clone_with_conv_f2($module, /, path=None)\n" +"--\n" +"\n"); + +#define CLONE_WITH_CONV_F2_METHODDEF \ + {"clone_with_conv_f2", _PyCFunction_CAST(clone_with_conv_f2), METH_FASTCALL|METH_KEYWORDS, clone_with_conv_f2__doc__}, + +static PyObject * +clone_with_conv_f2_impl(PyObject *module, custom_t path); + +static PyObject * +clone_with_conv_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "clone_with_conv_f2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + custom_t path = { + .name = "clone_with_conv_f2", + }; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (!custom_converter(args[0], &path)) { + goto exit; + } +skip_optional_pos: + return_value = clone_with_conv_f2_impl(module, path); + +exit: + return return_value; +} +/*[clinic end generated code: output=f58202a6e5df2d16 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index 89573222572594..41dd50437956c4 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -9,7 +9,7 @@ preserve PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, -"compiler_codegen($module, /, ast, filename, optimize)\n" +"compiler_codegen($module, /, ast, filename, optimize, compile_mode=0)\n" "--\n" "\n" "Apply compiler code generation to an AST."); @@ -19,7 +19,8 @@ PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, static PyObject * _testinternalcapi_compiler_codegen_impl(PyObject *module, PyObject *ast, - PyObject *filename, int optimize); + PyObject *filename, int optimize, + int compile_mode); static PyObject * _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -27,14 +28,14 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 4 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(ast), &_Py_ID(filename), &_Py_ID(optimize), }, + .ob_item = { &_Py_ID(ast), &_Py_ID(filename), &_Py_ID(optimize), &_Py_ID(compile_mode), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -43,19 +44,21 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"ast", "filename", "optimize", NULL}; + static const char * const _keywords[] = {"ast", "filename", "optimize", "compile_mode", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "compiler_codegen", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *ast; PyObject *filename; int optimize; + int compile_mode = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 4, 0, argsbuf); if (!args) { goto exit; } @@ -65,7 +68,15 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s if (optimize == -1 && PyErr_Occurred()) { goto exit; } - return_value = _testinternalcapi_compiler_codegen_impl(module, ast, filename, optimize); + if (!noptargs) { + goto skip_optional_pos; + } + compile_mode = _PyLong_AsInt(args[3]); + if (compile_mode == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: + return_value = _testinternalcapi_compiler_codegen_impl(module, ast, filename, optimize, compile_mode); exit: return return_value; @@ -190,4 +201,4 @@ _testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=d5e08c9d67f9721f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ab661d56a14b1a1c input=a9049054013a1b77]*/ diff --git a/Modules/_sha3/clinic/sha3module.c.h b/Modules/clinic/sha3module.c.h similarity index 98% rename from Modules/_sha3/clinic/sha3module.c.h rename to Modules/clinic/sha3module.c.h index a0c7c1c043e515..299803a3420bf6 100644 --- a/Modules/_sha3/clinic/sha3module.c.h +++ b/Modules/clinic/sha3module.c.h @@ -12,7 +12,7 @@ PyDoc_STRVAR(py_sha3_new__doc__, "sha3_224(data=b\'\', /, *, usedforsecurity=True)\n" "--\n" "\n" -"Return a new BLAKE2b hash object."); +"Return a new SHA3 hash object."); static PyObject * py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity); @@ -193,4 +193,4 @@ _sha3_shake_128_hexdigest(SHA3object *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=747c3f34ddd14063 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=907cb475f3dc9ee0 input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index b4f7e5424b4ccf..914a697f8e173b 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -1411,6 +1411,7 @@ cmath_exec(PyObject *mod) static PyModuleDef_Slot cmath_slots[] = { {Py_mod_exec, cmath_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index df4e494ba8a973..fddde960a5fe9a 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -940,6 +940,7 @@ errno_exec(PyObject *module) static PyModuleDef_Slot errno_slots[] = { {Py_mod_exec, errno_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 9b4e4199cdc20a..428b090193f093 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1274,6 +1274,8 @@ PyExec_faulthandler(PyObject *module) { static PyModuleDef_Slot faulthandler_slots[] = { {Py_mod_exec, PyExec_faulthandler}, + // XXX gh-103092: fix isolation. + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 9a8ec8dc9858d7..6ca0b62bc5dca8 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -686,6 +686,7 @@ fcntl_exec(PyObject *module) static PyModuleDef_Slot fcntl_slots[] = { {Py_mod_exec, fcntl_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index f4d5186ff1552e..26ddcdd538a4d4 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -2044,6 +2044,7 @@ gcmodule_exec(PyObject *module) static PyModuleDef_Slot gcmodule_slots[] = { {Py_mod_exec, gcmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index f6298ca0ee84c1..57cdde6064c24e 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -327,6 +327,7 @@ grpmodule_exec(PyObject *module) static PyModuleDef_Slot grpmodule_slots[] = { {Py_mod_exec, grpmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index c986e02867ca82..555eab09935e9e 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -4693,6 +4693,7 @@ itertoolsmodule_exec(PyObject *mod) static struct PyModuleDef_Slot itertoolsmodule_slots[] = { {Py_mod_exec, itertoolsmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 4a2381d9611776..3737a9654575ab 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -4064,6 +4064,7 @@ static PyMethodDef math_methods[] = { static PyModuleDef_Slot math_slots[] = { {Py_mod_exec, math_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/md5module.c b/Modules/md5module.c index 4f7bc77a8836a3..86605771d9643f 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -340,6 +340,7 @@ md5_exec(PyObject *m) static PyModuleDef_Slot _md5_slots[] = { {Py_mod_exec, md5_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index fe76ca6eafaa88..a470dd3c2f3bba 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1713,6 +1713,7 @@ mmap_exec(PyObject *module) static PyModuleDef_Slot mmap_slots[] = { {Py_mod_exec, mmap_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/nismodule.c b/Modules/nismodule.c index ec7f6d8031e84b..6d094490cea731 100644 --- a/Modules/nismodule.c +++ b/Modules/nismodule.c @@ -502,6 +502,9 @@ nis_exec(PyObject *module) static PyModuleDef_Slot nis_slots[] = { {Py_mod_exec, nis_exec}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 02c0f401be4c9e..ac637316583d2d 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -2050,6 +2050,7 @@ overlapped_exec(PyObject *module) static PyModuleDef_Slot overlapped_slots[] = { {Py_mod_exec, overlapped_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index b395c265c72d0c..5022fdeb03703a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -16793,6 +16793,7 @@ posixmodule_exec(PyObject *m) static PyModuleDef_Slot posixmodile_slots[] = { {Py_mod_exec, posixmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index a757380bd09f70..cc2e2a43893971 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -336,6 +336,7 @@ pwdmodule_exec(PyObject *module) static PyModuleDef_Slot pwdmodule_slots[] = { {Py_mod_exec, pwdmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 0a744998b6c514..c0fbd4d39f0096 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -2056,6 +2056,9 @@ pyexpat_free(void *module) static PyModuleDef_Slot pyexpat_slots[] = { {Py_mod_exec, pyexpat_exec}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/resource.c b/Modules/resource.c index a97fb870062b82..2a8158c9be5359 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -514,6 +514,7 @@ resource_exec(PyObject *module) static struct PyModuleDef_Slot resource_slots[] = { {Py_mod_exec, resource_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 5a1e40d0b4a482..79bd5b59ab68f9 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -2651,6 +2651,7 @@ _select_exec(PyObject *m) static PyModuleDef_Slot _select_slots[] = { {Py_mod_exec, _select_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/sha1module.c b/Modules/sha1module.c index f8d4056fd34b65..bdb76c56f1a6e8 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -344,6 +344,7 @@ _sha1_exec(PyObject *module) static PyModuleDef_Slot _sha1_slots[] = { {Py_mod_exec, _sha1_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 72de20b44762d7..37d9b5c538fd0b 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -785,6 +785,7 @@ static int sha2_exec(PyObject *module) static PyModuleDef_Slot _sha2_slots[] = { {Py_mod_exec, sha2_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_sha3/sha3module.c b/Modules/sha3module.c similarity index 75% rename from Modules/_sha3/sha3module.c rename to Modules/sha3module.c index 633a0c0ea08d2a..f05187498a19b3 100644 --- a/Modules/_sha3/sha3module.c +++ b/Modules/sha3module.c @@ -22,23 +22,9 @@ #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" // _PyType_GetModuleState() -#include "../hashlib.h" - -#include "sha3.c" +#include "hashlib.h" #define SHA3_MAX_DIGESTSIZE 64 /* 64 Bytes (512 Bits) for 224 to 512 */ -#define SHA3_LANESIZE 0 -#define SHA3_state sha3_ctx_t -#define SHA3_init sha3_init -#define SHA3_process sha3_update -#define SHA3_done(state, digest) sha3_final(digest, state) -#define SHA3_squeeze(state, out, len) shake_xof(state), shake_out(state, out, len) -#define SHA3_copystate(dest, src) memcpy(&(dest), &(src), sizeof(SHA3_state)) - -// no optimization -#define KeccakOpt 0 - -typedef enum { SUCCESS = 1, FAIL = 0, BAD_HASHLEN = 2 } HashReturn; typedef struct { PyTypeObject *sha3_224_type; @@ -70,10 +56,11 @@ class _sha3.shake_256 "SHA3object *" "&SHAKE256type" /* The structure for storing SHA3 info */ +#include "_hacl/Hacl_Hash_SHA3.h" + typedef struct { PyObject_HEAD - SHA3_state hash_state; - PyThread_type_lock lock; + Hacl_Streaming_Keccak_state *hash_state; } SHA3object; #include "clinic/sha3module.c.h" @@ -86,10 +73,23 @@ newSHA3object(PyTypeObject *type) if (newobj == NULL) { return NULL; } - newobj->lock = NULL; return newobj; } +static void sha3_update(Hacl_Streaming_Keccak_state *state, uint8_t *buf, Py_ssize_t len) { + /* Note: we explicitly ignore the error code on the basis that it would take > + * 1 billion years to hash more than 2^64 bytes. */ +#if PY_SSIZE_T_MAX > UINT32_MAX + while (len > UINT32_MAX) { + Hacl_Streaming_Keccak_update(state, buf, UINT32_MAX); + len -= UINT32_MAX; + buf += UINT32_MAX; + } +#endif + /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ + Hacl_Streaming_Keccak_update(state, buf, (uint32_t) len); +} + /*[clinic input] @classmethod _sha3.sha3_224.__new__ as py_sha3_new @@ -98,14 +98,13 @@ _sha3.sha3_224.__new__ as py_sha3_new * usedforsecurity: bool = True -Return a new BLAKE2b hash object. +Return a new SHA3 hash object. [clinic start generated code]*/ static PyObject * py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) -/*[clinic end generated code: output=90409addc5d5e8b0 input=bcfcdf2e4368347a]*/ +/*[clinic end generated code: output=90409addc5d5e8b0 input=637e5f8f6a93982a]*/ { - HashReturn res; Py_buffer buf = {NULL, NULL}; SHA3State *state = _PyType_GetModuleState(type); SHA3object *self = newSHA3object(type); @@ -116,49 +115,29 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) assert(state != NULL); if (type == state->sha3_224_type) { - res = sha3_init(&self->hash_state, 28); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_224); } else if (type == state->sha3_256_type) { - res = sha3_init(&self->hash_state, 32); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_256); } else if (type == state->sha3_384_type) { - res = sha3_init(&self->hash_state, 48); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_384); } else if (type == state->sha3_512_type) { - res = sha3_init(&self->hash_state, 64); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_512); } else if (type == state->shake_128_type) { - res = sha3_init(&self->hash_state, 16); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_Shake128); } else if (type == state->shake_256_type) { - res = sha3_init(&self->hash_state, 32); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_Shake256); } else { PyErr_BadInternalCall(); goto error; } - if (res != SUCCESS) { - PyErr_SetString(PyExc_RuntimeError, - "internal error in SHA3 initialize()"); - goto error; - } - if (data) { GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); - if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* invariant: New objects can't be accessed by other code yet, - * thus it's safe to release the GIL without locking the object. - */ - Py_BEGIN_ALLOW_THREADS - res = SHA3_process(&self->hash_state, buf.buf, buf.len); - Py_END_ALLOW_THREADS - } - else { - res = SHA3_process(&self->hash_state, buf.buf, buf.len); - } - if (res != SUCCESS) { - PyErr_SetString(PyExc_RuntimeError, - "internal error in SHA3 Update()"); - goto error; - } - PyBuffer_Release(&buf); + sha3_update(self->hash_state, buf.buf, buf.len); } + PyBuffer_Release(&buf); + return (PyObject *)self; error: @@ -177,10 +156,7 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) static void SHA3_dealloc(SHA3object *self) { - if (self->lock) { - PyThread_free_lock(self->lock); - } - + Hacl_Streaming_Keccak_free(self->hash_state); PyTypeObject *tp = Py_TYPE(self); PyObject_Free(self); Py_DECREF(tp); @@ -205,9 +181,7 @@ _sha3_sha3_224_copy_impl(SHA3object *self) if ((newobj = newSHA3object(Py_TYPE(self))) == NULL) { return NULL; } - ENTER_HASHLIB(self); - SHA3_copystate(newobj->hash_state, self->hash_state); - LEAVE_HASHLIB(self); + newobj->hash_state = Hacl_Streaming_Keccak_copy(self->hash_state); return (PyObject *)newobj; } @@ -222,20 +196,12 @@ static PyObject * _sha3_sha3_224_digest_impl(SHA3object *self) /*[clinic end generated code: output=fd531842e20b2d5b input=5b2a659536bbd248]*/ { - unsigned char digest[SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE]; - SHA3_state temp; - HashReturn res; - - ENTER_HASHLIB(self); - SHA3_copystate(temp, self->hash_state); - LEAVE_HASHLIB(self); - res = SHA3_done(&temp, digest); - if (res != SUCCESS) { - PyErr_SetString(PyExc_RuntimeError, "internal error in SHA3 Final()"); - return NULL; - } + unsigned char digest[SHA3_MAX_DIGESTSIZE]; + // This function errors out if the algorithm is Shake. Here, we know this + // not to be the case, and therefore do not perform error checking. + Hacl_Streaming_Keccak_finish(self->hash_state, digest); return PyBytes_FromStringAndSize((const char *)digest, - self->hash_state.mdlen); + Hacl_Streaming_Keccak_hash_len(self->hash_state)); } @@ -249,21 +215,10 @@ static PyObject * _sha3_sha3_224_hexdigest_impl(SHA3object *self) /*[clinic end generated code: output=75ad03257906918d input=2d91bb6e0d114ee3]*/ { - unsigned char digest[SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE]; - SHA3_state temp; - HashReturn res; - - /* Get the raw (binary) digest value */ - ENTER_HASHLIB(self); - SHA3_copystate(temp, self->hash_state); - LEAVE_HASHLIB(self); - res = SHA3_done(&temp, digest); - if (res != SUCCESS) { - PyErr_SetString(PyExc_RuntimeError, "internal error in SHA3 Final()"); - return NULL; - } + unsigned char digest[SHA3_MAX_DIGESTSIZE]; + Hacl_Streaming_Keccak_finish(self->hash_state, digest); return _Py_strhex((const char *)digest, - self->hash_state.mdlen); + Hacl_Streaming_Keccak_hash_len(self->hash_state)); } @@ -281,36 +236,8 @@ _sha3_sha3_224_update(SHA3object *self, PyObject *data) /*[clinic end generated code: output=d3223352286ed357 input=a887f54dcc4ae227]*/ { Py_buffer buf; - HashReturn res; - GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - - /* add new data, the function takes the length in bits not bytes */ - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); - } - /* Once a lock exists all code paths must be synchronized. We have to - * release the GIL even for small buffers as acquiring the lock may take - * an unlimited amount of time when another thread updates this object - * with lots of data. */ - if (self->lock) { - Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); - res = SHA3_process(&self->hash_state, buf.buf, buf.len); - PyThread_release_lock(self->lock); - Py_END_ALLOW_THREADS - } - else { - res = SHA3_process(&self->hash_state, buf.buf, buf.len); - } - - if (res != SUCCESS) { - PyBuffer_Release(&buf); - PyErr_SetString(PyExc_RuntimeError, - "internal error in SHA3 Update()"); - return NULL; - } - + sha3_update(self->hash_state, buf.buf, buf.len); PyBuffer_Release(&buf); Py_RETURN_NONE; } @@ -328,7 +255,7 @@ static PyMethodDef SHA3_methods[] = { static PyObject * SHA3_get_block_size(SHA3object *self, void *closure) { - int rate = self->hash_state.rsiz; + uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state); return PyLong_FromLong(rate); } @@ -363,14 +290,19 @@ SHA3_get_name(SHA3object *self, void *closure) static PyObject * SHA3_get_digest_size(SHA3object *self, void *closure) { - return PyLong_FromLong(self->hash_state.mdlen); + // Preserving previous behavior: variable-length algorithms return 0 + if (Hacl_Streaming_Keccak_is_shake(self->hash_state)) + return PyLong_FromLong(0); + else + return PyLong_FromLong(Hacl_Streaming_Keccak_hash_len(self->hash_state)); } static PyObject * SHA3_get_capacity_bits(SHA3object *self, void *closure) { - int capacity = 1600 - self->hash_state.rsiz * 8; + uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state) * 8; + int capacity = 1600 - rate; return PyLong_FromLong(capacity); } @@ -378,7 +310,7 @@ SHA3_get_capacity_bits(SHA3object *self, void *closure) static PyObject * SHA3_get_rate_bits(SHA3object *self, void *closure) { - unsigned int rate = self->hash_state.rsiz * 8; + uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state) * 8; return PyLong_FromLong(rate); } @@ -455,28 +387,26 @@ static PyObject * _SHAKE_digest(SHA3object *self, unsigned long digestlen, int hex) { unsigned char *digest = NULL; - SHA3_state temp; PyObject *result = NULL; if (digestlen >= (1 << 29)) { PyErr_SetString(PyExc_ValueError, "length is too large"); return NULL; } - /* ExtractLane needs at least SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE and - * SHA3_LANESIZE extra space. - */ - digest = (unsigned char*)PyMem_Malloc(digestlen + SHA3_LANESIZE); + digest = (unsigned char*)PyMem_Malloc(digestlen); if (digest == NULL) { return PyErr_NoMemory(); } - /* Get the raw (binary) digest value */ - ENTER_HASHLIB(self); - SHA3_copystate(temp, self->hash_state); - LEAVE_HASHLIB(self); - SHA3_squeeze(&temp, digest, digestlen); + /* Get the raw (binary) digest value. The HACL functions errors out if: + * - the algorith is not shake -- not the case here + * - the output length is zero -- we follow the existing behavior and return + * an empty digest, without raising an error */ + if (digestlen > 0) { + Hacl_Streaming_Keccak_squeeze(self->hash_state, digest, digestlen); + } if (hex) { - result = _Py_strhex((const char *)digest, digestlen); + result = _Py_strhex((const char *)digest, digestlen); } else { result = PyBytes_FromStringAndSize((const char *)digest, digestlen); @@ -628,11 +558,8 @@ _sha3_exec(PyObject *m) init_sha3type(shake_256_type, SHAKE256_spec); #undef init_sha3type - if (PyModule_AddIntConstant(m, "keccakopt", KeccakOpt) < 0) { - return -1; - } if (PyModule_AddStringConstant(m, "implementation", - "tiny_sha3") < 0) { + "HACL") < 0) { return -1; } @@ -641,6 +568,7 @@ _sha3_exec(PyObject *m) static PyModuleDef_Slot _sha3_slots[] = { {Py_mod_exec, _sha3_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index fdd1450050fa1b..2350236ad46b25 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1695,6 +1695,7 @@ _signal_module_free(void *module) static PyModuleDef_Slot signal_slots[] = { {Py_mod_exec, signal_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index e5478382e11f89..60219593be61e2 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -8870,6 +8870,7 @@ socket_exec(PyObject *m) static struct PyModuleDef_Slot socket_slots[] = { {Py_mod_exec, socket_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/spwdmodule.c b/Modules/spwdmodule.c index 42123c93b59365..13f1115feefa86 100644 --- a/Modules/spwdmodule.c +++ b/Modules/spwdmodule.c @@ -224,6 +224,7 @@ spwdmodule_exec(PyObject *module) static PyModuleDef_Slot spwdmodule_slots[] = { {Py_mod_exec, spwdmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index 91538b4fb15cbd..1cc319cc3410d8 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -100,6 +100,7 @@ symtable_init_constants(PyObject *m) static PyModuleDef_Slot symtable_slots[] = { {Py_mod_exec, symtable_init_constants}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index f45aa5227f1cbf..6db8de9c491dd9 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -406,6 +406,7 @@ syslog_exec(PyObject *module) static PyModuleDef_Slot syslog_slots[] = { {Py_mod_exec, syslog_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/termios.c b/Modules/termios.c index fcc8f042679870..6dc8200572bc0c 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -85,7 +85,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) int r; Py_BEGIN_ALLOW_THREADS - r = tcgetattr(fd, &mode); + r = tcgetattr(fd, &mode); Py_END_ALLOW_THREADS if (r == -1) { return PyErr_SetFromErrno(state->TermiosError); @@ -372,7 +372,7 @@ termios_tcgetwinsize_impl(PyObject *module, int fd) #if defined(TIOCGWINSZ) termiosmodulestate *state = PyModule_GetState(module); struct winsize w; - int r; + int r; Py_BEGIN_ALLOW_THREADS r = ioctl(fd, TIOCGWINSZ, &w); @@ -1253,6 +1253,7 @@ termios_exec(PyObject *mod) static PyModuleDef_Slot termios_slots[] = { {Py_mod_exec, termios_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/timemodule.c b/Modules/timemodule.c index c50e689bb6986c..3607855dbd8f27 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -2107,6 +2107,7 @@ time_module_free(void *module) static struct PyModuleDef_Slot time_slots[] = { {Py_mod_exec, time_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index c108f14871f946..41dcd5f8f883f2 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -1516,6 +1516,7 @@ unicodedata_exec(PyObject *module) static PyModuleDef_Slot unicodedata_slots[] = { {Py_mod_exec, unicodedata_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 5f5297ba6337af..3935c00fc26530 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -390,6 +390,7 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index 361c7e76d77f50..1ff3ef1cb6f296 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -293,6 +293,7 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/xxmodule.c b/Modules/xxmodule.c index a676fdb4ec773a..1e4e0ea3743ce3 100644 --- a/Modules/xxmodule.c +++ b/Modules/xxmodule.c @@ -383,6 +383,7 @@ xx_exec(PyObject *m) static struct PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/xxsubtype.c b/Modules/xxsubtype.c index 8512baf7cd0a2d..744ba7bf5d28b6 100644 --- a/Modules/xxsubtype.c +++ b/Modules/xxsubtype.c @@ -286,6 +286,7 @@ xxsubtype_exec(PyObject* m) static struct PyModuleDef_Slot xxsubtype_slots[] = { {Py_mod_exec, xxsubtype_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index e2f7dbaca87a9f..b67844a67c315c 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -2109,6 +2109,7 @@ zlib_exec(PyObject *mod) static PyModuleDef_Slot zlib_slots[] = { {Py_mod_exec, zlib_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 49d4dd524696a5..c36db59baaa10d 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -61,6 +61,7 @@ static void bytearray_releasebuffer(PyByteArrayObject *obj, Py_buffer *view) { obj->ob_exports--; + assert(obj->ob_exports >= 0); } static int diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index e7e85cc19cda75..abbf3eeb16c35c 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -423,9 +423,6 @@ formatfloat(PyObject *v, int flags, int prec, int type, if (flags & F_ALT) { dtoa_flags |= Py_DTSF_ALT; } - if (flags & F_NO_NEG_0) { - dtoa_flags |= Py_DTSF_NO_NEG_0; - } p = PyOS_double_to_string(x, type, prec, dtoa_flags, NULL); if (p == NULL) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index ef0070199ab2c0..d0eca447c012ea 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -38,7 +38,7 @@ PyFrame_GetLineNumber(PyFrameObject *f) return f->f_lineno; } else { - return _PyInterpreterFrame_GetLine(f->f_frame); + return PyUnstable_InterpreterFrame_GetLine(f->f_frame); } } diff --git a/Objects/genobject.c b/Objects/genobject.c index 6316fa9865fe65..937d497753e970 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -12,6 +12,7 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "structmember.h" // PyMemberDef #include "opcode.h" // SEND +#include "frameobject.h" // _PyInterpreterFrame_GetLine #include "pystats.h" static PyObject *gen_close(PyGenObject *, PyObject *); @@ -1322,7 +1323,7 @@ compute_cr_origin(int origin_depth, _PyInterpreterFrame *current_frame) frame = current_frame; for (int i = 0; i < frame_count; ++i) { PyCodeObject *code = frame->f_code; - int line = _PyInterpreterFrame_GetLine(frame); + int line = PyUnstable_InterpreterFrame_GetLine(frame); PyObject *frameinfo = Py_BuildValue("OiO", code->co_filename, line, code->co_name); if (!frameinfo) { diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index f008a8cc3e0474..b0168044d9f85a 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -193,6 +193,20 @@ PyTypeObject _PyManagedBuffer_Type = { return -1; \ } +#define CHECK_RESTRICTED(mv) \ + if (((PyMemoryViewObject *)(mv))->flags & _Py_MEMORYVIEW_RESTRICTED) { \ + PyErr_SetString(PyExc_ValueError, \ + "cannot create new view on restricted memoryview"); \ + return NULL; \ + } + +#define CHECK_RESTRICTED_INT(mv) \ + if (((PyMemoryViewObject *)(mv))->flags & _Py_MEMORYVIEW_RESTRICTED) { \ + PyErr_SetString(PyExc_ValueError, \ + "cannot create new view on restricted memoryview"); \ + return -1; \ + } + /* See gh-92888. These macros signal that we need to check the memoryview again due to possible read after frees. */ #define CHECK_RELEASED_AGAIN(mv) CHECK_RELEASED(mv) @@ -781,7 +795,7 @@ PyMemoryView_FromBuffer(const Py_buffer *info) using the given flags. If the object is a memoryview, the new memoryview must be registered with the same managed buffer. Otherwise, a new managed buffer is created. */ -PyObject * +static PyObject * PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) { _PyManagedBufferObject *mbuf; @@ -789,6 +803,7 @@ PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) if (PyMemoryView_Check(v)) { PyMemoryViewObject *mv = (PyMemoryViewObject *)v; CHECK_RELEASED(mv); + CHECK_RESTRICTED(mv); return mbuf_add_view(mv->mbuf, &mv->view); } else if (PyObject_CheckBuffer(v)) { @@ -806,6 +821,30 @@ PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) Py_TYPE(v)->tp_name); return NULL; } + +/* Create a memoryview from an object that implements the buffer protocol, + using the given flags. + If the object is a memoryview, the new memoryview must be registered + with the same managed buffer. Otherwise, a new managed buffer is created. */ +PyObject * +_PyMemoryView_FromBufferProc(PyObject *v, int flags, getbufferproc bufferproc) +{ + _PyManagedBufferObject *mbuf = mbuf_alloc(); + if (mbuf == NULL) + return NULL; + + int res = bufferproc(v, &mbuf->master, flags); + if (res < 0) { + mbuf->master.obj = NULL; + Py_DECREF(mbuf); + return NULL; + } + + PyObject *ret = mbuf_add_view(mbuf, NULL); + Py_DECREF(mbuf); + return ret; +} + /* Create a memoryview from an object that implements the buffer protocol. If the object is a memoryview, the new memoryview must be registered with the same managed buffer. Otherwise, a new managed buffer is created. */ @@ -1397,6 +1436,7 @@ memoryview_cast_impl(PyMemoryViewObject *self, PyObject *format, Py_ssize_t ndim = 1; CHECK_RELEASED(self); + CHECK_RESTRICTED(self); if (!MV_C_CONTIGUOUS(self->flags)) { PyErr_SetString(PyExc_TypeError, @@ -1452,6 +1492,7 @@ memoryview_toreadonly_impl(PyMemoryViewObject *self) /*[clinic end generated code: output=2c7e056f04c99e62 input=dc06d20f19ba236f]*/ { CHECK_RELEASED(self); + CHECK_RESTRICTED(self); /* Even if self is already readonly, we still need to create a new * object for .release() to work correctly. */ @@ -1474,6 +1515,7 @@ memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) int baseflags = self->flags; CHECK_RELEASED_INT(self); + CHECK_RESTRICTED_INT(self); /* start with complete information */ *view = *base; @@ -2535,6 +2577,7 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key) return memory_item(self, index); } else if (PySlice_Check(key)) { + CHECK_RESTRICTED(self); PyMemoryViewObject *sliced; sliced = (PyMemoryViewObject *)mbuf_add_view(self->mbuf, view); diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index ffcd90ed122e77..985be58d02c784 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -245,9 +245,12 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio PyObject *(*create)(PyObject *, PyModuleDef*) = NULL; PyObject *nameobj; PyObject *m = NULL; + int has_multiple_interpreters_slot = 0; + void *multiple_interpreters = (void *)0; int has_execution_slots = 0; const char *name; int ret; + PyInterpreterState *interp = _PyInterpreterState_GET(); PyModuleDef_Init(def); @@ -287,6 +290,17 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio case Py_mod_exec: has_execution_slots = 1; break; + case Py_mod_multiple_interpreters: + if (has_multiple_interpreters_slot) { + PyErr_Format( + PyExc_SystemError, + "module %s has more than one 'multiple interpreters' slots", + name); + goto error; + } + multiple_interpreters = cur_slot->value; + has_multiple_interpreters_slot = 1; + break; default: assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT); PyErr_Format( @@ -297,6 +311,26 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio } } + /* By default, multi-phase init modules are expected + to work under multiple interpreters. */ + if (!has_multiple_interpreters_slot) { + multiple_interpreters = Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED; + } + if (multiple_interpreters == Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED) { + if (!_Py_IsMainInterpreter(interp) + && _PyImport_CheckSubinterpIncompatibleExtensionAllowed(name) < 0) + { + goto error; + } + } + else if (multiple_interpreters != Py_MOD_PER_INTERPRETER_GIL_SUPPORTED + && interp->ceval.own_gil + && !_Py_IsMainInterpreter(interp) + && _PyImport_CheckSubinterpIncompatibleExtensionAllowed(name) < 0) + { + goto error; + } + if (create) { m = create(spec, def); if (m == NULL) { @@ -421,6 +455,9 @@ PyModule_ExecDef(PyObject *module, PyModuleDef *def) return -1; } break; + case Py_mod_multiple_interpreters: + /* handled in PyModule_FromDefAndSpec2 */ + break; default: PyErr_Format( PyExc_SystemError, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 456b10ee01d6bc..98fac276a873e1 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6,7 +6,7 @@ #include "pycore_symtable.h" // _Py_Mangle() #include "pycore_dict.h" // _PyDict_KeysSize() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_memoryobject.h" // PyMemoryView_FromObjectAndFlags() +#include "pycore_memoryobject.h" // _PyMemoryView_FromBufferProc() #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" // _PyType_HasFeature() #include "pycore_long.h" // _PyLong_IsNegative() @@ -56,6 +56,11 @@ typedef struct PySlot_Offset { short slot_offset; } PySlot_Offset; +static void +slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer); + +static void +releasebuffer_call_python(PyObject *self, Py_buffer *buffer); static PyObject * slot_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds); @@ -8078,7 +8083,8 @@ wrap_buffer(PyObject *self, PyObject *args, void *wrapped) return NULL; } - return PyMemoryView_FromObjectAndFlags(self, Py_SAFE_DOWNCAST(flags, Py_ssize_t, int)); + return _PyMemoryView_FromBufferProc(self, Py_SAFE_DOWNCAST(flags, Py_ssize_t, int), + (getbufferproc)wrapped); } static PyObject * @@ -8094,6 +8100,10 @@ wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) return NULL; } PyMemoryViewObject *mview = (PyMemoryViewObject *)arg; + if (mview->view.obj == NULL) { + // Already released, ignore + Py_RETURN_NONE; + } if (mview->view.obj != self) { PyErr_SetString(PyExc_ValueError, "memoryview's buffer is not this object"); @@ -8978,12 +8988,26 @@ bufferwrapper_releasebuf(PyObject *self, Py_buffer *view) { PyBufferWrapper *bw = (PyBufferWrapper *)self; - assert(PyMemoryView_Check(bw->mv)); - Py_TYPE(bw->mv)->tp_as_buffer->bf_releasebuffer(bw->mv, view); - if (Py_TYPE(bw->obj)->tp_as_buffer != NULL - && Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer != NULL) { - Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer(bw->obj, view); + if (bw->mv == NULL || bw->obj == NULL) { + // Already released + return; + } + + PyObject *mv = bw->mv; + PyObject *obj = bw->obj; + + assert(PyMemoryView_Check(mv)); + Py_TYPE(mv)->tp_as_buffer->bf_releasebuffer(mv, view); + // We only need to call bf_releasebuffer if it's a Python function. If it's a C + // bf_releasebuf, it will be called when the memoryview is released. + if (((PyMemoryViewObject *)mv)->view.obj != obj + && Py_TYPE(obj)->tp_as_buffer != NULL + && Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer == slot_bf_releasebuffer) { + releasebuffer_call_python(obj, view); } + + Py_CLEAR(bw->mv); + Py_CLEAR(bw->obj); } static PyBufferProcs bufferwrapper_as_buffer = { @@ -9047,31 +9071,112 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) return -1; } +static int +releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) +{ + PyTypeObject *self_type = Py_TYPE(self); + PyObject *mro = lookup_tp_mro(self_type); + if (mro == NULL) { + return -1; + } + + assert(PyTuple_Check(mro)); + Py_ssize_t n = PyTuple_GET_SIZE(mro); + Py_ssize_t i; + + /* No need to check the last one: it's gonna be skipped anyway. */ + for (i = 0; i < n -1; i++) { + if ((PyObject *)(self_type) == PyTuple_GET_ITEM(mro, i)) + break; + } + i++; /* skip self_type */ + if (i >= n) + return -1; + + releasebufferproc base_releasebuffer = NULL; + for (; i < n; i++) { + PyObject *obj = PyTuple_GET_ITEM(mro, i); + if (!PyType_Check(obj)) { + continue; + } + PyTypeObject *base_type = (PyTypeObject *)obj; + if (base_type->tp_as_buffer != NULL + && base_type->tp_as_buffer->bf_releasebuffer != NULL + && base_type->tp_as_buffer->bf_releasebuffer != slot_bf_releasebuffer) { + base_releasebuffer = base_type->tp_as_buffer->bf_releasebuffer; + break; + } + } + + if (base_releasebuffer != NULL) { + base_releasebuffer(self, buffer); + } + return 0; +} + static void -slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) +releasebuffer_call_python(PyObject *self, Py_buffer *buffer) { PyObject *mv; - if (Py_TYPE(buffer->obj) == &_PyBufferWrapper_Type) { + bool is_buffer_wrapper = Py_TYPE(buffer->obj) == &_PyBufferWrapper_Type; + if (is_buffer_wrapper) { // Make sure we pass the same memoryview to // __release_buffer__() that __buffer__() returned. - mv = Py_NewRef(((PyBufferWrapper *)buffer->obj)->mv); + PyBufferWrapper *bw = (PyBufferWrapper *)buffer->obj; + if (bw->mv == NULL) { + return; + } + mv = Py_NewRef(bw->mv); } else { + // This means we are not dealing with a memoryview returned + // from a Python __buffer__ function. mv = PyMemoryView_FromBuffer(buffer); if (mv == NULL) { PyErr_WriteUnraisable(self); return; } + // Set the memoryview to restricted mode, which forbids + // users from saving any reference to the underlying buffer + // (e.g., by doing .cast()). This is necessary to ensure + // no Python code retains a reference to the to-be-released + // buffer. + ((PyMemoryViewObject *)mv)->flags |= _Py_MEMORYVIEW_RESTRICTED; } PyObject *stack[2] = {self, mv}; PyObject *ret = vectorcall_method(&_Py_ID(__release_buffer__), stack, 2); - Py_DECREF(mv); if (ret == NULL) { PyErr_WriteUnraisable(self); } else { Py_DECREF(ret); } + if (!is_buffer_wrapper) { + PyObject_CallMethodNoArgs(mv, &_Py_ID(release)); + } + Py_DECREF(mv); +} + +/* + * bf_releasebuffer is very delicate, because we need to ensure that + * C bf_releasebuffer slots are called correctly (or we'll leak memory), + * but we cannot trust any __release_buffer__ implemented in Python to + * do so correctly. Therefore, if a base class has a C bf_releasebuffer + * slot, we call it directly here. That is safe because this function + * only gets called from C callers of the bf_releasebuffer slot. Python + * code that calls __release_buffer__ directly instead goes through + * wrap_releasebuffer(), which doesn't call the bf_releasebuffer slot + * directly but instead simply releases the associated memoryview. + */ +static void +slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) +{ + releasebuffer_call_python(self, buffer); + if (releasebuffer_maybe_call_super(self, buffer) < 0) { + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(self); + } + } } static PyObject * diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 6ae68cc20f7dee..7726f2fb17afde 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13452,8 +13452,6 @@ formatfloat(PyObject *v, struct unicode_format_arg_t *arg, if (arg->flags & F_ALT) dtoa_flags |= Py_DTSF_ALT; - if (arg->flags & F_NO_NEG_0) - dtoa_flags |= Py_DTSF_NO_NEG_0; p = PyOS_double_to_string(x, arg->ch, prec, dtoa_flags, NULL); if (p == NULL) return -1; @@ -15192,12 +15190,18 @@ static PyMethodDef _string_methods[] = { {NULL, NULL} }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL} +}; + static struct PyModuleDef _string_module = { PyModuleDef_HEAD_INIT, .m_name = "_string", .m_doc = PyDoc_STR("string helper module"), .m_size = 0, .m_methods = _string_methods, + .m_slots = module_slots, }; PyMODINIT_FUNC diff --git a/PC/_testconsole.c b/PC/_testconsole.c index f14a2d45b1be26..3221b985d01ba0 100644 --- a/PC/_testconsole.c +++ b/PC/_testconsole.c @@ -31,6 +31,7 @@ static int execfunc(PyObject *m) PyModuleDef_Slot testconsole_slots[] = { {Py_mod_exec, execfunc}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 090254befc934d..53ef26b732f615 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -661,6 +661,7 @@ exec_module(PyObject* m) static PyModuleDef_Slot msvcrt_slots[] = { {Py_mod_exec, exec_module}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/PC/winreg.c b/PC/winreg.c index 4884125c3609ad..e2d5322f458c2a 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -2184,6 +2184,7 @@ exec_module(PyObject *m) static PyModuleDef_Slot winreg_slots[] = { {Py_mod_exec, exec_module}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/PC/winsound.c b/PC/winsound.c index 17ce2ef423b1f9..68a917810f884d 100644 --- a/PC/winsound.c +++ b/PC/winsound.c @@ -235,6 +235,7 @@ exec_module(PyObject *module) static PyModuleDef_Slot sound_slots[] = { {Py_mod_exec, exec_module}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 8aafcb786a6064..28b1517c6f6b3a 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -375,13 +375,16 @@ + + + + - @@ -404,17 +407,15 @@ - - - + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 07476f30833372..75e6fbb13f98ba 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -776,6 +776,15 @@ Modules + + Modules + + + Modules + + + Modules + Modules @@ -794,9 +803,6 @@ Modules - - Modules - Modules @@ -875,16 +881,13 @@ Modules - - Modules - Modules - + Modules - + Modules diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index b44e303ac2594b..5d5a05a70ca7ec 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1206,6 +1206,7 @@ def visitModule(self, mod): self.emit(""" static PyModuleDef_Slot astmodule_slots[] = { {Py_mod_exec, astmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 52d0d9a534cb6a..91ffabac56c7b3 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -48,7 +48,7 @@ static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) { } static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) { assert(tok->tok_mode_stack_index >= 0); - assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); + assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]); } #else diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 6c878474afb192..81ab71c0fc3b29 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -12193,6 +12193,7 @@ astmodule_exec(PyObject *m) static PyModuleDef_Slot astmodule_slots[] = { {Py_mod_exec, astmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 416dc5971bca3d..3394a5108cb535 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -151,6 +151,7 @@ static PyMethodDef tokenize_methods[] = { static PyModuleDef_Slot tokenizemodule_slots[] = { {Py_mod_exec, tokenizemodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Python/_warnings.c b/Python/_warnings.c index d510381c365b66..5644db9a3770cb 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1449,6 +1449,7 @@ warnings_module_exec(PyObject *module) static PyModuleDef_Slot warnings_slots[] = { {Py_mod_exec, warnings_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 8270fa8e372d93..3883ec9e21c765 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -317,7 +317,6 @@ simple_format_arg_parse(PyObject *fmt, Py_ssize_t *ppos, case ' ': *flags |= F_BLANK; continue; case '#': *flags |= F_ALT; continue; case '0': *flags |= F_ZERO; continue; - case 'z': *flags |= F_NO_NEG_0; continue; } break; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 8840bbabe4b584..ddddc03ca316e0 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3014,9 +3014,16 @@ static PyMethodDef builtin_methods[] = { }; PyDoc_STRVAR(builtin_doc, -"Built-in functions, exceptions, and other objects.\n\ +"Built-in functions, types, exceptions, and other objects.\n\ \n\ -Noteworthy: None is the `nil' object; Ellipsis represents `...' in slices."); +This module provides direct access to all 'built-in'\n\ +identifiers of Python; for example, builtins.len is\n\ +the full name for the built-in function len().\n\ +\n\ +This module is not normally accessed explicitly by most\n\ +applications, but can be useful in modules that provide\n\ +objects with the same name as a built-in value, but in\n\ +which the built-in of that name is also needed."); static struct PyModuleDef builtinsmodule = { PyModuleDef_HEAD_INIT, diff --git a/Python/ceval.c b/Python/ceval.c index 958689debc87f8..56a3b123f46331 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -27,6 +27,7 @@ #include "pycore_dict.h" #include "dictobject.h" #include "pycore_frame.h" +#include "frameobject.h" // _PyInterpreterFrame_GetLine #include "opcode.h" #include "pydtrace.h" #include "setobject.h" @@ -785,7 +786,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", frame->f_code->co_filename, - _PyInterpreterFrame_GetLine(frame), + PyUnstable_InterpreterFrame_GetLine(frame), opcode); goto error; diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 4d501c89661743..9958856bae8019 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -229,6 +229,9 @@ static void _gil_initialize(struct _gil_runtime_state *gil) static int gil_created(struct _gil_runtime_state *gil) { + if (gil == NULL) { + return 0; + } return (_Py_atomic_load_explicit(&gil->locked, _Py_memory_order_acquire) >= 0); } @@ -273,10 +276,9 @@ static void recreate_gil(struct _gil_runtime_state *gil) #endif static void -drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2, - PyThreadState *tstate) +drop_gil(struct _ceval_state *ceval, PyThreadState *tstate) { - struct _gil_runtime_state *gil = &ceval->gil; + struct _gil_runtime_state *gil = ceval->gil; if (!_Py_atomic_load_relaxed(&gil->locked)) { Py_FatalError("drop_gil: GIL is not locked"); } @@ -296,7 +298,7 @@ drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2, MUTEX_UNLOCK(gil->mutex); #ifdef FORCE_SWITCHING - if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request) && tstate != NULL) { + if (_Py_atomic_load_relaxed(&ceval->gil_drop_request) && tstate != NULL) { MUTEX_LOCK(gil->switch_mutex); /* Not switched yet => wait */ if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) == tstate) @@ -358,9 +360,8 @@ take_gil(PyThreadState *tstate) assert(is_tstate_valid(tstate)); PyInterpreterState *interp = tstate->interp; - struct _ceval_runtime_state *ceval = &interp->runtime->ceval; - struct _ceval_state *ceval2 = &interp->ceval; - struct _gil_runtime_state *gil = &ceval->gil; + struct _ceval_state *ceval = &interp->ceval; + struct _gil_runtime_state *gil = ceval->gil; /* Check that _PyEval_InitThreads() was called to create the lock */ assert(gil_created(gil)); @@ -434,12 +435,12 @@ take_gil(PyThreadState *tstate) in take_gil() while the main thread called wait_for_thread_shutdown() from Py_Finalize(). */ MUTEX_UNLOCK(gil->mutex); - drop_gil(ceval, ceval2, tstate); + drop_gil(ceval, tstate); PyThread_exit_thread(); } assert(is_tstate_valid(tstate)); - if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request)) { + if (_Py_atomic_load_relaxed(&ceval->gil_drop_request)) { RESET_GIL_DROP_REQUEST(interp); } else { @@ -448,7 +449,7 @@ take_gil(PyThreadState *tstate) handle signals. Note: RESET_GIL_DROP_REQUEST() calls COMPUTE_EVAL_BREAKER(). */ - COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); + COMPUTE_EVAL_BREAKER(interp, &_PyRuntime.ceval, ceval); } /* Don't access tstate if the thread must exit */ @@ -463,63 +464,133 @@ take_gil(PyThreadState *tstate) void _PyEval_SetSwitchInterval(unsigned long microseconds) { - struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil; + /* XXX per-interpreter GIL */ + PyInterpreterState *interp = _PyInterpreterState_Main(); + struct _gil_runtime_state *gil = interp->ceval.gil; + assert(gil != NULL); gil->interval = microseconds; } unsigned long _PyEval_GetSwitchInterval(void) { - struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil; + /* XXX per-interpreter GIL */ + PyInterpreterState *interp = _PyInterpreterState_Main(); + struct _gil_runtime_state *gil = interp->ceval.gil; + assert(gil != NULL); return gil->interval; } int -_PyEval_ThreadsInitialized(_PyRuntimeState *runtime) +_PyEval_ThreadsInitialized(void) { - return gil_created(&runtime->ceval.gil); + /* XXX per-interpreter GIL */ + PyInterpreterState *interp = _PyInterpreterState_Main(); + if (interp == NULL) { + return 0; + } + struct _gil_runtime_state *gil = interp->ceval.gil; + return gil_created(gil); } int PyEval_ThreadsInitialized(void) { - _PyRuntimeState *runtime = &_PyRuntime; - return _PyEval_ThreadsInitialized(runtime); + return _PyEval_ThreadsInitialized(); } -PyStatus -_PyEval_InitGIL(PyThreadState *tstate) +static inline int +current_thread_holds_gil(struct _gil_runtime_state *gil, PyThreadState *tstate) { - if (!_Py_IsMainInterpreter(tstate->interp)) { - /* Currently, the GIL is shared by all interpreters, - and only the main interpreter is responsible to create - and destroy it. */ - return _PyStatus_OK(); + if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) != tstate) { + return 0; } + return _Py_atomic_load_relaxed(&gil->locked); +} - struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil; - assert(!gil_created(gil)); +static void +init_shared_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) +{ + assert(gil_created(gil)); + interp->ceval.gil = gil; + interp->ceval.own_gil = 0; +} - PyThread_init_thread(); +static void +init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) +{ + assert(!gil_created(gil)); create_gil(gil); + assert(gil_created(gil)); + interp->ceval.gil = gil; + interp->ceval.own_gil = 1; +} - take_gil(tstate); +PyStatus +_PyEval_InitGIL(PyThreadState *tstate, int own_gil) +{ + assert(tstate->interp->ceval.gil == NULL); + int locked; + if (!own_gil) { + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + assert(tstate->interp != main_interp); + struct _gil_runtime_state *gil = main_interp->ceval.gil; + init_shared_gil(tstate->interp, gil); + locked = current_thread_holds_gil(gil, tstate); + } + /* XXX per-interpreter GIL */ + else if (!_Py_IsMainInterpreter(tstate->interp)) { + /* Currently, the GIL is shared by all interpreters, + and only the main interpreter is responsible to create + and destroy it. */ + struct _gil_runtime_state *main_gil = _PyInterpreterState_Main()->ceval.gil; + init_shared_gil(tstate->interp, main_gil); + // XXX For now we lie. + tstate->interp->ceval.own_gil = 1; + locked = current_thread_holds_gil(main_gil, tstate); + } + else { + PyThread_init_thread(); + // XXX per-interpreter GIL: switch to interp->_gil. + init_own_gil(tstate->interp, &tstate->interp->runtime->ceval.gil); + locked = 0; + } + if (!locked) { + take_gil(tstate); + } - assert(gil_created(gil)); return _PyStatus_OK(); } void _PyEval_FiniGIL(PyInterpreterState *interp) { + if (interp->ceval.gil == NULL) { + /* It was already finalized (or hasn't been initialized yet). */ + assert(!interp->ceval.own_gil); + return; + } + else if (!interp->ceval.own_gil) { +#ifdef Py_DEBUG + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + assert(interp != main_interp); + assert(interp->ceval.gil == main_interp->ceval.gil); +#endif + interp->ceval.gil = NULL; + return; + } + + /* XXX per-interpreter GIL */ + struct _gil_runtime_state *gil = &interp->runtime->ceval.gil; if (!_Py_IsMainInterpreter(interp)) { /* Currently, the GIL is shared by all interpreters, and only the main interpreter is responsible to create and destroy it. */ + assert(interp->ceval.gil == gil); + interp->ceval.gil = NULL; return; } - struct _gil_runtime_state *gil = &interp->runtime->ceval.gil; if (!gil_created(gil)) { /* First Py_InitializeFromConfig() call: the GIL doesn't exist yet: do nothing. */ @@ -528,6 +599,7 @@ _PyEval_FiniGIL(PyInterpreterState *interp) destroy_gil(gil); assert(!gil_created(gil)); + interp->ceval.gil = NULL; } void @@ -555,22 +627,27 @@ PyEval_AcquireLock(void) void PyEval_ReleaseLock(void) { - _PyRuntimeState *runtime = &_PyRuntime; PyThreadState *tstate = _PyThreadState_GET(); /* This function must succeed when the current thread state is NULL. We therefore avoid PyThreadState_Get() which dumps a fatal error in debug mode. */ - struct _ceval_runtime_state *ceval = &runtime->ceval; - struct _ceval_state *ceval2 = &tstate->interp->ceval; - drop_gil(ceval, ceval2, tstate); + struct _ceval_state *ceval = &tstate->interp->ceval; + drop_gil(ceval, tstate); +} + +void +_PyEval_AcquireLock(PyThreadState *tstate) +{ + _Py_EnsureTstateNotNULL(tstate); + take_gil(tstate); } void _PyEval_ReleaseLock(PyThreadState *tstate) { - struct _ceval_runtime_state *ceval = &tstate->interp->runtime->ceval; - struct _ceval_state *ceval2 = &tstate->interp->ceval; - drop_gil(ceval, ceval2, tstate); + _Py_EnsureTstateNotNULL(tstate); + struct _ceval_state *ceval = &tstate->interp->ceval; + drop_gil(ceval, tstate); } void @@ -580,7 +657,7 @@ PyEval_AcquireThread(PyThreadState *tstate) take_gil(tstate); - if (_PyThreadState_Swap(tstate->interp->runtime, tstate) != NULL) { + if (_PyThreadState_SwapNoGIL(tstate) != NULL) { Py_FatalError("non-NULL old thread state"); } } @@ -590,14 +667,12 @@ PyEval_ReleaseThread(PyThreadState *tstate) { assert(is_tstate_valid(tstate)); - _PyRuntimeState *runtime = tstate->interp->runtime; - PyThreadState *new_tstate = _PyThreadState_Swap(runtime, NULL); + PyThreadState *new_tstate = _PyThreadState_SwapNoGIL(NULL); if (new_tstate != tstate) { Py_FatalError("wrong thread state"); } - struct _ceval_runtime_state *ceval = &runtime->ceval; - struct _ceval_state *ceval2 = &tstate->interp->ceval; - drop_gil(ceval, ceval2, tstate); + struct _ceval_state *ceval = &tstate->interp->ceval; + drop_gil(ceval, tstate); } #ifdef HAVE_FORK @@ -607,9 +682,9 @@ PyEval_ReleaseThread(PyThreadState *tstate) PyStatus _PyEval_ReInitThreads(PyThreadState *tstate) { - _PyRuntimeState *runtime = tstate->interp->runtime; + assert(tstate->interp == _PyInterpreterState_Main()); - struct _gil_runtime_state *gil = &runtime->ceval.gil; + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; if (!gil_created(gil)) { return _PyStatus_OK(); } @@ -640,14 +715,12 @@ _PyEval_SignalAsyncExc(PyInterpreterState *interp) PyThreadState * PyEval_SaveThread(void) { - _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = _PyThreadState_Swap(runtime, NULL); + PyThreadState *tstate = _PyThreadState_SwapNoGIL(NULL); _Py_EnsureTstateNotNULL(tstate); - struct _ceval_runtime_state *ceval = &runtime->ceval; - struct _ceval_state *ceval2 = &tstate->interp->ceval; - assert(gil_created(&ceval->gil)); - drop_gil(ceval, ceval2, tstate); + struct _ceval_state *ceval = &tstate->interp->ceval; + assert(gil_created(ceval->gil)); + drop_gil(ceval, tstate); return tstate; } @@ -658,7 +731,7 @@ PyEval_RestoreThread(PyThreadState *tstate) take_gil(tstate); - _PyThreadState_Swap(tstate->interp->runtime, tstate); + _PyThreadState_SwapNoGIL(tstate); } @@ -906,6 +979,7 @@ Py_MakePendingCalls(void) void _PyEval_InitRuntimeState(struct _ceval_runtime_state *ceval) { + /* XXX per-interpreter GIL */ _gil_initialize(&ceval->gil); } @@ -961,16 +1035,16 @@ _Py_HandlePending(PyThreadState *tstate) /* GIL drop request */ if (_Py_atomic_load_relaxed_int32(&interp_ceval_state->gil_drop_request)) { /* Give another thread a chance */ - if (_PyThreadState_Swap(runtime, NULL) != tstate) { + if (_PyThreadState_SwapNoGIL(NULL) != tstate) { Py_FatalError("tstate mix-up"); } - drop_gil(ceval, interp_ceval_state, tstate); + drop_gil(interp_ceval_state, tstate); /* Other threads may run now */ take_gil(tstate); - if (_PyThreadState_Swap(runtime, tstate) != NULL) { + if (_PyThreadState_SwapNoGIL(tstate) != NULL) { Py_FatalError("orphan tstate"); } } diff --git a/Python/compile.c b/Python/compile.c index cbe5403aafbc87..f875e4e17e0a9f 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -7258,7 +7258,7 @@ cfg_to_instructions(cfg_builder *g) PyObject * _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, - int optimize) + int optimize, int compile_mode) { PyObject *res = NULL; @@ -7272,7 +7272,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return NULL; } - mod_ty mod = PyAST_obj2mod(ast, arena, 0 /* exec */); + mod_ty mod = PyAST_obj2mod(ast, arena, compile_mode); if (mod == NULL || !_PyAST_Validate(mod)) { _PyArena_Free(arena); return NULL; @@ -7287,6 +7287,10 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, if (compiler_codegen(c, mod) < 0) { goto finally; } + int addNone = mod->kind != Expression_kind; + if (add_return_at_end(c, addNone) < 0) { + return NULL; + } res = instr_sequence_to_instructions(INSTR_SEQUENCE(c)); diff --git a/Python/frame.c b/Python/frame.c index c2c0be30113912..d792b92fa57560 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -144,8 +144,24 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) Py_DECREF(frame->f_funcobj); } +/* Unstable API functions */ + +PyCodeObject * +PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame) +{ + PyCodeObject *code = frame->f_code; + Py_INCREF(code); + return code; +} + +int +PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame) +{ + return _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); +} + int -_PyInterpreterFrame_GetLine(_PyInterpreterFrame *frame) +PyUnstable_InterpreterFrame_GetLine(_PyInterpreterFrame *frame) { int addr = _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); return PyCode_Addr2Line(frame->f_code, addr); diff --git a/Python/import.c b/Python/import.c index 0bf107b28d3990..9e1857d5f3e4e6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3840,6 +3840,7 @@ imp_module_exec(PyObject *module) static PyModuleDef_Slot imp_slots[] = { {Py_mod_exec, imp_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Python/marshal.c b/Python/marshal.c index 2966139cec9ae9..208996b05fc484 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1870,6 +1870,7 @@ marshal_module_exec(PyObject *mod) static PyModuleDef_Slot marshalmodule_slots[] = { {Py_mod_exec, marshal_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 1957ab82c33951..3b183280e1f24c 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -253,7 +253,7 @@ perf_map_write_entry(void *state, const void *code_addr, NULL); return; } - fprintf(method_file, "%p %x py::%s:%s\n", code_addr, code_size, entry, + fprintf(method_file, "%" PRIxPTR " %x py::%s:%s\n", (uintptr_t) code_addr, code_size, entry, filename); fflush(method_file); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 97957d3f17e6db..61f87c5eba60ed 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -585,12 +585,13 @@ init_interp_settings(PyInterpreterState *interp, static PyStatus -init_interp_create_gil(PyThreadState *tstate) +init_interp_create_gil(PyThreadState *tstate, int own_gil) { PyStatus status; /* finalize_interp_delete() comment explains why _PyEval_FiniGIL() is only called here. */ + // XXX This is broken with a per-interpreter GIL. _PyEval_FiniGIL(tstate->interp); /* Auto-thread-state API */ @@ -600,7 +601,7 @@ init_interp_create_gil(PyThreadState *tstate) } /* Create the GIL and take it */ - status = _PyEval_InitGIL(tstate); + status = _PyEval_InitGIL(tstate, own_gil); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -632,7 +633,9 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return status; } - const PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; + PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; + // The main interpreter always has its own GIL. + config.own_gil = 1; status = init_interp_settings(interp, &config); if (_PyStatus_EXCEPTION(status)) { return status; @@ -643,9 +646,10 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return _PyStatus_ERR("can't make first thread"); } _PyThreadState_Bind(tstate); - (void) PyThreadState_Swap(tstate); + // XXX For now we do this before the GIL is created. + (void) _PyThreadState_SwapNoGIL(tstate); - status = init_interp_create_gil(tstate); + status = init_interp_create_gil(tstate, config.own_gil); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -2023,11 +2027,20 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) } _PyThreadState_Bind(tstate); - PyThreadState *save_tstate = PyThreadState_Swap(tstate); + // XXX For now we do this before the GIL is created. + PyThreadState *save_tstate = _PyThreadState_SwapNoGIL(tstate); + int has_gil = 0; + + /* From this point until the init_interp_create_gil() call, + we must not do anything that requires that the GIL be held + (or otherwise exist). That applies whether or not the new + interpreter has its own GIL (e.g. the main interpreter). */ /* Copy the current interpreter config into the new interpreter */ const PyConfig *src_config; if (save_tstate != NULL) { + // XXX Might new_interpreter() have been called without the GIL held? + _PyEval_ReleaseLock(save_tstate); src_config = _PyInterpreterState_GetConfig(save_tstate->interp); } else @@ -2037,20 +2050,23 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) src_config = _PyInterpreterState_GetConfig(main_interp); } + /* This does not require that the GIL be held. */ status = _PyConfig_Copy(&interp->config, src_config); if (_PyStatus_EXCEPTION(status)) { goto error; } + /* This does not require that the GIL be held. */ status = init_interp_settings(interp, config); if (_PyStatus_EXCEPTION(status)) { goto error; } - status = init_interp_create_gil(tstate); + status = init_interp_create_gil(tstate, config->own_gil); if (_PyStatus_EXCEPTION(status)) { goto error; } + has_gil = 1; status = pycore_interp_init(tstate); if (_PyStatus_EXCEPTION(status)) { @@ -2070,7 +2086,12 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) /* Oops, it didn't work. Undo it all. */ PyErr_PrintEx(0); - PyThreadState_Swap(save_tstate); + if (has_gil) { + PyThreadState_Swap(save_tstate); + } + else { + _PyThreadState_SwapNoGIL(save_tstate); + } PyThreadState_Clear(tstate); PyThreadState_Delete(tstate); PyInterpreterState_Delete(interp); diff --git a/Python/pystate.c b/Python/pystate.c index f09d37657f9c27..f14934361dab78 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1863,17 +1863,11 @@ PyThreadState_Get(void) } -PyThreadState * -_PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) +static void +_swap_thread_states(_PyRuntimeState *runtime, + PyThreadState *oldts, PyThreadState *newts) { -#if defined(Py_DEBUG) - /* This can be called from PyEval_RestoreThread(). Similar - to it, we need to ensure errno doesn't change. - */ - int err = errno; -#endif - PyThreadState *oldts = current_fast_get(runtime); - + // XXX Do this only if oldts != NULL? current_fast_clear(runtime); if (oldts != NULL) { @@ -1887,6 +1881,20 @@ _PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) current_fast_set(runtime, newts); tstate_activate(newts); } +} + +PyThreadState * +_PyThreadState_SwapNoGIL(PyThreadState *newts) +{ +#if defined(Py_DEBUG) + /* This can be called from PyEval_RestoreThread(). Similar + to it, we need to ensure errno doesn't change. + */ + int err = errno; +#endif + + PyThreadState *oldts = current_fast_get(&_PyRuntime); + _swap_thread_states(&_PyRuntime, oldts, newts); #if defined(Py_DEBUG) errno = err; @@ -1894,6 +1902,20 @@ _PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) return oldts; } +PyThreadState * +_PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) +{ + PyThreadState *oldts = current_fast_get(runtime); + if (oldts != NULL) { + _PyEval_ReleaseLock(oldts); + } + _swap_thread_states(runtime, oldts, newts); + if (newts != NULL) { + _PyEval_AcquireLock(newts); + } + return oldts; +} + PyThreadState * PyThreadState_Swap(PyThreadState *newts) { @@ -2186,7 +2208,7 @@ PyGILState_Ensure(void) /* Ensure that _PyEval_InitThreads() and _PyGILState_Init() have been called by Py_Initialize() */ - assert(_PyEval_ThreadsInitialized(runtime)); + assert(_PyEval_ThreadsInitialized()); assert(gilstate_tss_initialized(runtime)); assert(runtime->gilstate.autoInterpreterState != NULL); diff --git a/Python/traceback.c b/Python/traceback.c index 097f69c76abfe1..b2479542047308 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -1180,7 +1180,7 @@ dump_frame(int fd, _PyInterpreterFrame *frame) PUTS(fd, "???"); } - int lineno = _PyInterpreterFrame_GetLine(frame); + int lineno = PyUnstable_InterpreterFrame_GetLine(frame); PUTS(fd, ", line "); if (lineno >= 0) { _Py_DumpDecimal(fd, (size_t)lineno); diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 5924ab7860d8d5..9bd54db0f59c51 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -114,6 +114,7 @@ def clean_lines(text): Modules/md5module.c Modules/_hacl/include Modules/sha1module.c Modules/_hacl/include Modules/sha2module.c Modules/_hacl/include +Modules/sha3module.c Modules/_hacl/include Objects/stringlib/*.h Objects # possible system-installed headers, just in case @@ -271,13 +272,6 @@ def clean_lines(text): Modules/expat/xmlparse.c XML_POOR_ENTROPY 1 Modules/_dbmmodule.c HAVE_GDBM_DASH_NDBM_H 1 -# from Modules/_sha3/sha3module.c -Modules/_sha3/kcp/KeccakP-1600-inplace32BI.c PLATFORM_BYTE_ORDER 4321 # force big-endian -Modules/_sha3/kcp/*.c KeccakOpt 64 -Modules/_sha3/kcp/*.c KeccakP200_excluded 1 -Modules/_sha3/kcp/*.c KeccakP400_excluded 1 -Modules/_sha3/kcp/*.c KeccakP800_excluded 1 - # others Modules/_sre/sre_lib.h LOCAL(type) static inline type Modules/_sre/sre_lib.h SRE(F) sre_ucs2_##F diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 165bd74587d7c7..453f63ec3f1cfb 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -318,12 +318,9 @@ Python/instrumentation.c - _PyInstrumentation_MISSING - ## static types Modules/_io/bufferedio.c - PyBufferedIOBase_Type - -Modules/_io/bytesio.c - _PyBytesIOBuffer_Type - Modules/_io/iobase.c - PyIOBase_Type - Modules/_io/iobase.c - PyRawIOBase_Type - -Modules/_io/textio.c - PyIncrementalNewlineDecoder_Type - Modules/_io/textio.c - PyTextIOBase_Type - -Modules/_io/winconsoleio.c - PyWindowsConsoleIO_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorBase_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorDerived_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorNopGet_Type - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index fee493ff7f1666..7ba116dcb171cf 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -216,6 +216,7 @@ Modules/_io/_iomodule.c - static_types - Modules/_io/textio.c - encodefuncs - Modules/_io/winconsoleio.c - _PyWindowsConsoleIO_Type - Modules/_localemodule.c - langinfo_constants - +Modules/_lsprof.c - callback_table - Modules/_pickle.c - READ_WHOLE_LINE - Modules/_sqlite/module.c - error_codes - Modules/_sre/sre.c pattern_repr flag_names - diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index d3e120c0e7a8ee..a6f330d1502dad 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -43,7 +43,18 @@ NO_VARARG = "PY_SSIZE_T_MAX" CLINIC_PREFIX = "__clinic_" -CLINIC_PREFIXED_ARGS = {"args"} +CLINIC_PREFIXED_ARGS = { + "_keywords", + "_parser", + "args", + "argsbuf", + "fastargs", + "kwargs", + "kwnames", + "nargs", + "noptargs", + "return_value", +} class Unspecified: def __repr__(self): @@ -345,6 +356,7 @@ def __init__(self): # you should check the _return_value for errors, and # "goto exit" if there are any. self.return_conversion = [] + self.converter_retval = "_return_value" # The C statements required to do some operations # after the end of parsing but before cleaning up. @@ -803,13 +815,11 @@ def output_templates(self, f): # parser_body_fields remembers the fields passed in to the # previous call to parser_body. this is used for an awful hack. parser_body_fields = () - parser_body_declarations = '' def parser_body(prototype, *fields, declarations=''): - nonlocal parser_body_fields, parser_body_declarations + nonlocal parser_body_fields add, output = text_accumulator() add(prototype) parser_body_fields = fields - parser_body_declarations = declarations fields = list(fields) fields.insert(0, normalize_snippet(""" @@ -1943,12 +1953,12 @@ def dump(self): return_converters = {} -def write_file(filename, new_contents): +def write_file(filename, new_contents, force=False): try: with open(filename, 'r', encoding="utf-8") as fp: old_contents = fp.read() - if old_contents == new_contents: + if old_contents == new_contents and not force: # no change: avoid modifying the file modification time return except FileNotFoundError: @@ -2112,6 +2122,8 @@ def parse(self, input): traceback.format_exc().rstrip()) printer.print_block(block) + clinic_out = [] + # these are destinations not buffers for name, destination in self.destinations.items(): if destination.type == 'suppress': @@ -2151,10 +2163,11 @@ def parse(self, input): block.input = 'preserve\n' printer_2 = BlockPrinter(self.language) printer_2.print_block(block, core_includes=True) - write_file(destination.filename, printer_2.f.getvalue()) + pair = destination.filename, printer_2.f.getvalue() + clinic_out.append(pair) continue - return printer.f.getvalue() + return printer.f.getvalue(), clinic_out def _module_and_class(self, fields): @@ -2210,9 +2223,13 @@ def parse_file(filename, *, verify=True, output=None): return clinic = Clinic(language, verify=verify, filename=filename) - cooked = clinic.parse(raw) + src_out, clinic_out = clinic.parse(raw) - write_file(output, cooked) + # If clinic output changed, force updating the source file as well. + force = bool(clinic_out) + write_file(output, src_out, force=force) + for fn, data in clinic_out: + write_file(fn, data) def compute_checksum(input, length=None): @@ -3877,15 +3894,15 @@ def __init__(self, *, py_default=None, **kwargs): def return_converter_init(self): pass - def declare(self, data, name="_return_value"): + def declare(self, data): line = [] add = line.append add(self.type) if not self.type.endswith('*'): add(' ') - add(name + ';') + add(data.converter_retval + ';') data.declarations.append(''.join(line)) - data.return_value = name + data.return_value = data.converter_retval def err_occurred_if(self, expr, data): data.return_conversion.append('if (({}) && PyErr_Occurred()) {{\n goto exit;\n}}\n'.format(expr)) @@ -3907,8 +3924,10 @@ class bool_return_converter(CReturnConverter): def render(self, function, data): self.declare(data) - self.err_occurred_if("_return_value == -1", data) - data.return_conversion.append('return_value = PyBool_FromLong((long)_return_value);\n') + self.err_occurred_if(f"{data.converter_retval} == -1", data) + data.return_conversion.append( + f'return_value = PyBool_FromLong((long){data.converter_retval});\n' + ) class long_return_converter(CReturnConverter): type = 'long' @@ -3918,9 +3937,10 @@ class long_return_converter(CReturnConverter): def render(self, function, data): self.declare(data) - self.err_occurred_if("_return_value == {}-1".format(self.unsigned_cast), data) + self.err_occurred_if(f"{data.converter_retval} == {self.unsigned_cast}-1", data) data.return_conversion.append( - ''.join(('return_value = ', self.conversion_fn, '(', self.cast, '_return_value);\n'))) + f'return_value = {self.conversion_fn}({self.cast}{data.converter_retval});\n' + ) class int_return_converter(long_return_converter): type = 'int' @@ -3962,9 +3982,10 @@ class double_return_converter(CReturnConverter): def render(self, function, data): self.declare(data) - self.err_occurred_if("_return_value == -1.0", data) + self.err_occurred_if(f"{data.converter_retval} == -1.0", data) data.return_conversion.append( - 'return_value = PyFloat_FromDouble(' + self.cast + '_return_value);\n') + f'return_value = PyFloat_FromDouble({self.cast}{data.converter_retval});\n' + ) class float_return_converter(double_return_converter): type = 'float' diff --git a/Tools/msi/bundle/bootstrap/pch.h b/Tools/msi/bundle/bootstrap/pch.h index b0aa5111dabd0d..6d0974b34c61e7 100644 --- a/Tools/msi/bundle/bootstrap/pch.h +++ b/Tools/msi/bundle/bootstrap/pch.h @@ -5,7 +5,7 @@ // The license and further copyright text can be found in the file // LICENSE.TXT at the root directory of the distribution. // -// +// // // Precompiled header for standard bootstrapper application. // diff --git a/Tools/msi/bundle/bootstrap/resource.h b/Tools/msi/bundle/bootstrap/resource.h index 53c03c319f091f..d951e651f6d20d 100644 --- a/Tools/msi/bundle/bootstrap/resource.h +++ b/Tools/msi/bundle/bootstrap/resource.h @@ -14,7 +14,7 @@ // Next default values for new objects -// +// #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS #define _APS_NEXT_RESOURCE_VALUE 102