From e3aa50bb8c5336b0246b69c17431b141da06e2d7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 12 May 2025 19:42:08 -0600 Subject: [PATCH 01/17] Move the interpreters module to the official stdlib. --- .github/CODEOWNERS | 2 +- .../support => }/interpreters/__init__.py | 0 .../support => }/interpreters/_crossinterp.py | 0 .../support => }/interpreters/channels.py | 0 Lib/{test/support => }/interpreters/queues.py | 0 Lib/test/test__interpchannels.py | 2 +- .../test_interpreter_pool.py | 2 +- Lib/test/test_interpreters/test_api.py | 26 +++++++++---------- Lib/test/test_interpreters/test_channels.py | 16 ++++++------ Lib/test/test_interpreters/test_lifecycle.py | 4 +-- Lib/test/test_interpreters/test_queues.py | 18 ++++++------- Lib/test/test_interpreters/test_stress.py | 2 +- Lib/test/test_interpreters/utils.py | 2 +- Lib/test/test_sys.py | 2 +- Lib/test/test_threading.py | 2 +- Lib/test/test_types.py | 6 ++--- Makefile.pre.in | 2 +- Modules/_interpchannelsmodule.c | 6 +---- Modules/_interpqueuesmodule.c | 8 ++---- Modules/_testinternalcapi.c | 4 +-- Python/stdlib_module_names.h | 1 + 21 files changed, 49 insertions(+), 56 deletions(-) rename Lib/{test/support => }/interpreters/__init__.py (100%) rename Lib/{test/support => }/interpreters/_crossinterp.py (100%) rename Lib/{test/support => }/interpreters/channels.py (100%) rename Lib/{test/support => }/interpreters/queues.py (100%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 775d9c63260c83..94305128e00a1e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -281,7 +281,7 @@ Doc/howto/clinic.rst @erlend-aasland # Subinterpreters **/*interpreteridobject.* @ericsnowcurrently **/*crossinterp* @ericsnowcurrently -Lib/test/support/interpreters/ @ericsnowcurrently +Lib/interpreters/ @ericsnowcurrently Modules/_interp*module.c @ericsnowcurrently Lib/test/test_interpreters/ @ericsnowcurrently diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/interpreters/__init__.py similarity index 100% rename from Lib/test/support/interpreters/__init__.py rename to Lib/interpreters/__init__.py diff --git a/Lib/test/support/interpreters/_crossinterp.py b/Lib/interpreters/_crossinterp.py similarity index 100% rename from Lib/test/support/interpreters/_crossinterp.py rename to Lib/interpreters/_crossinterp.py diff --git a/Lib/test/support/interpreters/channels.py b/Lib/interpreters/channels.py similarity index 100% rename from Lib/test/support/interpreters/channels.py rename to Lib/interpreters/channels.py diff --git a/Lib/test/support/interpreters/queues.py b/Lib/interpreters/queues.py similarity index 100% rename from Lib/test/support/interpreters/queues.py rename to Lib/interpreters/queues.py diff --git a/Lib/test/test__interpchannels.py b/Lib/test/test__interpchannels.py index 88eee03a3de93a..68b19157aa3a90 100644 --- a/Lib/test/test__interpchannels.py +++ b/Lib/test/test__interpchannels.py @@ -9,7 +9,7 @@ from test.support import import_helper, skip_if_sanitizer _channels = import_helper.import_module('_interpchannels') -from test.support.interpreters import _crossinterp +from interpreters import _crossinterp from test.test__interpreters import ( _interpreters, _run_output, diff --git a/Lib/test/test_concurrent_futures/test_interpreter_pool.py b/Lib/test/test_concurrent_futures/test_interpreter_pool.py index f6c62ae4b2021b..f4511ccc0174b2 100644 --- a/Lib/test/test_concurrent_futures/test_interpreter_pool.py +++ b/Lib/test/test_concurrent_futures/test_interpreter_pool.py @@ -8,10 +8,10 @@ from concurrent.futures.interpreter import ( ExecutionFailed, BrokenInterpreterPool, ) +from interpreters import queues import _interpreters from test import support import test.test_asyncio.utils as testasyncio_utils -from test.support.interpreters import queues from .executor import ExecutorTest, mul from .util import BaseTestCase, InterpreterPoolMixin, setup_module diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index b3c9ef8efba37a..c2a775b91e7090 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -13,11 +13,11 @@ from test.support import import_helper # Raise SkipTest if subinterpreters not supported. _interpreters = import_helper.import_module('_interpreters') +import interpreters from test.support import Py_GIL_DISABLED -from test.support import interpreters from test.support import force_not_colorized import test._crossinterp_definitions as defs -from test.support.interpreters import ( +from interpreters import ( InterpreterError, InterpreterNotFoundError, ExecutionFailed, ) from .utils import ( @@ -133,7 +133,7 @@ def test_in_subinterpreter(self): main, = interpreters.list_all() interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + import interpreters interp = interpreters.create() print(interp.id) """)) @@ -196,7 +196,7 @@ def test_subinterpreter(self): main = interpreters.get_main() interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + import interpreters cur = interpreters.get_current() print(cur.id) """)) @@ -213,7 +213,7 @@ def test_idempotent(self): with self.subTest('subinterpreter'): interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + import interpreters cur = interpreters.get_current() print(id(cur)) cur = interpreters.get_current() @@ -225,7 +225,7 @@ def test_idempotent(self): with self.subTest('per-interpreter'): interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + import interpreters cur = interpreters.get_current() print(id(cur)) """)) @@ -582,7 +582,7 @@ def test_from_current(self): main, = interpreters.list_all() interp = interpreters.create() out = _run_output(interp, dedent(f""" - from test.support import interpreters + import interpreters interp = interpreters.Interpreter({interp.id}) try: interp.close() @@ -599,7 +599,7 @@ def test_from_sibling(self): self.assertEqual(set(interpreters.list_all()), {main, interp1, interp2}) interp1.exec(dedent(f""" - from test.support import interpreters + import interpreters interp2 = interpreters.Interpreter({interp2.id}) interp2.close() interp3 = interpreters.create() @@ -806,7 +806,7 @@ def eggs(): ham() """) scriptfile = self.make_script('script.py', tempdir, text=""" - from test.support import interpreters + import interpreters def script(): import spam @@ -827,7 +827,7 @@ def script(): ~~~~~~~~~~~^^^^^^^^ {interpmod_line.strip()} raise ExecutionFailed(excinfo) - test.support.interpreters.ExecutionFailed: RuntimeError: uh-oh! + interpreters.ExecutionFailed: RuntimeError: uh-oh! Uncaught in the interpreter: @@ -1281,7 +1281,7 @@ def run(text): # no module indirection with self.subTest('no indirection'): text = run(f""" - from test.support import interpreters + import interpreters def spam(): # This a global var... @@ -1301,7 +1301,7 @@ def run(interp, func): """) with self.subTest('indirect as func, direct interp'): text = run(f""" - from test.support import interpreters + import interpreters import mymod def spam(): @@ -1317,7 +1317,7 @@ def spam(): # indirect as func, indirect interp new_mod('mymod', f""" - from test.support import interpreters + import interpreters def run(func): interp = interpreters.create() return interp.call(func) diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 0c027b17cea68c..c153d93a2e09fe 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -8,8 +8,8 @@ from test.support import import_helper # Raise SkipTest if subinterpreters not supported. _channels = import_helper.import_module('_interpchannels') -from test.support import interpreters -from test.support.interpreters import channels +import interpreters +from interpreters import channels from .utils import _run_output, TestBase @@ -171,7 +171,7 @@ def test_send_recv_main(self): def test_send_recv_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from test.support.interpreters import channels + from interpreters import channels r, s = channels.create() orig = b'spam' s.send_nowait(orig) @@ -244,7 +244,7 @@ def test_send_recv_nowait_main_with_default(self): def test_send_recv_nowait_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from test.support.interpreters import channels + from interpreters import channels r, s = channels.create() orig = b'spam' s.send_nowait(orig) @@ -387,7 +387,7 @@ def common(rch, sch, unbound=None, presize=0): interp = interpreters.create() _run_output(interp, dedent(f""" - from test.support.interpreters import channels + from interpreters import channels sch = channels.SendChannel({sch.id}) obj1 = b'spam' obj2 = b'eggs' @@ -482,7 +482,7 @@ def test_send_cleared_with_subinterpreter_mixed(self): self.assertEqual(_channels.get_count(rch.id), 0) _run_output(interp, dedent(f""" - from test.support.interpreters import channels + from interpreters import channels sch = channels.SendChannel({sch.id}) sch.send_nowait(1, unbounditems=channels.UNBOUND) sch.send_nowait(2, unbounditems=channels.UNBOUND_ERROR) @@ -518,7 +518,7 @@ def test_send_cleared_with_subinterpreter_multiple(self): sch.send_nowait(1) _run_output(interp1, dedent(f""" - from test.support.interpreters import channels + from interpreters import channels rch = channels.RecvChannel({rch.id}) sch = channels.SendChannel({sch.id}) obj1 = rch.recv() @@ -526,7 +526,7 @@ def test_send_cleared_with_subinterpreter_multiple(self): sch.send_nowait(obj1, unbounditems=channels.UNBOUND_REMOVE) """)) _run_output(interp2, dedent(f""" - from test.support.interpreters import channels + from interpreters import channels rch = channels.RecvChannel({rch.id}) sch = channels.SendChannel({sch.id}) obj2 = rch.recv() diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py index ac24f6568acd95..4f1b6c3eecb4c4 100644 --- a/Lib/test/test_interpreters/test_lifecycle.py +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -119,7 +119,7 @@ def test_sys_path_0(self): # The main interpreter's sys.path[0] should be used by subinterpreters. script = ''' import sys - from test.support import interpreters + import interpreters orig = sys.path[0] @@ -170,7 +170,7 @@ def test_gh_109793(self): # is reported, even when subinterpreters get cleaned up at the end. import subprocess argv = [sys.executable, '-c', '''if True: - from test.support import interpreters + import interpreters interp = interpreters.create() raise Exception '''] diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 757373904d7a43..3cc5cf6c162238 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -7,8 +7,8 @@ from test.support import import_helper, Py_DEBUG # Raise SkipTest if subinterpreters not supported. _queues = import_helper.import_module('_interpqueues') -from test.support import interpreters -from test.support.interpreters import queues, _crossinterp +import interpreters +from interpreters import queues, _crossinterp from .utils import _run_output, TestBase as _TestBase @@ -126,7 +126,7 @@ def test_shareable(self): interp = interpreters.create() interp.exec(dedent(f""" - from test.support.interpreters import queues + from interpreters import queues queue1 = queues.Queue({queue1.id}) """)); @@ -324,7 +324,7 @@ def test_put_get_full_fallback(self): def test_put_get_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from test.support.interpreters import queues + from interpreters import queues queue = queues.create() """)) for methname in ('get', 'get_nowait'): @@ -351,7 +351,7 @@ def test_put_get_different_interpreters(self): out = _run_output( interp, dedent(f""" - from test.support.interpreters import queues + from interpreters import queues queue1 = queues.Queue({queue1.id}) queue2 = queues.Queue({queue2.id}) assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1' @@ -390,7 +390,7 @@ def common(queue, unbound=None, presize=0): interp = interpreters.create() _run_output(interp, dedent(f""" - from test.support.interpreters import queues + from interpreters import queues queue = queues.Queue({queue.id}) obj1 = b'spam' obj2 = b'eggs' @@ -468,7 +468,7 @@ def test_put_cleared_with_subinterpreter_mixed(self): queue = queues.create() interp = interpreters.create() _run_output(interp, dedent(f""" - from test.support.interpreters import queues + from interpreters import queues queue = queues.Queue({queue.id}) queue.put(1, unbounditems=queues.UNBOUND) queue.put(2, unbounditems=queues.UNBOUND_ERROR) @@ -504,14 +504,14 @@ def test_put_cleared_with_subinterpreter_multiple(self): queue.put(1) _run_output(interp1, dedent(f""" - from test.support.interpreters import queues + from interpreters import queues queue = queues.Queue({queue.id}) obj1 = queue.get() queue.put(2, unbounditems=queues.UNBOUND) queue.put(obj1, unbounditems=queues.UNBOUND_REMOVE) """)) _run_output(interp2, dedent(f""" - from test.support.interpreters import queues + from interpreters import queues queue = queues.Queue({queue.id}) obj2 = queue.get() obj1 = queue.get() diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py index fae2f38cb5534b..31ebc3cfa0bdec 100644 --- a/Lib/test/test_interpreters/test_stress.py +++ b/Lib/test/test_interpreters/test_stress.py @@ -6,7 +6,7 @@ from test.support import threading_helper # Raise SkipTest if subinterpreters not supported. import_helper.import_module('_interpreters') -from test.support import interpreters +import interpreters from .utils import TestBase diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index c25e0fb7475e7e..ad5aa088bcdd96 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -21,7 +21,7 @@ import _interpreters except ImportError as exc: raise unittest.SkipTest(str(exc)) -from test.support import interpreters +import interpreters try: diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 65d15610ed1505..c9268825d93b02 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -24,7 +24,7 @@ from test.support import force_not_colorized from test.support import SHORT_TIMEOUT try: - from test.support import interpreters + import interpreters except ImportError: interpreters = None import textwrap diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 59b3a749d2fffa..5ece72d49c97c8 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -28,7 +28,7 @@ from test import support try: - from test.support import interpreters + import interpreters except ImportError: interpreters = None diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 9011e0e1962820..14df31f5efed50 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -2513,10 +2513,10 @@ class SubinterpreterTests(unittest.TestCase): def setUpClass(cls): global interpreters try: - from test.support import interpreters + import interpreters except ModuleNotFoundError: raise unittest.SkipTest('subinterpreters required') - import test.support.interpreters.channels # noqa: F401 + import interpreters.channels # noqa: F401 @cpython_only @no_rerun('channels (and queues) might have a refleak; see gh-122199') @@ -2547,7 +2547,7 @@ def collate_results(raw): main_results = collate_results(raw) interp = interpreters.create() - interp.exec('from test.support import interpreters') + interp.exec('import interpreters') interp.prepare_main(sch=sch) interp.exec(script) raw = rch.recv_nowait() diff --git a/Makefile.pre.in b/Makefile.pre.in index b5703fbe6ae974..5e913989f3cb45 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2526,6 +2526,7 @@ LIBSUBDIRS= asyncio \ http \ idlelib idlelib/Icons \ importlib importlib/resources importlib/metadata \ + interpreters \ json \ logging \ multiprocessing multiprocessing/dummy \ @@ -2573,7 +2574,6 @@ TESTSUBDIRS= idlelib/idle_test \ test/subprocessdata \ test/support \ test/support/_hypothesis_stubs \ - test/support/interpreters \ test/test_asyncio \ test/test_capi \ test/test_cext \ diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index ea2e5f99dfa308..d65066e8c3a631 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -2744,11 +2744,7 @@ _get_current_channelend_type(int end) // Force the module to be loaded, to register the type. PyObject *highlevel = PyImport_ImportModule("interpreters.channels"); if (highlevel == NULL) { - PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters.channels"); - if (highlevel == NULL) { - return NULL; - } + return NULL; } Py_DECREF(highlevel); if (end == CHANNEL_SEND) { diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 71d8fd8716cd94..046fc6ebf0d1a5 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -138,11 +138,7 @@ ensure_highlevel_module_loaded(void) { PyObject *highlevel = PyImport_ImportModule("interpreters.queues"); if (highlevel == NULL) { - PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters.queues"); - if (highlevel == NULL) { - return -1; - } + return -1; } Py_DECREF(highlevel); return 0; @@ -299,7 +295,7 @@ add_QueueError(PyObject *mod) { module_state *state = get_module_state(mod); -#define PREFIX "test.support.interpreters." +#define PREFIX "interpreters." #define ADD_EXCTYPE(NAME, BASE, DOC) \ assert(state->NAME == NULL); \ if (add_exctype(mod, &state->NAME, PREFIX #NAME, DOC, BASE) < 0) { \ diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 136e6a7a015049..04a4fd0d0552c2 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1785,9 +1785,9 @@ exec_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) /* To run some code in a sub-interpreter. -Generally you can use test.support.interpreters, +Generally you can use the interpreters module, but we keep this helper as a distinct implementation. -That's especially important for testing test.support.interpreters. +That's especially important for testing the interpreters module. */ static PyObject * run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 56e349a544c079..86dc2621017754 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -177,6 +177,7 @@ static const char* _Py_stdlib_module_names[] = { "imaplib", "importlib", "inspect", +"interpreters", "io", "ipaddress", "itertools", From 88e47fbb8aee2db39a118875179a5f865f7dc7a4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 30 May 2025 09:46:44 -0600 Subject: [PATCH 02/17] Add a NEWS entry. --- .../next/Library/2025-05-30-09-46-21.gh-issue-134939.Pu3nnm.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2025-05-30-09-46-21.gh-issue-134939.Pu3nnm.rst diff --git a/Misc/NEWS.d/next/Library/2025-05-30-09-46-21.gh-issue-134939.Pu3nnm.rst b/Misc/NEWS.d/next/Library/2025-05-30-09-46-21.gh-issue-134939.Pu3nnm.rst new file mode 100644 index 00000000000000..b16db06f86b238 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-30-09-46-21.gh-issue-134939.Pu3nnm.rst @@ -0,0 +1 @@ +Add the :mod:`interpreters` module. See :pep:`734`. From ce3505b1cceaa3d14cb48b7642e0a825784671a5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 30 May 2025 11:41:13 -0600 Subject: [PATCH 03/17] Add a whatsnew entry. --- Doc/whatsnew/3.14.rst | 95 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 561d1a8914b50c..1bc8c3d50cacae 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -83,6 +83,7 @@ and improvements in user-friendliness and correctness. .. PEP-sized items next. * :ref:`PEP 649 and 749: deferred evaluation of annotations ` +* :ref:`PEP 734: Multiple Interpreters in the Stdlib ` * :ref:`PEP 741: Python Configuration C API ` * :ref:`PEP 750: Template strings ` * :ref:`PEP 758: Allow except and except* expressions without parentheses ` @@ -123,6 +124,98 @@ of Python. See :ref:`below ` for details. New features ============ +.. _whatsnew314-pep734: + +PEP 734: Multiple Interpreters in the Stdlib +-------------------------------------------- + +The CPython runtime supports running multiple copies of Python in the +same process simultaneously and has done so for over 20 years. +Each of these separate copies is called an "interpreter". +However, the feature has been available only through the C-API. + +That limitation is removed in the 3.14 release, +with the new :mod:`interpreters` module! + +There are at least two notable reasons why using multiple interpreters +is worth considering: + +* they support a new (to Python), human-friendly concurrency model +* true multi-core parallelism + +For some use cases, concurrency in software enables efficiency and +can simplify software, at a high level. At the same time, implementing +and maintaining all but the simplest concurrency is often a struggle +for the human brain. That especially applies to plain threads +(e.g. :mod:`threading`), where all memory is shared between all threads. + +With multiple isolated interpreters, you can take advantage of a class +of concurrency models, like CSP or the actor model, that have found +success in other programming languages, like Smalltalk, Erlang, +Haskell, and Go. Think of multiple interpreters like threads +but with opt-in sharing. + +Regarding multi-core parallelism: as of the 3.12 release, interpreters +are now sufficiently isolated from one another to be used in parallel. +(See :pep:`684`.) This unlocks a variety of CPU-intensive use cases +for Python that were limited by the :term:`GIL`. + +Using multiple interpreters is similar in many ways to +:mod:`multiprocessing`, in that they both provide isolated logical +"processes" that can run in parallel, with no sharing by default. +. However, when using multiple interpreters, an application will use +fewer system resources and will operate more efficiently (since it +stays within the same process). Think of multiple interpreters as +having the isolation of processes with the efficiency of threads. + +.. XXX Add an example or two. +.. XXX Link to the not-yet-added HOWTO doc. + +While the feature has been around for decades, multiple interpreters +have not been used widely, due to low awareness and the lack of a stdlib +module. Consequently, they currently have several notable limitations, +which will improve significantly now that the feature is finally +going mainstream. + +Current limitations: + +* starting each interpreter has not been optimized yet +* each interpreter uses more memory than necessary + (we will be working next on extensive internal sharing between + interpreters) +* there aren't many options *yet* for truly sharing objects or other + data between interpreters (other than :type:`memoryview`) +* many extension modules on PyPI are not compatible with multiple + interpreters yet (stdlib extension modules *are* compatible) +* the approach to writing applications that use multiple isolated + interpreters is mostly unfamiliar to Python users, for now + +The impact of these limitations will depend on future CPython +improvements, how interpreters are used, and what the community solves +through PyPI packages. Depending on the use case, the limitations may +not have much impact, so try it out! + +Furthermore, future CPython releases will reduce or eliminate overhead +and provide utilities that are less appropriate on PyPI. In the +meantime, most of the limitations can also be addressed through +extension modules, meaning PyPI packages can fill any gap for 3.14, and +even back to 3.12 where interpreters were finally properly isolated and +stopped sharing the :term:`GIL`. Likewise, we expect to slowly see +libraries on PyPI for high-level abstractions on top of interpreters. + +Regarding extension modules, work is in progress to update some PyPI +projects, as well as tools like Cython, PyBind11, Nanobind, and Py03. +The steps for isolating an extension module are found at +:ref:`isolating-extensions-howto`. Isolating a module has a lot of +overlap with what is required to support +:ref:`free-threadeding `, +so the ongoing work in the community in that area will help accelerate +support for multiple interpreters. + +Also added in 3.14: :ref:`concurrent.futures.InterpreterPoolExecutor +`. + + .. _whatsnew314-pep750: PEP 750: Template strings @@ -1108,6 +1201,8 @@ calendar concurrent.futures ------------------ +.. _whatsnew314-concurrent-futures-interp-pool: + * Add :class:`~concurrent.futures.InterpreterPoolExecutor`, which exposes "subinterpreters" (multiple Python interpreters in the same process) to Python code. This is separate from the proposed API From 0a5d0e454aaa2c416ee98a4db9b8e44eaaa483b3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 30 May 2025 11:55:51 -0600 Subject: [PATCH 04/17] Add initial docs. --- Doc/library/concurrency.rst | 2 + Doc/library/interpreters.rst | 160 +++++++++++++++++++++++++++++++++++ Doc/library/python.rst | 1 + 3 files changed, 163 insertions(+) create mode 100644 Doc/library/interpreters.rst diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index 5be1a1106b09a0..e06bad038a2ae4 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -23,6 +23,8 @@ multitasking). Here's an overview: queue.rst contextvars.rst +Also see the :mod:`interpreters` module. + The following are support modules for some of the above services: diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst new file mode 100644 index 00000000000000..58e3f0d7a48302 --- /dev/null +++ b/Doc/library/interpreters.rst @@ -0,0 +1,160 @@ +:mod:`!interpreters` --- Multiple Interpreters in the Same Process +================================================================== + +.. module:: interpreters + :synopsis: Multiple Interpreters in the Same Process + +.. moduleauthor:: Eric Snow +.. sectionauthor:: Eric Snow + +.. versionadded:: 3.14 + +**Source code:** :source:`Lib/interpreters/__init__.py` + +-------------- + + +Introduction +------------ + +The :mod:`!interpreters` module constructs higher-level interfaces +on top of the lower level :mod:`!_interpreters` module. + +.. XXX Add references to the upcoming HOWTO docs in the seealso block. + +.. seealso:: + + :ref:`isolating-extensions-howto` + how to update an extension module to support multiple interpreters + + :pep:`554` + + :pep:`734` + + :pep:`684` + +.. XXX Why do we disallow multiple interpreters on WASM? + +.. include:: ../includes/wasm-notavail.rst + + +Key Details +----------- + +Before we dive into examples, there are a small number of details +to keep in mind about using multiple interpreters: + +* isolated, by default +* no implicit threads +* not all PyPI packages support use in multiple interpreters yet + +.. XXX Are there other relevant details to list? + + +API Summary +----------- + ++----------------------------------+----------------------------------------------+ +| signature | description | ++==================================+==============================================+ +| ``list_all() -> [Interpreter]`` | Get all existing interpreters. | ++----------------------------------+----------------------------------------------+ +| ``get_current() -> Interpreter`` | Get the currently running interpreter. | ++----------------------------------+----------------------------------------------+ +| ``get_main() -> Interpreter`` | Get the main interpreter. | ++----------------------------------+----------------------------------------------+ +| ``create() -> Interpreter`` | Initialize a new (idle) Python interpreter. | ++----------------------------------+----------------------------------------------+ + +| + ++---------------------------------------------------+---------------------------------------------------+ +| signature | description | ++===================================================+===================================================+ +| ``class Interpreter`` | A single interpreter. | ++---------------------------------------------------+---------------------------------------------------+ +| ``.id`` | The interpreter's ID (read-only). | ++---------------------------------------------------+---------------------------------------------------+ +| ``.whence`` | Where the interpreter came from (read-only). | ++---------------------------------------------------+---------------------------------------------------+ +| ``.is_running() -> bool`` | Is the interpreter currently executing code | +| | in its :mod:`!__main__` module? | ++---------------------------------------------------+---------------------------------------------------+ +| ``.close()`` | Finalize and destroy the interpreter. | ++---------------------------------------------------+---------------------------------------------------+ +| ``.prepare_main(ns=None, **kwargs)`` | Bind "shareable" objects in :mod:`!__main__`. | ++---------------------------------------------------+---------------------------------------------------+ +| ``.exec(src_str, /, dedent=True)`` | | Run the given source code in the interpreter | +| | | (in the current thread). | ++---------------------------------------------------+---------------------------------------------------+ +| ``.call(callable, /, *args, **kwargs)`` | | Run the given function in the interpreter | +| | | (in the current thread). | ++---------------------------------------------------+---------------------------------------------------+ +| ``.call_in_thread(callable, /, *args, **kwargs)`` | | Run the given function in the interpreter | +| | | (in a new thread). | ++---------------------------------------------------+---------------------------------------------------+ + +Exceptions: + ++--------------------------+------------------+---------------------------------------------------+ +| class | base class | description | ++==========================+==================+===================================================+ +| InterpreterError | Exception | An interpreter-related error happened. | ++--------------------------+------------------+---------------------------------------------------+ +| InterpreterNotFoundError | InterpreterError | The targeted interpreter no longer exists. | ++--------------------------+------------------+---------------------------------------------------+ +| ExecutionFailed | InterpreterError | The running code raised an uncaught exception. | ++--------------------------+------------------+---------------------------------------------------+ +| NotShareableError | TypeError | The object cannot be sent to another interpreter. | ++--------------------------+------------------+---------------------------------------------------+ + +.. XXX Document the ExecutionFailed attrs. + + +.. XXX Add API summary for communicating between interpreters. + + +.. _interp-examples: + +Basic Usage +----------- + +Creating an interpreter and running code in it: + +:: + + import interpreters + + interp = interpreters.create() + + # Run in the current OS thread. + + interp.exec('print("spam!")') + + interp.exec("""if True: + print('spam!') + """) + + from textwrap import dedent + interp.exec(dedent(""" + print('spam!') + """)) + + def run(): + print('spam!') + + interp.call(run) + + # Run in new OS thread. + + t = interp.call_in_thread(run) + t.join() + + +.. XXX Describe module functions in more detail. + + +.. XXX Describe module types in more detail. + + +.. XXX Explain about object "sharing". diff --git a/Doc/library/python.rst b/Doc/library/python.rst index c2c231af7c3033..30179a0531d59a 100644 --- a/Doc/library/python.rst +++ b/Doc/library/python.rst @@ -17,6 +17,7 @@ overview: builtins.rst __main__.rst warnings.rst + interpreters.rst dataclasses.rst contextlib.rst abc.rst From fe8635362f547dad33906343a69dea9c305757f5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 30 May 2025 15:25:22 -0600 Subject: [PATCH 05/17] Fix formatting and typos. --- Doc/library/interpreters.rst | 174 +++++++++++++++++++++-------------- Doc/whatsnew/3.14.rst | 2 +- 2 files changed, 106 insertions(+), 70 deletions(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index 58e3f0d7a48302..a182e083f4d679 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -51,70 +51,112 @@ to keep in mind about using multiple interpreters: .. XXX Are there other relevant details to list? -API Summary ------------ +Reference +--------- + +This module defines the following functions: + +.. function:: list_all() + + -> [Interpreter]`` + Get all existing interpreters. + +.. function:: get_current() + + -> Interpreter + Get the currently running interpreter. + +.. function:: get_main() + + -> Interpreter + Get the main interpreter. + +.. function:: create() + + -> Interpreter + Initialize a new (idle) Python interpreter. + + +Interpreter objects +^^^^^^^^^^^^^^^^^^^ + +.. class:: Interpreter(id) + + A single interpreter in the current process. + + Generally, :class:`Interpreter` shouldn't be called directly. + Instead, use :func:`create` or one of the other module functions. + + .. attribute:: id + + (read-only) + + The interpreter's ID. + + .. attribute:: whence + + (read-only) + + Where the interpreter came from. + + .. method:: is_running() + + Is the interpreter currently executing code in its + :mod:`!__main__` module? + + .. method:: close() + + Finalize and destroy the interpreter. + + .. method:: prepare_main(ns=None, **kwargs) + + Bind "shareable" objects in the interpreter's + :mod:`!__main__` module. + + .. method:: exec(code, /, dedent=True) + + Run the given source code in the interpreter (in the current thread). + + .. method:: call(callable, /, *args, **kwargs) + + Run the given function in the interpreter (in the current thread). + + .. method:: call_in_thread(callable, /, *args, **kwargs) + + Run the given function in the interpreter (in a new thread). + +Exceptions +^^^^^^^^^^ + +.. exception:: InterpreterError + + This exception, a subclass of :exc:`Exception`, is raised when + an interpreter-related error happens. + +.. exception:: InterpreterNotFoundError + + This exception, a subclass of :exc:`InterpreterError`, is raised when + the targeted interpreter no longer exists. + +.. exception:: ExecutionFailed + + This exception, a subclass of :exc:`InterpreterError`, is raised when + the running code raised an uncaught exception. + + .. attribute:: excinfo + + A basic snapshot of the exception raised in the other interpreter. + +.. XXX Document the excinfoattrs? + +.. exception:: NotShareableError + + This exception, a subclass of :exc:`TypeError`, is raised when + an object cannot be sent to another interpreter. + + +.. XXX Add functions for communicating between interpreters. -+----------------------------------+----------------------------------------------+ -| signature | description | -+==================================+==============================================+ -| ``list_all() -> [Interpreter]`` | Get all existing interpreters. | -+----------------------------------+----------------------------------------------+ -| ``get_current() -> Interpreter`` | Get the currently running interpreter. | -+----------------------------------+----------------------------------------------+ -| ``get_main() -> Interpreter`` | Get the main interpreter. | -+----------------------------------+----------------------------------------------+ -| ``create() -> Interpreter`` | Initialize a new (idle) Python interpreter. | -+----------------------------------+----------------------------------------------+ - -| - -+---------------------------------------------------+---------------------------------------------------+ -| signature | description | -+===================================================+===================================================+ -| ``class Interpreter`` | A single interpreter. | -+---------------------------------------------------+---------------------------------------------------+ -| ``.id`` | The interpreter's ID (read-only). | -+---------------------------------------------------+---------------------------------------------------+ -| ``.whence`` | Where the interpreter came from (read-only). | -+---------------------------------------------------+---------------------------------------------------+ -| ``.is_running() -> bool`` | Is the interpreter currently executing code | -| | in its :mod:`!__main__` module? | -+---------------------------------------------------+---------------------------------------------------+ -| ``.close()`` | Finalize and destroy the interpreter. | -+---------------------------------------------------+---------------------------------------------------+ -| ``.prepare_main(ns=None, **kwargs)`` | Bind "shareable" objects in :mod:`!__main__`. | -+---------------------------------------------------+---------------------------------------------------+ -| ``.exec(src_str, /, dedent=True)`` | | Run the given source code in the interpreter | -| | | (in the current thread). | -+---------------------------------------------------+---------------------------------------------------+ -| ``.call(callable, /, *args, **kwargs)`` | | Run the given function in the interpreter | -| | | (in the current thread). | -+---------------------------------------------------+---------------------------------------------------+ -| ``.call_in_thread(callable, /, *args, **kwargs)`` | | Run the given function in the interpreter | -| | | (in a new thread). | -+---------------------------------------------------+---------------------------------------------------+ - -Exceptions: - -+--------------------------+------------------+---------------------------------------------------+ -| class | base class | description | -+==========================+==================+===================================================+ -| InterpreterError | Exception | An interpreter-related error happened. | -+--------------------------+------------------+---------------------------------------------------+ -| InterpreterNotFoundError | InterpreterError | The targeted interpreter no longer exists. | -+--------------------------+------------------+---------------------------------------------------+ -| ExecutionFailed | InterpreterError | The running code raised an uncaught exception. | -+--------------------------+------------------+---------------------------------------------------+ -| NotShareableError | TypeError | The object cannot be sent to another interpreter. | -+--------------------------+------------------+---------------------------------------------------+ - -.. XXX Document the ExecutionFailed attrs. - - -.. XXX Add API summary for communicating between interpreters. - - -.. _interp-examples: Basic Usage ----------- @@ -151,10 +193,4 @@ Creating an interpreter and running code in it: t.join() -.. XXX Describe module functions in more detail. - - -.. XXX Describe module types in more detail. - - .. XXX Explain about object "sharing". diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 1bc8c3d50cacae..89dd0034137a10 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -163,7 +163,7 @@ for Python that were limited by the :term:`GIL`. Using multiple interpreters is similar in many ways to :mod:`multiprocessing`, in that they both provide isolated logical "processes" that can run in parallel, with no sharing by default. -. However, when using multiple interpreters, an application will use +However, when using multiple interpreters, an application will use fewer system resources and will operate more efficiently (since it stays within the same process). Think of multiple interpreters as having the isolation of processes with the efficiency of threads. From 382cff24d0f25990b26af355a45647dade8adb44 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 30 May 2025 15:31:53 -0600 Subject: [PATCH 06/17] Fix doc lint. --- Doc/library/interpreters.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index a182e083f4d679..98532579d83e9b 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -58,23 +58,22 @@ This module defines the following functions: .. function:: list_all() - -> [Interpreter]`` - Get all existing interpreters. + Return a :class:`list` of :class:`Interpreter`, + one for each existing interpreter. .. function:: get_current() - -> Interpreter - Get the currently running interpreter. + Return an :class:`Interpreter` object for the currently running + interpreter. .. function:: get_main() - -> Interpreter - Get the main interpreter. + Return an :class:`Interpreter` object for the main interpreter. .. function:: create() - -> Interpreter - Initialize a new (idle) Python interpreter. + Initialize a new (idle) Python interpreter + and return a :class:`Interpreter` object for it. Interpreter objects From 5b6eb1773aa027b899b9690fc3d8cfaa05dc8fbd Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jun 2025 15:25:48 -0600 Subject: [PATCH 07/17] Minor docs fixes. --- Doc/library/interpreters.rst | 15 +++++++-------- Doc/whatsnew/3.14.rst | 5 ++++- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index 98532579d83e9b..17d8065018738c 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -58,7 +58,7 @@ This module defines the following functions: .. function:: list_all() - Return a :class:`list` of :class:`Interpreter`, + Return a :class:`list` of :class:`Interpreter` objects, one for each existing interpreter. .. function:: get_current() @@ -96,12 +96,12 @@ Interpreter objects (read-only) - Where the interpreter came from. + A string describing where the interpreter came from. .. method:: is_running() - Is the interpreter currently executing code in its - :mod:`!__main__` module? + Return ``True`` if the interpreter is currently executing code + in its :mod:`!__main__` module and ``False`` otherwise. .. method:: close() @@ -118,7 +118,8 @@ Interpreter objects .. method:: call(callable, /, *args, **kwargs) - Run the given function in the interpreter (in the current thread). + Return the result of calling running the given function in the + interpreter (in the current thread). .. method:: call_in_thread(callable, /, *args, **kwargs) @@ -160,9 +161,7 @@ Exceptions Basic Usage ----------- -Creating an interpreter and running code in it: - -:: +Creating an interpreter and running code in it:: import interpreters diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 89dd0034137a10..68db35a59a3893 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -204,7 +204,7 @@ stopped sharing the :term:`GIL`. Likewise, we expect to slowly see libraries on PyPI for high-level abstractions on top of interpreters. Regarding extension modules, work is in progress to update some PyPI -projects, as well as tools like Cython, PyBind11, Nanobind, and Py03. +projects, as well as tools like Cython, PyBind11, Nanobind, and PyO3. The steps for isolating an extension module are found at :ref:`isolating-extensions-howto`. Isolating a module has a lot of overlap with what is required to support @@ -215,6 +215,9 @@ support for multiple interpreters. Also added in 3.14: :ref:`concurrent.futures.InterpreterPoolExecutor `. +.. seealso:: + :pep:`734`. + .. _whatsnew314-pep750: From 226e75c1054fda1ff2379b6f9e2bf59fffcd67e4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jun 2025 15:26:07 -0600 Subject: [PATCH 08/17] Clarify about isolation. --- Doc/library/interpreters.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index 17d8065018738c..1196b68fdc9777 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -50,6 +50,10 @@ to keep in mind about using multiple interpreters: .. XXX Are there other relevant details to list? +In the context of multiple interpreters, "isolated" means each +interpreter shares no state between them. In practice, there is some +process-global data they all share, but that is managed by the runtime. + Reference --------- From eb7f7eb7b5d8a91791567ac0d87f7c295d8428d4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jun 2025 16:01:08 -0600 Subject: [PATCH 09/17] Hide the channels module. --- .github/CODEOWNERS | 4 ++++ .../support}/channels.py | 4 ++-- Lib/test/test_interpreters/test_channels.py | 14 +++++++------- Lib/test/test_types.py | 4 ++-- Modules/_interpchannelsmodule.c | 19 ++++++++++++++++--- 5 files changed, 31 insertions(+), 14 deletions(-) rename Lib/{interpreters => test/support}/channels.py (99%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 94305128e00a1e..e7fdbe9c220688 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -284,6 +284,10 @@ Doc/howto/clinic.rst @erlend-aasland Lib/interpreters/ @ericsnowcurrently Modules/_interp*module.c @ericsnowcurrently Lib/test/test_interpreters/ @ericsnowcurrently +Lib/test/test__interp*.py @ericsnowcurrently +Lib/test/support/channels.py @ericsnowcurrently +Lib/concurrent/futures/interpreter.py @ericsnowcurrently +Doc/library/interpreters.rst @ericsnowcurrently # Android **/*Android* @mhsmith @freakboy3742 diff --git a/Lib/interpreters/channels.py b/Lib/test/support/channels.py similarity index 99% rename from Lib/interpreters/channels.py rename to Lib/test/support/channels.py index 1724759b75a999..27b363ee0fa699 100644 --- a/Lib/interpreters/channels.py +++ b/Lib/test/support/channels.py @@ -2,14 +2,14 @@ import time import _interpchannels as _channels -from . import _crossinterp +from interpreters import _crossinterp # aliases: from _interpchannels import ( ChannelError, ChannelNotFoundError, ChannelClosedError, # noqa: F401 ChannelEmptyError, ChannelNotEmptyError, # noqa: F401 ) -from ._crossinterp import ( +from interpreters._crossinterp import ( UNBOUND_ERROR, UNBOUND_REMOVE, ) diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index c153d93a2e09fe..25d926b2ba9795 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -9,7 +9,7 @@ # Raise SkipTest if subinterpreters not supported. _channels = import_helper.import_module('_interpchannels') import interpreters -from interpreters import channels +from test.support import channels from .utils import _run_output, TestBase @@ -171,7 +171,7 @@ def test_send_recv_main(self): def test_send_recv_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from interpreters import channels + from test.support import channels r, s = channels.create() orig = b'spam' s.send_nowait(orig) @@ -244,7 +244,7 @@ def test_send_recv_nowait_main_with_default(self): def test_send_recv_nowait_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from interpreters import channels + from test.support import channels r, s = channels.create() orig = b'spam' s.send_nowait(orig) @@ -387,7 +387,7 @@ def common(rch, sch, unbound=None, presize=0): interp = interpreters.create() _run_output(interp, dedent(f""" - from interpreters import channels + from test.support import channels sch = channels.SendChannel({sch.id}) obj1 = b'spam' obj2 = b'eggs' @@ -482,7 +482,7 @@ def test_send_cleared_with_subinterpreter_mixed(self): self.assertEqual(_channels.get_count(rch.id), 0) _run_output(interp, dedent(f""" - from interpreters import channels + from test.support import channels sch = channels.SendChannel({sch.id}) sch.send_nowait(1, unbounditems=channels.UNBOUND) sch.send_nowait(2, unbounditems=channels.UNBOUND_ERROR) @@ -518,7 +518,7 @@ def test_send_cleared_with_subinterpreter_multiple(self): sch.send_nowait(1) _run_output(interp1, dedent(f""" - from interpreters import channels + from test.support import channels rch = channels.RecvChannel({rch.id}) sch = channels.SendChannel({sch.id}) obj1 = rch.recv() @@ -526,7 +526,7 @@ def test_send_cleared_with_subinterpreter_multiple(self): sch.send_nowait(obj1, unbounditems=channels.UNBOUND_REMOVE) """)) _run_output(interp2, dedent(f""" - from interpreters import channels + from test.support import channels rch = channels.RecvChannel({rch.id}) sch = channels.SendChannel({sch.id}) obj2 = rch.recv() diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 14df31f5efed50..3a447a06f4f0dd 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -2516,12 +2516,12 @@ def setUpClass(cls): import interpreters except ModuleNotFoundError: raise unittest.SkipTest('subinterpreters required') - import interpreters.channels # noqa: F401 + import test.support.channels # noqa: F401 @cpython_only @no_rerun('channels (and queues) might have a refleak; see gh-122199') def test_static_types_inherited_slots(self): - rch, sch = interpreters.channels.create() + rch, sch = test.support.channels.create() script = textwrap.dedent(""" import test.support diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index d65066e8c3a631..3b6d7c045d3fde 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -220,6 +220,21 @@ wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) return 0; } +static int +ensure_highlevel_module_loaded(void) +{ + PyObject *highlevel = PyImport_ImportModule("interpreters.channels"); + if (highlevel == NULL) { + PyErr_Clear(); + highlevel = PyImport_ImportModule("test.support.channels"); + if (highlevel == NULL) { + return -1; + } + } + Py_DECREF(highlevel); + return 0; +} + /* module state *************************************************************/ @@ -2742,11 +2757,9 @@ _get_current_channelend_type(int end) } if (cls == NULL) { // Force the module to be loaded, to register the type. - PyObject *highlevel = PyImport_ImportModule("interpreters.channels"); - if (highlevel == NULL) { + if (ensure_highlevel_module_loaded() < 0) { return NULL; } - Py_DECREF(highlevel); if (end == CHANNEL_SEND) { cls = state->send_channel_type; } From 491a642c74bac9c1fcffe40e82b2d54a051db047 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jun 2025 16:16:33 -0600 Subject: [PATCH 10/17] Fix a test. --- Lib/test/test_types.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 3a447a06f4f0dd..9d9240a49af931 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -2516,12 +2516,13 @@ def setUpClass(cls): import interpreters except ModuleNotFoundError: raise unittest.SkipTest('subinterpreters required') - import test.support.channels # noqa: F401 + from test.support import channels # noqa: F401 + cls.create_channel = staticmethod(channels.create) @cpython_only @no_rerun('channels (and queues) might have a refleak; see gh-122199') def test_static_types_inherited_slots(self): - rch, sch = test.support.channels.create() + rch, sch = self.create_channel() script = textwrap.dedent(""" import test.support From fb3e2150dc925048415ee8ec6241967ec0189f14 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 6 Jun 2025 13:32:20 -0600 Subject: [PATCH 11/17] Make interpreters.queues private. --- Lib/concurrent/futures/interpreter.py | 2 +- Lib/interpreters/__init__.py | 2 +- Lib/interpreters/_crossinterp.py | 2 +- Lib/interpreters/{queues.py => _queues.py} | 0 .../test_interpreter_pool.py | 2 +- Lib/test/test_interpreters/test_queues.py | 16 ++++++++-------- Modules/_interpqueuesmodule.c | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) rename Lib/interpreters/{queues.py => _queues.py} (100%) diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py index a2c4fbfd3fb831..f12b4ac33cda27 100644 --- a/Lib/concurrent/futures/interpreter.py +++ b/Lib/concurrent/futures/interpreter.py @@ -167,7 +167,7 @@ def run(self, task): except _interpqueues.QueueError: continue except ModuleNotFoundError: - # interpreters.queues doesn't exist, which means + # interpreters._queues doesn't exist, which means # QueueEmpty doesn't. Act as though it does. continue else: diff --git a/Lib/interpreters/__init__.py b/Lib/interpreters/__init__.py index 6d1b0690805d2d..fcf1469059ed1d 100644 --- a/Lib/interpreters/__init__.py +++ b/Lib/interpreters/__init__.py @@ -26,7 +26,7 @@ def __getattr__(name): if name in ('Queue', 'QueueEmpty', 'QueueFull', 'create_queue'): global create_queue, Queue, QueueEmpty, QueueFull ns = globals() - from .queues import ( + from ._queues import ( create as create_queue, Queue, QueueEmpty, QueueFull, ) diff --git a/Lib/interpreters/_crossinterp.py b/Lib/interpreters/_crossinterp.py index 544e197ba4c028..f47eb693ac861c 100644 --- a/Lib/interpreters/_crossinterp.py +++ b/Lib/interpreters/_crossinterp.py @@ -61,7 +61,7 @@ def __new__(cls): def __repr__(self): return f'{self._MODULE}.{self._NAME}' -# return f'interpreters.queues.UNBOUND' +# return f'interpreters._queues.UNBOUND' UNBOUND = object.__new__(UnboundItem) diff --git a/Lib/interpreters/queues.py b/Lib/interpreters/_queues.py similarity index 100% rename from Lib/interpreters/queues.py rename to Lib/interpreters/_queues.py diff --git a/Lib/test/test_concurrent_futures/test_interpreter_pool.py b/Lib/test/test_concurrent_futures/test_interpreter_pool.py index f4511ccc0174b2..c49480d0ad19e8 100644 --- a/Lib/test/test_concurrent_futures/test_interpreter_pool.py +++ b/Lib/test/test_concurrent_futures/test_interpreter_pool.py @@ -8,7 +8,7 @@ from concurrent.futures.interpreter import ( ExecutionFailed, BrokenInterpreterPool, ) -from interpreters import queues +from interpreters import _queues as queues import _interpreters from test import support import test.test_asyncio.utils as testasyncio_utils diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 3cc5cf6c162238..e3a8928c96490f 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -8,7 +8,7 @@ # Raise SkipTest if subinterpreters not supported. _queues = import_helper.import_module('_interpqueues') import interpreters -from interpreters import queues, _crossinterp +from interpreters import _queues as queues, _crossinterp from .utils import _run_output, TestBase as _TestBase @@ -126,7 +126,7 @@ def test_shareable(self): interp = interpreters.create() interp.exec(dedent(f""" - from interpreters import queues + from interpreters import _queues as queues queue1 = queues.Queue({queue1.id}) """)); @@ -324,7 +324,7 @@ def test_put_get_full_fallback(self): def test_put_get_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from interpreters import queues + from interpreters import _queues as queues queue = queues.create() """)) for methname in ('get', 'get_nowait'): @@ -351,7 +351,7 @@ def test_put_get_different_interpreters(self): out = _run_output( interp, dedent(f""" - from interpreters import queues + from interpreters import _queues as queues queue1 = queues.Queue({queue1.id}) queue2 = queues.Queue({queue2.id}) assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1' @@ -390,7 +390,7 @@ def common(queue, unbound=None, presize=0): interp = interpreters.create() _run_output(interp, dedent(f""" - from interpreters import queues + from interpreters import _queues as queues queue = queues.Queue({queue.id}) obj1 = b'spam' obj2 = b'eggs' @@ -468,7 +468,7 @@ def test_put_cleared_with_subinterpreter_mixed(self): queue = queues.create() interp = interpreters.create() _run_output(interp, dedent(f""" - from interpreters import queues + from interpreters import _queues as queues queue = queues.Queue({queue.id}) queue.put(1, unbounditems=queues.UNBOUND) queue.put(2, unbounditems=queues.UNBOUND_ERROR) @@ -504,14 +504,14 @@ def test_put_cleared_with_subinterpreter_multiple(self): queue.put(1) _run_output(interp1, dedent(f""" - from interpreters import queues + from interpreters import _queues as queues queue = queues.Queue({queue.id}) obj1 = queue.get() queue.put(2, unbounditems=queues.UNBOUND) queue.put(obj1, unbounditems=queues.UNBOUND_REMOVE) """)) _run_output(interp2, dedent(f""" - from interpreters import queues + from interpreters import _queues as queues queue = queues.Queue({queue.id}) obj2 = queue.get() obj1 = queue.get() diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 046fc6ebf0d1a5..42f2b8fdd1f807 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -136,7 +136,7 @@ idarg_int64_converter(PyObject *arg, void *ptr) static int ensure_highlevel_module_loaded(void) { - PyObject *highlevel = PyImport_ImportModule("interpreters.queues"); + PyObject *highlevel = PyImport_ImportModule("interpreters._queues"); if (highlevel == NULL) { return -1; } From cdfa1e7c70222ae2c935166fdeda749b8f00f913 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 6 Jun 2025 12:21:52 -0600 Subject: [PATCH 12/17] interpreters -> concurrent.interpreters --- .github/CODEOWNERS | 6 ++--- Doc/library/concurrency.rst | 3 +-- ...reters.rst => concurrent.interpreters.rst} | 14 +++++----- Doc/library/concurrent.rst | 3 ++- Doc/library/python.rst | 6 ++++- Doc/whatsnew/3.14.rst | 2 +- Lib/{ => concurrent}/interpreters/__init__.py | 0 .../interpreters/_crossinterp.py | 0 Lib/{ => concurrent}/interpreters/_queues.py | 0 Lib/test/support/channels.py | 4 +-- Lib/test/test__interpchannels.py | 2 +- .../test_interpreter_pool.py | 2 +- Lib/test/test_interpreters/test_api.py | 26 +++++++++---------- Lib/test/test_interpreters/test_channels.py | 2 +- Lib/test/test_interpreters/test_lifecycle.py | 4 +-- Lib/test/test_interpreters/test_queues.py | 18 ++++++------- Lib/test/test_interpreters/test_stress.py | 2 +- Lib/test/test_interpreters/utils.py | 2 +- Lib/test/test_sys.py | 2 +- Lib/test/test_threading.py | 2 +- Lib/test/test_types.py | 4 +-- Makefile.pre.in | 3 +-- Modules/_interpchannelsmodule.c | 3 ++- Modules/_interpqueuesmodule.c | 5 ++-- Python/crossinterp_exceptions.h | 6 ++--- Python/stdlib_module_names.h | 1 - 26 files changed, 63 insertions(+), 59 deletions(-) rename Doc/library/{interpreters.rst => concurrent.interpreters.rst} (92%) rename Lib/{ => concurrent}/interpreters/__init__.py (100%) rename Lib/{ => concurrent}/interpreters/_crossinterp.py (100%) rename Lib/{ => concurrent}/interpreters/_queues.py (100%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e7fdbe9c220688..63a28490043899 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -281,13 +281,13 @@ Doc/howto/clinic.rst @erlend-aasland # Subinterpreters **/*interpreteridobject.* @ericsnowcurrently **/*crossinterp* @ericsnowcurrently -Lib/interpreters/ @ericsnowcurrently Modules/_interp*module.c @ericsnowcurrently -Lib/test/test_interpreters/ @ericsnowcurrently Lib/test/test__interp*.py @ericsnowcurrently +Lib/concurrent/interpreters/ @ericsnowcurrently Lib/test/support/channels.py @ericsnowcurrently +Doc/library/concurrent.interpreters.rst @ericsnowcurrently +Lib/test/test_interpreters/ @ericsnowcurrently Lib/concurrent/futures/interpreter.py @ericsnowcurrently -Doc/library/interpreters.rst @ericsnowcurrently # Android **/*Android* @mhsmith @freakboy3742 diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index e06bad038a2ae4..18f9443cbfea20 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -18,13 +18,12 @@ multitasking). Here's an overview: multiprocessing.shared_memory.rst concurrent.rst concurrent.futures.rst + concurrent.interpreters.rst subprocess.rst sched.rst queue.rst contextvars.rst -Also see the :mod:`interpreters` module. - The following are support modules for some of the above services: diff --git a/Doc/library/interpreters.rst b/Doc/library/concurrent.interpreters.rst similarity index 92% rename from Doc/library/interpreters.rst rename to Doc/library/concurrent.interpreters.rst index 1196b68fdc9777..7b389ae587b328 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/concurrent.interpreters.rst @@ -1,7 +1,7 @@ -:mod:`!interpreters` --- Multiple Interpreters in the Same Process -================================================================== +:mod:`concurrent.!interpreters` --- Multiple Interpreters in the Same Process +============================================================================= -.. module:: interpreters +.. module:: concurrent.interpreters :synopsis: Multiple Interpreters in the Same Process .. moduleauthor:: Eric Snow @@ -9,7 +9,7 @@ .. versionadded:: 3.14 -**Source code:** :source:`Lib/interpreters/__init__.py` +**Source code:** :source:`Lib/concurrent/interpreters.py` -------------- @@ -17,8 +17,8 @@ Introduction ------------ -The :mod:`!interpreters` module constructs higher-level interfaces -on top of the lower level :mod:`!_interpreters` module. +The :mod:`!concurrent.interpreters` module constructs higher-level +interfaces on top of the lower level :mod:`!_interpreters` module. .. XXX Add references to the upcoming HOWTO docs in the seealso block. @@ -167,7 +167,7 @@ Basic Usage Creating an interpreter and running code in it:: - import interpreters + from concurrent import interpreters interp = interpreters.create() diff --git a/Doc/library/concurrent.rst b/Doc/library/concurrent.rst index 8caea78bbb57e8..47e84b42cae67b 100644 --- a/Doc/library/concurrent.rst +++ b/Doc/library/concurrent.rst @@ -1,6 +1,7 @@ The :mod:`!concurrent` package ============================== -Currently, there is only one module in this package: +Currently, there are only two modules in this package: * :mod:`concurrent.futures` -- Launching parallel tasks +* :mod:`concurrent.interpreters` -- Multiple interpreters in the same process diff --git a/Doc/library/python.rst b/Doc/library/python.rst index 30179a0531d59a..c5c762e11b99e5 100644 --- a/Doc/library/python.rst +++ b/Doc/library/python.rst @@ -17,7 +17,6 @@ overview: builtins.rst __main__.rst warnings.rst - interpreters.rst dataclasses.rst contextlib.rst abc.rst @@ -28,3 +27,8 @@ overview: inspect.rst annotationlib.rst site.rst + +.. seealso:: + + * See the :mod:`concurrent.interpreters` module, which similarly + exposes core runtime functionality. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 68db35a59a3893..384bd02766cbb4 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -135,7 +135,7 @@ Each of these separate copies is called an "interpreter". However, the feature has been available only through the C-API. That limitation is removed in the 3.14 release, -with the new :mod:`interpreters` module! +with the new :mod:`concurrent.interpreters` module! There are at least two notable reasons why using multiple interpreters is worth considering: diff --git a/Lib/interpreters/__init__.py b/Lib/concurrent/interpreters/__init__.py similarity index 100% rename from Lib/interpreters/__init__.py rename to Lib/concurrent/interpreters/__init__.py diff --git a/Lib/interpreters/_crossinterp.py b/Lib/concurrent/interpreters/_crossinterp.py similarity index 100% rename from Lib/interpreters/_crossinterp.py rename to Lib/concurrent/interpreters/_crossinterp.py diff --git a/Lib/interpreters/_queues.py b/Lib/concurrent/interpreters/_queues.py similarity index 100% rename from Lib/interpreters/_queues.py rename to Lib/concurrent/interpreters/_queues.py diff --git a/Lib/test/support/channels.py b/Lib/test/support/channels.py index 27b363ee0fa699..b2de24d9d3e534 100644 --- a/Lib/test/support/channels.py +++ b/Lib/test/support/channels.py @@ -2,14 +2,14 @@ import time import _interpchannels as _channels -from interpreters import _crossinterp +from concurrent.interpreters import _crossinterp # aliases: from _interpchannels import ( ChannelError, ChannelNotFoundError, ChannelClosedError, # noqa: F401 ChannelEmptyError, ChannelNotEmptyError, # noqa: F401 ) -from interpreters._crossinterp import ( +from concurrent.interpreters._crossinterp import ( UNBOUND_ERROR, UNBOUND_REMOVE, ) diff --git a/Lib/test/test__interpchannels.py b/Lib/test/test__interpchannels.py index 68b19157aa3a90..858d31a73cf4f4 100644 --- a/Lib/test/test__interpchannels.py +++ b/Lib/test/test__interpchannels.py @@ -9,7 +9,7 @@ from test.support import import_helper, skip_if_sanitizer _channels = import_helper.import_module('_interpchannels') -from interpreters import _crossinterp +from concurrent.interpreters import _crossinterp from test.test__interpreters import ( _interpreters, _run_output, diff --git a/Lib/test/test_concurrent_futures/test_interpreter_pool.py b/Lib/test/test_concurrent_futures/test_interpreter_pool.py index c49480d0ad19e8..5fd5684e1035e9 100644 --- a/Lib/test/test_concurrent_futures/test_interpreter_pool.py +++ b/Lib/test/test_concurrent_futures/test_interpreter_pool.py @@ -8,7 +8,7 @@ from concurrent.futures.interpreter import ( ExecutionFailed, BrokenInterpreterPool, ) -from interpreters import _queues as queues +from concurrent.interpreters import _queues as queues import _interpreters from test import support import test.test_asyncio.utils as testasyncio_utils diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index c2a775b91e7090..1403cd145b6787 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -13,11 +13,11 @@ from test.support import import_helper # Raise SkipTest if subinterpreters not supported. _interpreters = import_helper.import_module('_interpreters') -import interpreters +from concurrent import interpreters from test.support import Py_GIL_DISABLED from test.support import force_not_colorized import test._crossinterp_definitions as defs -from interpreters import ( +from concurrent.interpreters import ( InterpreterError, InterpreterNotFoundError, ExecutionFailed, ) from .utils import ( @@ -133,7 +133,7 @@ def test_in_subinterpreter(self): main, = interpreters.list_all() interp = interpreters.create() out = _run_output(interp, dedent(""" - import interpreters + from concurrent import interpreters interp = interpreters.create() print(interp.id) """)) @@ -196,7 +196,7 @@ def test_subinterpreter(self): main = interpreters.get_main() interp = interpreters.create() out = _run_output(interp, dedent(""" - import interpreters + from concurrent import interpreters cur = interpreters.get_current() print(cur.id) """)) @@ -213,7 +213,7 @@ def test_idempotent(self): with self.subTest('subinterpreter'): interp = interpreters.create() out = _run_output(interp, dedent(""" - import interpreters + from concurrent import interpreters cur = interpreters.get_current() print(id(cur)) cur = interpreters.get_current() @@ -225,7 +225,7 @@ def test_idempotent(self): with self.subTest('per-interpreter'): interp = interpreters.create() out = _run_output(interp, dedent(""" - import interpreters + from concurrent import interpreters cur = interpreters.get_current() print(id(cur)) """)) @@ -582,7 +582,7 @@ def test_from_current(self): main, = interpreters.list_all() interp = interpreters.create() out = _run_output(interp, dedent(f""" - import interpreters + from concurrent import interpreters interp = interpreters.Interpreter({interp.id}) try: interp.close() @@ -599,7 +599,7 @@ def test_from_sibling(self): self.assertEqual(set(interpreters.list_all()), {main, interp1, interp2}) interp1.exec(dedent(f""" - import interpreters + from concurrent import interpreters interp2 = interpreters.Interpreter({interp2.id}) interp2.close() interp3 = interpreters.create() @@ -806,7 +806,7 @@ def eggs(): ham() """) scriptfile = self.make_script('script.py', tempdir, text=""" - import interpreters + from concurrent import interpreters def script(): import spam @@ -827,7 +827,7 @@ def script(): ~~~~~~~~~~~^^^^^^^^ {interpmod_line.strip()} raise ExecutionFailed(excinfo) - interpreters.ExecutionFailed: RuntimeError: uh-oh! + concurrent.interpreters.ExecutionFailed: RuntimeError: uh-oh! Uncaught in the interpreter: @@ -1281,7 +1281,7 @@ def run(text): # no module indirection with self.subTest('no indirection'): text = run(f""" - import interpreters + from concurrent import interpreters def spam(): # This a global var... @@ -1301,7 +1301,7 @@ def run(interp, func): """) with self.subTest('indirect as func, direct interp'): text = run(f""" - import interpreters + from concurrent import interpreters import mymod def spam(): @@ -1317,7 +1317,7 @@ def spam(): # indirect as func, indirect interp new_mod('mymod', f""" - import interpreters + from concurrent import interpreters def run(func): interp = interpreters.create() return interp.call(func) diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 25d926b2ba9795..109ddf344539ad 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -8,7 +8,7 @@ from test.support import import_helper # Raise SkipTest if subinterpreters not supported. _channels = import_helper.import_module('_interpchannels') -import interpreters +from concurrent import interpreters from test.support import channels from .utils import _run_output, TestBase diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py index 4f1b6c3eecb4c4..15537ac6cc8f82 100644 --- a/Lib/test/test_interpreters/test_lifecycle.py +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -119,7 +119,7 @@ def test_sys_path_0(self): # The main interpreter's sys.path[0] should be used by subinterpreters. script = ''' import sys - import interpreters + from concurrent import interpreters orig = sys.path[0] @@ -170,7 +170,7 @@ def test_gh_109793(self): # is reported, even when subinterpreters get cleaned up at the end. import subprocess argv = [sys.executable, '-c', '''if True: - import interpreters + from concurrent import interpreters interp = interpreters.create() raise Exception '''] diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index e3a8928c96490f..3e982d76e86314 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -7,8 +7,8 @@ from test.support import import_helper, Py_DEBUG # Raise SkipTest if subinterpreters not supported. _queues = import_helper.import_module('_interpqueues') -import interpreters -from interpreters import _queues as queues, _crossinterp +from concurrent import interpreters +from concurrent.interpreters import _queues as queues, _crossinterp from .utils import _run_output, TestBase as _TestBase @@ -126,7 +126,7 @@ def test_shareable(self): interp = interpreters.create() interp.exec(dedent(f""" - from interpreters import _queues as queues + from concurrent.interpreters import _queues as queues queue1 = queues.Queue({queue1.id}) """)); @@ -324,7 +324,7 @@ def test_put_get_full_fallback(self): def test_put_get_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from interpreters import _queues as queues + from concurrent.interpreters import _queues as queues queue = queues.create() """)) for methname in ('get', 'get_nowait'): @@ -351,7 +351,7 @@ def test_put_get_different_interpreters(self): out = _run_output( interp, dedent(f""" - from interpreters import _queues as queues + from concurrent.interpreters import _queues as queues queue1 = queues.Queue({queue1.id}) queue2 = queues.Queue({queue2.id}) assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1' @@ -390,7 +390,7 @@ def common(queue, unbound=None, presize=0): interp = interpreters.create() _run_output(interp, dedent(f""" - from interpreters import _queues as queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) obj1 = b'spam' obj2 = b'eggs' @@ -468,7 +468,7 @@ def test_put_cleared_with_subinterpreter_mixed(self): queue = queues.create() interp = interpreters.create() _run_output(interp, dedent(f""" - from interpreters import _queues as queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) queue.put(1, unbounditems=queues.UNBOUND) queue.put(2, unbounditems=queues.UNBOUND_ERROR) @@ -504,14 +504,14 @@ def test_put_cleared_with_subinterpreter_multiple(self): queue.put(1) _run_output(interp1, dedent(f""" - from interpreters import _queues as queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) obj1 = queue.get() queue.put(2, unbounditems=queues.UNBOUND) queue.put(obj1, unbounditems=queues.UNBOUND_REMOVE) """)) _run_output(interp2, dedent(f""" - from interpreters import _queues as queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) obj2 = queue.get() obj1 = queue.get() diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py index 31ebc3cfa0bdec..e25e67a0d4f445 100644 --- a/Lib/test/test_interpreters/test_stress.py +++ b/Lib/test/test_interpreters/test_stress.py @@ -6,7 +6,7 @@ from test.support import threading_helper # Raise SkipTest if subinterpreters not supported. import_helper.import_module('_interpreters') -import interpreters +from concurrent import interpreters from .utils import TestBase diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index ad5aa088bcdd96..ae09aa457b48c7 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -21,7 +21,7 @@ import _interpreters except ImportError as exc: raise unittest.SkipTest(str(exc)) -import interpreters +from concurrent import interpreters try: diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index c9268825d93b02..f6e997e4106622 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -24,7 +24,7 @@ from test.support import force_not_colorized from test.support import SHORT_TIMEOUT try: - import interpreters + from concurrent import interpreters except ImportError: interpreters = None import textwrap diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 5ece72d49c97c8..125c27446986c0 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -28,7 +28,7 @@ from test import support try: - import interpreters + from concurrent import interpreters except ImportError: interpreters = None diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 9d9240a49af931..a117413301bebe 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -2513,7 +2513,7 @@ class SubinterpreterTests(unittest.TestCase): def setUpClass(cls): global interpreters try: - import interpreters + from concurrent import interpreters except ModuleNotFoundError: raise unittest.SkipTest('subinterpreters required') from test.support import channels # noqa: F401 @@ -2548,7 +2548,7 @@ def collate_results(raw): main_results = collate_results(raw) interp = interpreters.create() - interp.exec('import interpreters') + interp.exec('from concurrent import interpreters') interp.prepare_main(sch=sch) interp.exec(script) raw = rch.recv_nowait() diff --git a/Makefile.pre.in b/Makefile.pre.in index 5e913989f3cb45..66b34b779f27cb 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2514,7 +2514,7 @@ XMLLIBSUBDIRS= xml xml/dom xml/etree xml/parsers xml/sax LIBSUBDIRS= asyncio \ collections \ compression compression/_common compression/zstd \ - concurrent concurrent/futures \ + concurrent concurrent/futures concurrent/interpreters \ csv \ ctypes ctypes/macholib \ curses \ @@ -2526,7 +2526,6 @@ LIBSUBDIRS= asyncio \ http \ idlelib idlelib/Icons \ importlib importlib/resources importlib/metadata \ - interpreters \ json \ logging \ multiprocessing multiprocessing/dummy \ diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index 3b6d7c045d3fde..ee5e2b005e0a5b 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -223,7 +223,8 @@ wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) static int ensure_highlevel_module_loaded(void) { - PyObject *highlevel = PyImport_ImportModule("interpreters.channels"); + PyObject *highlevel = + PyImport_ImportModule("concurrent.interpreters._channels"); if (highlevel == NULL) { PyErr_Clear(); highlevel = PyImport_ImportModule("test.support.channels"); diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 42f2b8fdd1f807..e22709d5119b7c 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -136,7 +136,8 @@ idarg_int64_converter(PyObject *arg, void *ptr) static int ensure_highlevel_module_loaded(void) { - PyObject *highlevel = PyImport_ImportModule("interpreters._queues"); + PyObject *highlevel = + PyImport_ImportModule("concurrent.interpreters._queues"); if (highlevel == NULL) { return -1; } @@ -295,7 +296,7 @@ add_QueueError(PyObject *mod) { module_state *state = get_module_state(mod); -#define PREFIX "interpreters." +#define PREFIX "concurrent.interpreters." #define ADD_EXCTYPE(NAME, BASE, DOC) \ assert(state->NAME == NULL); \ if (add_exctype(mod, &state->NAME, PREFIX #NAME, DOC, BASE) < 0) { \ diff --git a/Python/crossinterp_exceptions.h b/Python/crossinterp_exceptions.h index ca4ca1cf123e49..12cd61db1b6762 100644 --- a/Python/crossinterp_exceptions.h +++ b/Python/crossinterp_exceptions.h @@ -24,7 +24,7 @@ _ensure_current_cause(PyThreadState *tstate, PyObject *cause) static PyTypeObject _PyExc_InterpreterError = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "interpreters.InterpreterError", + .tp_name = "concurrent.interpreters.InterpreterError", .tp_doc = PyDoc_STR("A cross-interpreter operation failed"), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse, @@ -37,7 +37,7 @@ PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError; static PyTypeObject _PyExc_InterpreterNotFoundError = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "interpreters.InterpreterNotFoundError", + .tp_name = "concurrent.interpreters.InterpreterNotFoundError", .tp_doc = PyDoc_STR("An interpreter was not found"), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse, @@ -51,7 +51,7 @@ PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFou static int _init_notshareableerror(exceptions_t *state) { - const char *name = "interpreters.NotShareableError"; + const char *name = "concurrent.interpreters.NotShareableError"; PyObject *base = PyExc_TypeError; PyObject *ns = NULL; PyObject *exctype = PyErr_NewException(name, base, ns); diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 86dc2621017754..56e349a544c079 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -177,7 +177,6 @@ static const char* _Py_stdlib_module_names[] = { "imaplib", "importlib", "inspect", -"interpreters", "io", "ipaddress", "itertools", From fe1c1ae467f0ad52b1a9dc20ed16b4ac0cd384f4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 6 Jun 2025 15:12:36 -0600 Subject: [PATCH 13/17] Update Doc/library/concurrent.interpreters.rst Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/concurrent.interpreters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/concurrent.interpreters.rst b/Doc/library/concurrent.interpreters.rst index 7b389ae587b328..4711326466e3b7 100644 --- a/Doc/library/concurrent.interpreters.rst +++ b/Doc/library/concurrent.interpreters.rst @@ -1,4 +1,4 @@ -:mod:`concurrent.!interpreters` --- Multiple Interpreters in the Same Process +:mod:`!concurrent.interpreters` --- Multiple Interpreters in the Same Process ============================================================================= .. module:: concurrent.interpreters From 399042f48fe5ac74acec9f9b7dc6374a6b941399 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 6 Jun 2025 15:12:47 -0600 Subject: [PATCH 14/17] Update Doc/library/concurrent.interpreters.rst Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/concurrent.interpreters.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/concurrent.interpreters.rst b/Doc/library/concurrent.interpreters.rst index 4711326466e3b7..15ce29d6178210 100644 --- a/Doc/library/concurrent.interpreters.rst +++ b/Doc/library/concurrent.interpreters.rst @@ -50,8 +50,8 @@ to keep in mind about using multiple interpreters: .. XXX Are there other relevant details to list? -In the context of multiple interpreters, "isolated" means each -interpreter shares no state between them. In practice, there is some +In the context of multiple interpreters, "isolated" means that +different interpreters do not share any state. In practice, there is some process-global data they all share, but that is managed by the runtime. From 907bb8cd282acd5792f4df58cf73e3a634d1700e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 6 Jun 2025 15:23:54 -0600 Subject: [PATCH 15/17] Fix the NEWS entry. --- .../next/Library/2025-05-30-09-46-21.gh-issue-134939.Pu3nnm.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2025-05-30-09-46-21.gh-issue-134939.Pu3nnm.rst b/Misc/NEWS.d/next/Library/2025-05-30-09-46-21.gh-issue-134939.Pu3nnm.rst index b16db06f86b238..2bda69bff52156 100644 --- a/Misc/NEWS.d/next/Library/2025-05-30-09-46-21.gh-issue-134939.Pu3nnm.rst +++ b/Misc/NEWS.d/next/Library/2025-05-30-09-46-21.gh-issue-134939.Pu3nnm.rst @@ -1 +1 @@ -Add the :mod:`interpreters` module. See :pep:`734`. +Add the :mod:`concurrent.interpreters` module. See :pep:`734`. From e4cbc6607e4e3e57b194584f3292f082abb95d8a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 6 Jun 2025 15:41:52 -0600 Subject: [PATCH 16/17] Drop the module __getattr__. --- Lib/concurrent/interpreters/__init__.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/Lib/concurrent/interpreters/__init__.py b/Lib/concurrent/interpreters/__init__.py index fcf1469059ed1d..0fd661249a276c 100644 --- a/Lib/concurrent/interpreters/__init__.py +++ b/Lib/concurrent/interpreters/__init__.py @@ -9,6 +9,10 @@ InterpreterError, InterpreterNotFoundError, NotShareableError, is_shareable, ) +from ._queues import ( + create as create_queue, + Queue, QueueEmpty, QueueFull, +) __all__ = [ @@ -20,21 +24,6 @@ ] -_queuemod = None - -def __getattr__(name): - if name in ('Queue', 'QueueEmpty', 'QueueFull', 'create_queue'): - global create_queue, Queue, QueueEmpty, QueueFull - ns = globals() - from ._queues import ( - create as create_queue, - Queue, QueueEmpty, QueueFull, - ) - return ns[name] - else: - raise AttributeError(name) - - _EXEC_FAILURE_STR = """ {superstr} From c0b637ef7d2d2627596b52d23ce2cd3b2160c925 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 9 Jun 2025 10:55:21 -0600 Subject: [PATCH 17/17] Apply suggestions from code review. Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/concurrent.interpreters.rst | 8 ++++---- Doc/library/concurrent.rst | 2 +- Doc/whatsnew/3.14.rst | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Doc/library/concurrent.interpreters.rst b/Doc/library/concurrent.interpreters.rst index 15ce29d6178210..8860418e87a585 100644 --- a/Doc/library/concurrent.interpreters.rst +++ b/Doc/library/concurrent.interpreters.rst @@ -1,8 +1,8 @@ -:mod:`!concurrent.interpreters` --- Multiple Interpreters in the Same Process +:mod:`!concurrent.interpreters` --- Multiple interpreters in the same process ============================================================================= .. module:: concurrent.interpreters - :synopsis: Multiple Interpreters in the Same Process + :synopsis: Multiple interpreters in the same process .. moduleauthor:: Eric Snow .. sectionauthor:: Eric Snow @@ -38,7 +38,7 @@ interfaces on top of the lower level :mod:`!_interpreters` module. .. include:: ../includes/wasm-notavail.rst -Key Details +Key details ----------- Before we dive into examples, there are a small number of details @@ -162,7 +162,7 @@ Exceptions .. XXX Add functions for communicating between interpreters. -Basic Usage +Basic usage ----------- Creating an interpreter and running code in it:: diff --git a/Doc/library/concurrent.rst b/Doc/library/concurrent.rst index 47e84b42cae67b..748c72c733bba2 100644 --- a/Doc/library/concurrent.rst +++ b/Doc/library/concurrent.rst @@ -1,7 +1,7 @@ The :mod:`!concurrent` package ============================== -Currently, there are only two modules in this package: +This package contains the following modules: * :mod:`concurrent.futures` -- Launching parallel tasks * :mod:`concurrent.interpreters` -- Multiple interpreters in the same process diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 384bd02766cbb4..542db7df56c3f7 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -132,10 +132,10 @@ PEP 734: Multiple Interpreters in the Stdlib The CPython runtime supports running multiple copies of Python in the same process simultaneously and has done so for over 20 years. Each of these separate copies is called an "interpreter". -However, the feature has been available only through the C-API. +However, the feature had been available only through the C-API. That limitation is removed in the 3.14 release, -with the new :mod:`concurrent.interpreters` module! +with the new :mod:`concurrent.interpreters` module. There are at least two notable reasons why using multiple interpreters is worth considering: @@ -147,7 +147,7 @@ For some use cases, concurrency in software enables efficiency and can simplify software, at a high level. At the same time, implementing and maintaining all but the simplest concurrency is often a struggle for the human brain. That especially applies to plain threads -(e.g. :mod:`threading`), where all memory is shared between all threads. +(for example, :mod:`threading`), where all memory is shared between all threads. With multiple isolated interpreters, you can take advantage of a class of concurrency models, like CSP or the actor model, that have found @@ -204,11 +204,11 @@ stopped sharing the :term:`GIL`. Likewise, we expect to slowly see libraries on PyPI for high-level abstractions on top of interpreters. Regarding extension modules, work is in progress to update some PyPI -projects, as well as tools like Cython, PyBind11, Nanobind, and PyO3. +projects, as well as tools like Cython, pybind11, nanobind, and PyO3. The steps for isolating an extension module are found at :ref:`isolating-extensions-howto`. Isolating a module has a lot of overlap with what is required to support -:ref:`free-threadeding `, +:ref:`free-threading `, so the ongoing work in the community in that area will help accelerate support for multiple interpreters.