From 204d712aea6698a3f2e47a18f67a5448ff2fcadf Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Sep 2023 19:28:32 -0600 Subject: [PATCH 1/8] Add a test. --- Lib/test/test_interpreters.py | 100 ++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 9c0dac7d6c61fb..2a75286e697244 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -1,5 +1,7 @@ import contextlib +import json import os +import os.path import sys import threading from textwrap import dedent @@ -9,6 +11,7 @@ from test import support from test.support import import_helper from test.support import threading_helper +from test.support import os_helper _interpreters = import_helper.import_module('_xxsubinterpreters') _channels = import_helper.import_module('_xxinterpchannels') from test.support import interpreters @@ -488,6 +491,103 @@ def task(): pass +class StartupTests(TestBase): + + # We want to ensure the initial state of subinterpreters + # matches expectations. + + def create_temp_dir(self): + import tempfile + tmp = tempfile.mkdtemp(prefix='test_interpreters_') + tmp = os.path.realpath(tmp) + self.addCleanup(os_helper.rmtree, tmp) + return tmp + + def write_script(self, *path, text): + filename = os.path.join(*path) + dirname = os.path.dirname(filename) + if dirname: + os.makedirs(dirname, exist_ok=True) + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(dedent(text)) + return filename + + def run_cmd(self, cmd, *, cwd=None): + # This method is inspired by + # EmbeddingTestsMixin.run_embedded_interpreter() in test_embed.py. + import shlex + import subprocess + assert cmd.startswith('python3 '), repr(cmd) + if cmd.startswith('python3 '): + cmd = cmd.replace('python3', sys.executable, 1) + argv = shlex.split(cmd) + proc = subprocess.run( + argv, + cwd=cwd, + capture_output=True, + text=True, + ) + if proc.stderr != '': + # This is a hack until _PyThreadState_MustExit() is fixed. + proc.returncode = 1 + if proc.returncode != 0 and support.verbose: + print(f'--- {cmd} failed ---') + print(f'stdout:\n{proc.stdout}') + print(f'stderr:\n{proc.stderr}') + print('------') + self.assertEqual(proc.returncode, 0) + self.assertEqual(proc.stderr, '') + return proc.stdout + + def test_sys_path_0(self): + # The main interpreter's sys.path[0] should be used by subinterpreters. + + script = ''' + import sys + from test.support import interpreters + + orig = sys.path[0] + + interp = interpreters.create() + interp.run(f"""if True: + import json + import sys + print(json.dumps({{ + 'main': {orig!r}, + 'sub': sys.path[0], + }}, indent=4), flush=True) + """) + ''' + + # / + # pkg/ + # __init__.py + # __main__.py + # script.py + # script.py + cwd = self.create_temp_dir() + self.write_script(cwd, 'pkg', '__init__.py', text='') + self.write_script(cwd, 'pkg', '__main__.py', text=script) + self.write_script(cwd, 'pkg', 'script.py', text=script) + self.write_script(cwd, 'script.py', text=script) + + cases = [ + ('python3 script.py', cwd), + ('python3 -m script', cwd), + ('python3 -m pkg', cwd), + ('python3 -m pkg.script', cwd), + ('python3 -c "import script"', ''), + ] + for cmd, expected in cases: + with self.subTest(cmd): + out = self.run_cmd(cmd, cwd=cwd) + data = json.loads(out) + sp0_main, sp0_sub = data['main'], data['sub'] + self.assertEqual(sp0_sub, sp0_main) + self.assertEqual(sp0_sub, expected) + # XXX Also check them all with the -P cmdline flag? + + class FinalizationTests(TestBase): def test_gh_109793(self): From 1843e11451657f455f4865c678d16ac5d8b65403 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 19 Sep 2023 14:37:22 -0600 Subject: [PATCH 2/8] Add PyConfig.sys_path_0. --- Include/cpython/initconfig.h | 3 +++ Lib/test/test_embed.py | 1 + Python/initconfig.c | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index ee130467824daa..5d7b4e2d929e5b 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -204,6 +204,9 @@ typedef struct PyConfig { wchar_t *run_module; wchar_t *run_filename; + /* --- Set by Py_Main() -------------------------- */ + wchar_t *sys_path_0; + /* --- Private fields ---------------------------- */ // Install importlib? If equals to 0, importlib is not initialized at all. diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 7f1a4e665f3b5d..811514240a4b3c 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -504,6 +504,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'run_command': None, 'run_module': None, 'run_filename': None, + 'sys_path_0': None, '_install_importlib': 1, 'check_hash_pycs_mode': 'default', diff --git a/Python/initconfig.c b/Python/initconfig.c index a0467f51d4834e..85408581aa4493 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -672,6 +672,7 @@ PyConfig_Clear(PyConfig *config) CLEAR(config->exec_prefix); CLEAR(config->base_exec_prefix); CLEAR(config->platlibdir); + CLEAR(config->sys_path_0); CLEAR(config->filesystem_encoding); CLEAR(config->filesystem_errors); @@ -926,6 +927,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) COPY_WSTR_ATTR(exec_prefix); COPY_WSTR_ATTR(base_exec_prefix); COPY_WSTR_ATTR(platlibdir); + COPY_WSTR_ATTR(sys_path_0); COPY_ATTR(site_import); COPY_ATTR(bytes_warning); @@ -1038,6 +1040,7 @@ _PyConfig_AsDict(const PyConfig *config) SET_ITEM_WSTR(exec_prefix); SET_ITEM_WSTR(base_exec_prefix); SET_ITEM_WSTR(platlibdir); + SET_ITEM_WSTR(sys_path_0); SET_ITEM_INT(site_import); SET_ITEM_INT(bytes_warning); SET_ITEM_INT(warn_default_encoding); @@ -1367,6 +1370,7 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) GET_WSTR_OPT(base_prefix); GET_WSTR_OPT(exec_prefix); GET_WSTR_OPT(base_exec_prefix); + GET_WSTR_OPT(sys_path_0); GET_UINT(skip_source_first_line); GET_WSTR_OPT(run_command); @@ -3114,6 +3118,7 @@ _Py_DumpPathConfig(PyThreadState *tstate) PySys_WriteStderr(" import site = %i\n", config->site_import); PySys_WriteStderr(" is in build tree = %i\n", config->_is_python_build); DUMP_CONFIG("stdlib dir", stdlib_dir); + DUMP_CONFIG("sys.path[0]", sys_path_0); #undef DUMP_CONFIG #define DUMP_SYS(NAME) \ From 47278d58441626fa4881a9b41386f1c74b9bd9e5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 19 Sep 2023 14:57:52 -0600 Subject: [PATCH 3/8] Initialize config->sys_path_0 in Py_Main(). --- Lib/test/test_embed.py | 2 ++ Modules/main.c | 38 ++++++++++++++++++++++++++++---------- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 811514240a4b3c..96a5e4dca8d67f 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1132,6 +1132,7 @@ def test_init_run_main(self): 'program_name': './python3', 'run_command': code + '\n', 'parse_argv': 2, + 'sys_path_0': '', } self.check_all_configs("test_init_run_main", config, api=API_PYTHON) @@ -1147,6 +1148,7 @@ def test_init_main(self): 'run_command': code + '\n', 'parse_argv': 2, '_init_main': 0, + 'sys_path_0': '', } self.check_all_configs("test_init_main", config, api=API_PYTHON, diff --git a/Modules/main.c b/Modules/main.c index 7f88c97207475b..027d092a6e2443 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -560,6 +560,11 @@ pymain_run_python(int *exitcode) goto error; } + // XXX Calculate config->sys_path_0 in getpath.py. + // The tricky part is that we can't check the path importers yet + // at that point. + assert(config->sys_path_0 == NULL); + if (config->run_filename != NULL) { /* If filename is a package (ex: directory or ZIP file) which contains __main__.py, main_importer_path is set to filename and will be @@ -575,24 +580,37 @@ pymain_run_python(int *exitcode) // import readline and rlcompleter before script dir is added to sys.path pymain_import_readline(config); + PyObject *path0 = NULL; if (main_importer_path != NULL) { - if (pymain_sys_path_add_path0(interp, main_importer_path) < 0) { - goto error; - } + path0 = Py_NewRef(main_importer_path); } else if (!config->safe_path) { - PyObject *path0 = NULL; int res = _PyPathConfig_ComputeSysPath0(&config->argv, &path0); if (res < 0) { goto error; } - - if (res > 0) { - if (pymain_sys_path_add_path0(interp, path0) < 0) { - Py_DECREF(path0); - goto error; - } + else if (res == 0) { + Py_CLEAR(path0); + } + } + // XXX Apply config->sys_path_0 in init_interp_main(). We have + // to be sure to get readline/rlcompleter imported at the correct time. + if (path0 != NULL) { + wchar_t *wstr = PyUnicode_AsWideCharString(path0, NULL); + if (wstr == NULL) { Py_DECREF(path0); + goto error; + } + config->sys_path_0 = _PyMem_RawWcsdup(wstr); + PyMem_Free(wstr); + if (config->sys_path_0 == NULL) { + Py_DECREF(path0); + goto error; + } + int res = pymain_sys_path_add_path0(interp, path0); + Py_DECREF(path0); + if (res < 0) { + goto error; } } From d39b45b3fcf9f27f11f8755e0a7bd55a9bee745f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 19 Sep 2023 14:58:13 -0600 Subject: [PATCH 4/8] Set sys.path[0] for subinterpreters. --- Python/pylifecycle.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index aec8da10249d21..6e3b869d8ea1bb 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1165,6 +1165,31 @@ init_interp_main(PyThreadState *tstate) return status; } + if (!is_main_interp) { + // The main interpreter is handled in Py_Main(), for now. + if (config->sys_path_0 != NULL) { + PyObject *path0 = PyUnicode_FromWideChar(config->sys_path_0, -1); + if (path0 == NULL) { + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + PyObject *sysdict = interp->sysdict; + if (sysdict == NULL) { + Py_DECREF(path0); + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + PyObject *sys_path = PyDict_GetItemWithError(sysdict, &_Py_ID(path)); + if (sys_path == NULL) { + Py_DECREF(path0); + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + int res = PyList_Insert(sys_path, 0, path0); + Py_DECREF(path0); + if (res) { + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + } + } + if (is_main_interp) { /* Initialize warnings. */ PyObject *warnoptions = PySys_GetObject("warnoptions"); From 8b909df5920c467f54af8e44028caa1c2f0abb90 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 23 Sep 2023 10:51:25 -0600 Subject: [PATCH 5/8] Set sys.path[0] *after* site-import. --- Python/pylifecycle.c | 50 ++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 6e3b869d8ea1bb..300e06d83f2db5 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1165,31 +1165,6 @@ init_interp_main(PyThreadState *tstate) return status; } - if (!is_main_interp) { - // The main interpreter is handled in Py_Main(), for now. - if (config->sys_path_0 != NULL) { - PyObject *path0 = PyUnicode_FromWideChar(config->sys_path_0, -1); - if (path0 == NULL) { - return _PyStatus_ERR("can't initialize sys.path[0]"); - } - PyObject *sysdict = interp->sysdict; - if (sysdict == NULL) { - Py_DECREF(path0); - return _PyStatus_ERR("can't initialize sys.path[0]"); - } - PyObject *sys_path = PyDict_GetItemWithError(sysdict, &_Py_ID(path)); - if (sys_path == NULL) { - Py_DECREF(path0); - return _PyStatus_ERR("can't initialize sys.path[0]"); - } - int res = PyList_Insert(sys_path, 0, path0); - Py_DECREF(path0); - if (res) { - return _PyStatus_ERR("can't initialize sys.path[0]"); - } - } - } - if (is_main_interp) { /* Initialize warnings. */ PyObject *warnoptions = PySys_GetObject("warnoptions"); @@ -1236,6 +1211,31 @@ init_interp_main(PyThreadState *tstate) } } + if (!is_main_interp) { + // The main interpreter is handled in Py_Main(), for now. + if (config->sys_path_0 != NULL) { + PyObject *path0 = PyUnicode_FromWideChar(config->sys_path_0, -1); + if (path0 == NULL) { + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + PyObject *sysdict = interp->sysdict; + if (sysdict == NULL) { + Py_DECREF(path0); + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + PyObject *sys_path = PyDict_GetItemWithError(sysdict, &_Py_ID(path)); + if (sys_path == NULL) { + Py_DECREF(path0); + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + int res = PyList_Insert(sys_path, 0, path0); + Py_DECREF(path0); + if (res) { + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + } + } + assert(!_PyErr_Occurred(tstate)); return _PyStatus_OK(); From e36e0d2a4c651e9636eab3743edaa0a801bebf5e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 27 Sep 2023 18:03:51 -0600 Subject: [PATCH 6/8] Add a NEWS entry. --- .../2023-09-27-18-01-06.gh-issue-109853.coQQiL.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-27-18-01-06.gh-issue-109853.coQQiL.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-27-18-01-06.gh-issue-109853.coQQiL.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-27-18-01-06.gh-issue-109853.coQQiL.rst new file mode 100644 index 00000000000000..45de3ba8877b01 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-27-18-01-06.gh-issue-109853.coQQiL.rst @@ -0,0 +1 @@ +``sys.path[0]`` is now set correctly for subinterpreters. From b3501e70996e634b7fa4127246249c70c146b465 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 28 Sep 2023 11:50:50 -0600 Subject: [PATCH 7/8] Adjust the test. --- Lib/test/test_interpreters.py | 107 +++++++++++++++++++++++++--------- 1 file changed, 79 insertions(+), 28 deletions(-) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 2a75286e697244..9cd71e519036c3 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -496,6 +496,52 @@ class StartupTests(TestBase): # We want to ensure the initial state of subinterpreters # matches expectations. + _subtest_count = 0 + + @contextlib.contextmanager + def subTest(self, *args): + with super().subTest(*args) as ctx: + self._subtest_count += 1 + try: + yield ctx + finally: + if self._debugged_in_subtest: + if self._subtest_count == 1: + # The first subtest adds a leading newline, so we + # compensate here by not printing a trailing newline. + print('### end subtest debug ###', end='') + else: + print('### end subtest debug ###') + self._debugged_in_subtest = False + + def debug(self, msg, *, header=None): + if header: + self._debug(f'--- {header} ---') + if msg: + if msg.endswith(os.linesep): + self._debug(msg[:-len(os.linesep)]) + else: + self._debug(msg) + self._debug('') + self._debug('------') + else: + self._debug(msg) + + _debugged = False + _debugged_in_subtest = False + def _debug(self, msg): + if not self._debugged: + print() + self._debugged = True + if self._subtest is not None: + if True: + if not self._debugged_in_subtest: + self._debugged_in_subtest = True + print('### start subtest debug ###') + print(msg) + else: + print(msg) + def create_temp_dir(self): import tempfile tmp = tempfile.mkdtemp(prefix='test_interpreters_') @@ -512,36 +558,42 @@ def write_script(self, *path, text): outfile.write(dedent(text)) return filename - def run_cmd(self, cmd, *, cwd=None): + @support.requires_subprocess() + def run_python(self, argv, *, cwd=None): # This method is inspired by # EmbeddingTestsMixin.run_embedded_interpreter() in test_embed.py. import shlex import subprocess - assert cmd.startswith('python3 '), repr(cmd) - if cmd.startswith('python3 '): - cmd = cmd.replace('python3', sys.executable, 1) - argv = shlex.split(cmd) - proc = subprocess.run( - argv, - cwd=cwd, - capture_output=True, - text=True, - ) - if proc.stderr != '': - # This is a hack until _PyThreadState_MustExit() is fixed. - proc.returncode = 1 + if isinstance(argv, str): + argv = shlex.split(argv) + argv = [sys.executable, *argv] + try: + proc = subprocess.run( + argv, + cwd=cwd, + capture_output=True, + text=True, + ) + except Exception as exc: + self.debug(f'# cmd: {shlex.join(argv)}') + if isinstance(exc, FileNotFoundError) and not exc.filename: + if os.path.exists(argv[0]): + exists = 'exists' + else: + exists = 'does not exist' + self.debug(f'{argv[0]} {exists}') + raise # re-raise + assert proc.stderr == '' or proc.returncode != 0, proc.stderr if proc.returncode != 0 and support.verbose: - print(f'--- {cmd} failed ---') - print(f'stdout:\n{proc.stdout}') - print(f'stderr:\n{proc.stderr}') - print('------') + self.debug(f'# python3 {shlex.join(argv[1:])} failed:') + self.debug(proc.stdout, header='stdout') + self.debug(proc.stderr, header='stderr') self.assertEqual(proc.returncode, 0) self.assertEqual(proc.stderr, '') return proc.stdout def test_sys_path_0(self): # The main interpreter's sys.path[0] should be used by subinterpreters. - script = ''' import sys from test.support import interpreters @@ -558,7 +610,6 @@ def test_sys_path_0(self): }}, indent=4), flush=True) """) ''' - # / # pkg/ # __init__.py @@ -572,15 +623,15 @@ def test_sys_path_0(self): self.write_script(cwd, 'script.py', text=script) cases = [ - ('python3 script.py', cwd), - ('python3 -m script', cwd), - ('python3 -m pkg', cwd), - ('python3 -m pkg.script', cwd), - ('python3 -c "import script"', ''), + ('script.py', cwd), + ('-m script', cwd), + ('-m pkg', cwd), + ('-m pkg.script', cwd), + ('-c "import script"', ''), ] - for cmd, expected in cases: - with self.subTest(cmd): - out = self.run_cmd(cmd, cwd=cwd) + for argv, expected in cases: + with self.subTest(f'python3 {argv}'): + out = self.run_python(argv, cwd=cwd) data = json.loads(out) sp0_main, sp0_sub = data['main'], data['sub'] self.assertEqual(sp0_sub, sp0_main) From de7237bc53b47c45e084e22ce6467149f007f02d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Oct 2023 13:08:03 -0600 Subject: [PATCH 8/8] Use the right member type. --- Python/initconfig.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index 5558780dc97065..6b76b4dc681b74 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -97,7 +97,7 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(pythonpath_env, WSTR_OPT), SPEC(home, WSTR_OPT), SPEC(platlibdir, WSTR), - SPEC(sys_path_0, WSTR), + SPEC(sys_path_0, WSTR_OPT), SPEC(module_search_paths_set, UINT), SPEC(module_search_paths, WSTR_LIST), SPEC(stdlib_dir, WSTR_OPT),