From 3d7ac867a799fe8bf1698e09848f046fe6797cc7 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 11 Jan 2023 16:28:44 +0000 Subject: [PATCH 1/6] gh-100320: Ensure PythonPath registry key is read when the executable is not in the expected location. --- .../2023-01-11-16-28-09.gh-issue-100320.2DU2it.rst | 3 +++ Modules/getpath.py | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-01-11-16-28-09.gh-issue-100320.2DU2it.rst diff --git a/Misc/NEWS.d/next/Windows/2023-01-11-16-28-09.gh-issue-100320.2DU2it.rst b/Misc/NEWS.d/next/Windows/2023-01-11-16-28-09.gh-issue-100320.2DU2it.rst new file mode 100644 index 00000000000000..c206fc8520a5d9 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-01-11-16-28-09.gh-issue-100320.2DU2it.rst @@ -0,0 +1,3 @@ +Ensures the ``PythonPath`` registry key from an install is used when +launching from a different copy of Python that relies on an existing install +to provide a copy of its modules and standard library. diff --git a/Modules/getpath.py b/Modules/getpath.py index ab0d2dc0636ad4..2d0d89a0c878a3 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -701,8 +701,14 @@ def search_up(prefix, *landmarks, test=isfile): except OSError: break if isinstance(v, str): - pythonpath.append(v) + pythonpath.extend(v.split(DELIM)) i += 1 + # Paths from the core key get appended last, but only + # when home was not set and we aren't in a build dir + if not home_was_set and not build_prefix: + v = winreg.QueryValue(key, None) + if isinstance(v, str): + pythonpath.extend(v.split(DELIM)) finally: winreg.CloseKey(key) except OSError: From 13d0aeddc28601289e74a1984f9a0cc75f62ff54 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 11 Jan 2023 20:32:29 +0000 Subject: [PATCH 2/6] Revise calculation and ordering of platstdlib_dir --- Lib/test/test_getpath.py | 20 ++++++++++++-------- Modules/getpath.py | 32 ++++++++++++++++++++------------ 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index bdcf4a37191682..b9cbe1d92c436f 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -37,8 +37,9 @@ def test_normal_win32(self): module_search_paths_set=1, module_search_paths=[ r"C:\Python\python98.zip", - r"C:\Python\Lib", r"C:\Python\DLLs", + r"C:\Python\Lib", + r"C:\Python", ], ) actual = getpath(ns, expected) @@ -63,8 +64,8 @@ def test_buildtree_win32(self): module_search_paths_set=1, module_search_paths=[ r"C:\CPython\PCbuild\amd64\python98.zip", - r"C:\CPython\Lib", r"C:\CPython\PCbuild\amd64", + r"C:\CPython\Lib", ], ) actual = getpath(ns, expected) @@ -133,8 +134,9 @@ def test_registry_win32(self): r"C:\Python\python98.zip", "path1-dir", # should not contain not-subdirs - r"C:\Python\Lib", r"C:\Python\DLLs", + r"C:\Python\Lib", + r"C:\Python", ], ) actual = getpath(ns, expected) @@ -147,8 +149,9 @@ def test_registry_win32(self): module_search_paths_set=1, module_search_paths=[ r"C:\Python\python98.zip", - r"C:\Python\Lib", r"C:\Python\DLLs", + r"C:\Python\Lib", + r"C:\Python", ], ) actual = getpath(ns, expected) @@ -173,8 +176,9 @@ def test_symlink_normal_win32(self): module_search_paths_set=1, module_search_paths=[ r"C:\Python\python98.zip", - r"C:\Python\Lib", r"C:\Python\DLLs", + r"C:\Python\Lib", + r"C:\Python", ], ) actual = getpath(ns, expected) @@ -201,8 +205,8 @@ def test_symlink_buildtree_win32(self): module_search_paths_set=1, module_search_paths=[ r"C:\CPython\PCbuild\amd64\python98.zip", - r"C:\CPython\Lib", r"C:\CPython\PCbuild\amd64", + r"C:\CPython\Lib", ], ) actual = getpath(ns, expected) @@ -231,8 +235,8 @@ def test_buildtree_pythonhome_win32(self): module_search_paths_set=1, module_search_paths=[ r"C:\Out\python98.zip", - r"C:\CPython\Lib", r"C:\Out", + r"C:\CPython\Lib", ], ) actual = getpath(ns, expected) @@ -254,8 +258,8 @@ def test_no_dlls_win32(self): module_search_paths_set=1, module_search_paths=[ r"C:\Python\python98.zip", - r"C:\Python\Lib", r"C:\Python", + r"C:\Python\Lib", ], ) actual = getpath(ns, expected) diff --git a/Modules/getpath.py b/Modules/getpath.py index 2d0d89a0c878a3..dde608a2220cd7 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -600,22 +600,26 @@ def search_up(prefix, *landmarks, test=isfile): if executable_dir: if os_name == 'nt': # QUIRK: For compatibility and security, do not search for DLLs - # directory. The fallback below will cover it - exec_prefix = executable_dir + # directory. + if isdir(joinpath(executable_dir, PLATSTDLIB_LANDMARK)): + exec_prefix = executable_dir else: exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir) if not exec_prefix and EXEC_PREFIX: exec_prefix = EXEC_PREFIX - if not exec_prefix or not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)): + if not exec_prefix: if os_name == 'nt': # QUIRK: If DLLs is missing on Windows, don't warn, just assume # that it's all the same as prefix. - # gh-98790: We set platstdlib_dir here to avoid adding "DLLs" into - # sys.path when it doesn't exist, which would give site-packages - # precedence over executable_dir, which is *probably* where our PYDs - # live. Ideally, whoever changes our layout will tell us what the - # layout is, but in the past this worked, so it should keep working. - platstdlib_dir = exec_prefix = prefix + exec_prefix = prefix + if not platstdlib_dir: + # gh-98790: We set platstdlib_dir here to avoid adding "DLLs" into + # sys.path when it doesn't exist in the platstdlib place, which + # would give site-packages precedence over executable_dir where our + # PYDs *probably* live. Ideally, whoever changes our layout will tell + # us what the layout is, but in the past this worked, so it should + # keep working. + platstdlib_dir = exec_prefix else: warn('Could not find platform dependent libraries ') @@ -720,13 +724,17 @@ def search_up(prefix, *landmarks, test=isfile): pythonpath.append(joinpath(prefix, p)) # Then add stdlib_dir and platstdlib_dir - if os_name == 'nt' and venv_prefix: - # QUIRK: Windows generates paths differently in a venv + if os_name == 'nt': + # QUIRK: Windows generates paths differently if platstdlib_dir: pythonpath.append(platstdlib_dir) if stdlib_dir: pythonpath.append(stdlib_dir) - if executable_dir not in pythonpath: + if executable_dir and executable_dir not in pythonpath: + # QUIRK: the executable directory is on sys.path + # We keep it low priority, so that properly installed modules are + # found first. It may be earlier in the order if we found some + # reason to put it there. pythonpath.append(executable_dir) else: if stdlib_dir: From 8068122a2f643ef751f7872cf8e1df6d16134c92 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 11 Jan 2023 21:54:08 +0000 Subject: [PATCH 3/6] Fix test_embed tests --- Lib/test/test_embed.py | 17 ++++++++++------- Modules/getpath.py | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 2dda7ccf7bf80c..4a67d2882fbb95 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -714,8 +714,10 @@ def check_config(self, configs, expected): if MS_WINDOWS: value = config.get(key := 'program_name') if value and isinstance(value, str): - ext = '_d.exe' if debug_build(sys.executable) else '.exe' - config[key] = value[:len(value.lower().removesuffix(ext))] + value = value[:len(value.lower().removesuffix('.exe'))] + if debug_build(sys.executable): + value = value[:len(value.lower().removesuffix('_d'))] + config[key] = value for key, value in list(expected.items()): if value is self.IGNORE_CONFIG: config.pop(key, None) @@ -1292,7 +1294,7 @@ def test_init_setpythonhome(self): stdlib = os.path.join(home, "Lib") # Because we are specifying 'home', module search paths # are fairly static - expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')] + expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' stdlib = os.path.join(home, sys.platlibdir, f'python{version}') @@ -1333,7 +1335,8 @@ def test_init_is_python_build_with_home(self): stdlib = os.path.join(home, "Lib") # Because we are specifying 'home', module search paths # are fairly static - expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')] + #expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] + expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' stdlib = os.path.join(home, sys.platlibdir, f'python{version}') @@ -1361,7 +1364,7 @@ def test_init_is_python_build_with_home(self): config['_is_python_build'] = 1 exedir = os.path.dirname(sys.executable) with open(os.path.join(exedir, 'pybuilddir.txt'), encoding='utf8') as f: - expected_paths[2] = os.path.normpath( + expected_paths[1] = os.path.normpath( os.path.join(exedir, f'{f.read()}\n$'.splitlines()[0])) if not MS_WINDOWS: # PREFIX (default) is set when running in build directory @@ -1438,8 +1441,8 @@ def test_init_pybuilddir_win32(self): module_search_paths = self.module_search_paths() module_search_paths[-3] = os.path.join(tmpdir, os.path.basename(module_search_paths[-3])) - module_search_paths[-2] = stdlibdir - module_search_paths[-1] = tmpdir + module_search_paths[-2] = tmpdir + module_search_paths[-1] = stdlibdir executable = self.test_exe config = { diff --git a/Modules/getpath.py b/Modules/getpath.py index dde608a2220cd7..0196774d9cb6c3 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -709,7 +709,7 @@ def search_up(prefix, *landmarks, test=isfile): i += 1 # Paths from the core key get appended last, but only # when home was not set and we aren't in a build dir - if not home_was_set and not build_prefix: + if not home_was_set and not venv_prefix and not build_prefix: v = winreg.QueryValue(key, None) if isinstance(v, str): pythonpath.extend(v.split(DELIM)) From ab9c9ffbc62330a33be0751b9281cb06d8457843 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 11 Jan 2023 23:07:32 +0000 Subject: [PATCH 4/6] Fix test --- Lib/test/test_embed.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 4a67d2882fbb95..acffe234ecff6d 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1335,7 +1335,6 @@ def test_init_is_python_build_with_home(self): stdlib = os.path.join(home, "Lib") # Because we are specifying 'home', module search paths # are fairly static - #expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' @@ -1364,7 +1363,7 @@ def test_init_is_python_build_with_home(self): config['_is_python_build'] = 1 exedir = os.path.dirname(sys.executable) with open(os.path.join(exedir, 'pybuilddir.txt'), encoding='utf8') as f: - expected_paths[1] = os.path.normpath( + expected_paths[1 if MS_WINDOWS else 2] = os.path.normpath( os.path.join(exedir, f'{f.read()}\n$'.splitlines()[0])) if not MS_WINDOWS: # PREFIX (default) is set when running in build directory From 2c70610b847fd09f56007bfced3c266cb22e6852 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 12 Jan 2023 17:40:16 +0000 Subject: [PATCH 5/6] Simplify and fix path logic --- Modules/getpath.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Modules/getpath.py b/Modules/getpath.py index 0196774d9cb6c3..f11af9a2f39984 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -599,15 +599,13 @@ def search_up(prefix, *landmarks, test=isfile): if PLATSTDLIB_LANDMARK and not exec_prefix: if executable_dir: if os_name == 'nt': - # QUIRK: For compatibility and security, do not search for DLLs - # directory. - if isdir(joinpath(executable_dir, PLATSTDLIB_LANDMARK)): - exec_prefix = executable_dir + # QUIRK: Windows always assumed these were the same + exec_prefix = prefix else: exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir) if not exec_prefix and EXEC_PREFIX: exec_prefix = EXEC_PREFIX - if not exec_prefix: + if not exec_prefix or not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)): if os_name == 'nt': # QUIRK: If DLLs is missing on Windows, don't warn, just assume # that it's all the same as prefix. From 7f586e5f3e594c9040616d0da2856b24503379f6 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 13 Jan 2023 12:00:09 +0000 Subject: [PATCH 6/6] Logic tidying and better comments --- Modules/getpath.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Modules/getpath.py b/Modules/getpath.py index f11af9a2f39984..6a0883878778a5 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -597,23 +597,23 @@ def search_up(prefix, *landmarks, test=isfile): # Detect exec_prefix by searching from executable for the platstdlib_dir if PLATSTDLIB_LANDMARK and not exec_prefix: - if executable_dir: - if os_name == 'nt': - # QUIRK: Windows always assumed these were the same - exec_prefix = prefix - else: - exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir) + if os_name == 'nt': + # QUIRK: Windows always assumed these were the same + # gh-100320: Our PYDs are assumed to be relative to the Lib directory + # (that is, prefix) rather than the executable (that is, executable_dir) + exec_prefix = prefix + if not exec_prefix and executable_dir: + exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir) if not exec_prefix and EXEC_PREFIX: exec_prefix = EXEC_PREFIX if not exec_prefix or not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)): if os_name == 'nt': # QUIRK: If DLLs is missing on Windows, don't warn, just assume - # that it's all the same as prefix. - exec_prefix = prefix + # that they're in exec_prefix if not platstdlib_dir: # gh-98790: We set platstdlib_dir here to avoid adding "DLLs" into # sys.path when it doesn't exist in the platstdlib place, which - # would give site-packages precedence over executable_dir where our + # would give Lib packages precedence over executable_dir where our # PYDs *probably* live. Ideally, whoever changes our layout will tell # us what the layout is, but in the past this worked, so it should # keep working.