From 02320d5821f4a53571fa712e5e321ba48b4dd430 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sat, 26 Jul 2025 23:58:40 +0200 Subject: [PATCH 1/4] Update `{site,sysconfig}.py` from 3.13.7 --- Lib/site.py | 229 +++++--- Lib/{sysconfig.py => sysconfig/__init__.py} | 554 ++++++++------------ Lib/sysconfig/__main__.py | 248 +++++++++ Lib/test/test_site.py | 60 ++- Lib/test/test_sysconfig.py | 256 +++++++-- vm/src/stdlib/mod.rs | 2 + vm/src/stdlib/sysconfig.rs | 14 + 7 files changed, 901 insertions(+), 462 deletions(-) rename Lib/{sysconfig.py => sysconfig/__init__.py} (53%) create mode 100644 Lib/sysconfig/__main__.py create mode 100644 vm/src/stdlib/sysconfig.rs diff --git a/Lib/site.py b/Lib/site.py index acc8481b13..2983ca7154 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -75,6 +75,7 @@ import _sitebuiltins import io import stat +import errno # Prefixes for site-packages; add additional prefixes like /usr/local here PREFIXES = [sys.prefix, sys.exec_prefix] @@ -179,35 +180,46 @@ def addpackage(sitedir, name, known_paths): return _trace(f"Processing .pth file: {fullname!r}") try: - # locale encoding is not ideal especially on Windows. But we have used - # it for a long time. setuptools uses the locale encoding too. - f = io.TextIOWrapper(io.open_code(fullname), encoding="locale") + with io.open_code(fullname) as f: + pth_content = f.read() except OSError: return - with f: - for n, line in enumerate(f): - if line.startswith("#"): - continue - if line.strip() == "": + + try: + # Accept BOM markers in .pth files as we do in source files + # (Windows PowerShell 5.1 makes it hard to emit UTF-8 files without a BOM) + pth_content = pth_content.decode("utf-8-sig") + except UnicodeDecodeError: + # Fallback to locale encoding for backward compatibility. + # We will deprecate this fallback in the future. + import locale + pth_content = pth_content.decode(locale.getencoding()) + _trace(f"Cannot read {fullname!r} as UTF-8. " + f"Using fallback encoding {locale.getencoding()!r}") + + for n, line in enumerate(pth_content.splitlines(), 1): + if line.startswith("#"): + continue + if line.strip() == "": + continue + try: + if line.startswith(("import ", "import\t")): + exec(line) continue - try: - if line.startswith(("import ", "import\t")): - exec(line) - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if not dircase in known_paths and os.path.exists(dir): - sys.path.append(dir) - known_paths.add(dircase) - except Exception as exc: - print("Error processing line {:d} of {}:\n".format(n+1, fullname), - file=sys.stderr) - import traceback - for record in traceback.format_exception(exc): - for line in record.splitlines(): - print(' '+line, file=sys.stderr) - print("\nRemainder of file ignored", file=sys.stderr) - break + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if dircase not in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + except Exception as exc: + print(f"Error processing line {n:d} of {fullname}:\n", + file=sys.stderr) + import traceback + for record in traceback.format_exception(exc): + for line in record.splitlines(): + print(' '+line, file=sys.stderr) + print("\nRemainder of file ignored", file=sys.stderr) + break if reset: known_paths = None return known_paths @@ -270,14 +282,18 @@ def check_enableusersite(): # # See https://bugs.python.org/issue29585 +# Copy of sysconfig._get_implementation() +def _get_implementation(): + return 'RustPython' # XXX: RustPython; for site-packages + # Copy of sysconfig._getuserbase() def _getuserbase(): env_base = os.environ.get("PYTHONUSERBASE", None) if env_base: return env_base - # Emscripten, VxWorks, and WASI have no home directories - if sys.platform in {"emscripten", "vxworks", "wasi"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories + if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -285,8 +301,7 @@ def joinuser(*args): if os.name == "nt": base = os.environ.get("APPDATA") or "~" - # XXX: RUSTPYTHON; please keep this change for site-packages - return joinuser(base, "RustPython") + return joinuser(base, _get_implementation()) if sys.platform == "darwin" and sys._framework: return joinuser("~", "Library", sys._framework, @@ -298,15 +313,22 @@ def joinuser(*args): # Same to sysconfig.get_path('purelib', os.name+'_user') def _get_path(userbase): version = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' + implementation = _get_implementation() + implementation_lower = implementation.lower() if os.name == 'nt': ver_nodot = sys.winver.replace('.', '') - return f'{userbase}\\RustPython{ver_nodot}\\site-packages' + return f'{userbase}\\{implementation}{ver_nodot}\\site-packages' if sys.platform == 'darwin' and sys._framework: - return f'{userbase}/lib/rustpython/site-packages' + return f'{userbase}/lib/{implementation_lower}/site-packages' - return f'{userbase}/lib/rustpython{version[0]}.{version[1]}/site-packages' + # XXX: RUSTPYTHON + return f'{userbase}/lib/rustpython{version[0]}.{version[1]}{abi_thread}/site-packages' def getuserbase(): @@ -372,6 +394,12 @@ def getsitepackages(prefixes=None): continue seen.add(prefix) + implementation = _get_implementation().lower() + ver = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' if os.sep == '/': libdirs = [sys.platlibdir] if sys.platlibdir != "lib": @@ -379,8 +407,7 @@ def getsitepackages(prefixes=None): for libdir in libdirs: path = os.path.join(prefix, libdir, - # XXX: RUSTPYTHON; please keep this change for site-packages - "rustpython%d.%d" % sys.version_info[:2], + f"{implementation}{ver[0]}.{ver[1]}{abi_thread}", "site-packages") sitepackages.append(path) else: @@ -417,8 +444,9 @@ def setcopyright(): """Set 'copyright' and 'credits' in builtins""" builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright) builtins.credits = _sitebuiltins._Printer("credits", """\ - Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands - for supporting Python development. See www.python.org for more information.""") + Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software + Foundation, and a cast of thousands for supporting Python + development. See www.python.org for more information.""") files, dirs = [], [] # Not all modules are required to have a __file__ attribute. See # PEP 420 for more details. @@ -437,27 +465,76 @@ def setcopyright(): def sethelper(): builtins.help = _sitebuiltins._Helper() + +def gethistoryfile(): + """Check if the PYTHON_HISTORY environment variable is set and define + it as the .python_history file. If PYTHON_HISTORY is not set, use the + default .python_history file. + """ + if not sys.flags.ignore_environment: + history = os.environ.get("PYTHON_HISTORY") + if history: + return history + return os.path.join(os.path.expanduser('~'), + '.python_history') + + def enablerlcompleter(): """Enable default readline configuration on interactive prompts, by registering a sys.__interactivehook__. + """ + sys.__interactivehook__ = register_readline + + +def register_readline(): + """Configure readline completion on interactive prompts. If the readline module can be imported, the hook will set the Tab key as completion key and register ~/.python_history as history file. This can be overridden in the sitecustomize or usercustomize module, or in a PYTHONSTARTUP file. """ - def register_readline(): - import atexit + if not sys.flags.ignore_environment: + PYTHON_BASIC_REPL = os.getenv("PYTHON_BASIC_REPL") + else: + PYTHON_BASIC_REPL = False + + import atexit + + try: try: import readline - import rlcompleter except ImportError: - return + readline = None + else: + import rlcompleter # noqa: F401 + except ImportError: + return + try: + if PYTHON_BASIC_REPL: + CAN_USE_PYREPL = False + else: + original_path = sys.path + sys.path = [p for p in original_path if p != ''] + try: + import _pyrepl.readline + if os.name == "nt": + import _pyrepl.windows_console + console_errors = (_pyrepl.windows_console._error,) + else: + import _pyrepl.unix_console + console_errors = _pyrepl.unix_console._error + from _pyrepl.main import CAN_USE_PYREPL + finally: + sys.path = original_path + except ImportError: + return + + if readline is not None: # Reading the initialization (config) file may not be enough to set a # completion key, so we set one first and then read the file. - readline_doc = getattr(readline, '__doc__', '') - if readline_doc is not None and 'libedit' in readline_doc: + if readline.backend == 'editline': readline.parse_and_bind('bind ^I rl_complete') else: readline.parse_and_bind('tab: complete') @@ -471,30 +548,44 @@ def register_readline(): # want to ignore the exception. pass - if readline.get_current_history_length() == 0: - # If no history was loaded, default to .python_history. - # The guard is necessary to avoid doubling history size at - # each interpreter exit when readline was already configured - # through a PYTHONSTARTUP hook, see: - # http://bugs.python.org/issue5845#msg198636 - history = os.path.join(os.path.expanduser('~'), - '.python_history') + if readline is None or readline.get_current_history_length() == 0: + # If no history was loaded, default to .python_history, + # or PYTHON_HISTORY. + # The guard is necessary to avoid doubling history size at + # each interpreter exit when readline was already configured + # through a PYTHONSTARTUP hook, see: + # http://bugs.python.org/issue5845#msg198636 + history = gethistoryfile() + + if CAN_USE_PYREPL: + readline_module = _pyrepl.readline + exceptions = (OSError, *console_errors) + else: + if readline is None: + return + readline_module = readline + exceptions = OSError + + try: + readline_module.read_history_file(history) + except exceptions: + pass + + def write_history(): try: - readline.read_history_file(history) - except OSError: + readline_module.write_history_file(history) + except (FileNotFoundError, PermissionError): + # home directory does not exist or is not writable + # https://bugs.python.org/issue19891 pass + except OSError: + if errno.EROFS: + pass # gh-128066: read-only file system + else: + raise - def write_history(): - try: - readline.write_history_file(history) - except OSError: - # bpo-19891, bpo-41193: Home directory does not exist - # or is not writable, or the filesystem is read-only. - pass - - atexit.register(write_history) + atexit.register(write_history) - sys.__interactivehook__ = register_readline def venv(known_paths): global PREFIXES, ENABLE_USER_SITE @@ -679,17 +770,5 @@ def exists(path): print(textwrap.dedent(help % (sys.argv[0], os.pathsep))) sys.exit(10) -def gethistoryfile(): - """Check if the PYTHON_HISTORY environment variable is set and define - it as the .python_history file. If PYTHON_HISTORY is not set, use the - default .python_history file. - """ - if not sys.flags.ignore_environment: - history = os.environ.get("PYTHON_HISTORY") - if history: - return history - return os.path.join(os.path.expanduser('~'), - '.python_history') - if __name__ == '__main__': _script() diff --git a/Lib/sysconfig.py b/Lib/sysconfig/__init__.py similarity index 53% rename from Lib/sysconfig.py rename to Lib/sysconfig/__init__.py index 9999d6bbd5..3ad9df603f 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig/__init__.py @@ -1,10 +1,9 @@ -# XXX: RUSTPYTHON; Trick to make sysconfig work as RustPython -exec(r''' """Access to Python's configuration information.""" import os import sys -from os.path import pardir, realpath +import threading +from os.path import realpath __all__ = [ 'get_config_h_filename', @@ -22,29 +21,30 @@ # Keys for get_config_var() that are never converted to Python integers. _ALWAYS_STR = { + 'IPHONEOS_DEPLOYMENT_TARGET', 'MACOSX_DEPLOYMENT_TARGET', } _INSTALL_SCHEMES = { 'posix_prefix': { - 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', 'include': - '{installed_base}/include/python{py_version_short}{abiflags}', + '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': - '{installed_platbase}/include/python{py_version_short}{abiflags}', + '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}', 'scripts': '{base}/bin', 'data': '{base}', }, 'posix_home': { - 'stdlib': '{installed_base}/lib/python', - 'platstdlib': '{base}/lib/python', - 'purelib': '{base}/lib/python', - 'platlib': '{base}/lib/python', - 'include': '{installed_base}/include/python', - 'platinclude': '{installed_base}/include/python', + 'stdlib': '{installed_base}/lib/{implementation_lower}', + 'platstdlib': '{base}/lib/{implementation_lower}', + 'purelib': '{base}/lib/{implementation_lower}', + 'platlib': '{base}/lib/{implementation_lower}', + 'include': '{installed_base}/include/{implementation_lower}', + 'platinclude': '{installed_base}/include/{implementation_lower}', 'scripts': '{base}/bin', 'data': '{base}', }, @@ -58,6 +58,7 @@ 'scripts': '{base}/Scripts', 'data': '{base}', }, + # Downstream distributors can overwrite the default install scheme. # This is done to support downstream modifications where distributors change # the installation layout (eg. different site-packages directory). @@ -76,14 +77,14 @@ # Downstream distributors who patch posix_prefix/nt scheme are encouraged to # leave the following schemes unchanged 'posix_venv': { - 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', 'include': - '{installed_base}/include/python{py_version_short}{abiflags}', + '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': - '{installed_platbase}/include/python{py_version_short}{abiflags}', + '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}', 'scripts': '{base}/bin', 'data': '{base}', }, @@ -105,6 +106,8 @@ else: _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['posix_venv'] +def _get_implementation(): + return 'RustPython' # XXX: For site-packages # NOTE: site.py has copy of this function. # Sync it when modify this function. @@ -113,8 +116,8 @@ def _getuserbase(): if env_base: return env_base - # Emscripten, VxWorks, and WASI have no home directories - if sys.platform in {"emscripten", "vxworks", "wasi"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories + if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -122,7 +125,7 @@ def joinuser(*args): if os.name == "nt": base = os.environ.get("APPDATA") or "~" - return joinuser(base, "Python") + return joinuser(base, _get_implementation()) if sys.platform == "darwin" and sys._framework: return joinuser("~", "Library", sys._framework, @@ -136,29 +139,29 @@ def joinuser(*args): _INSTALL_SCHEMES |= { # NOTE: When modifying "purelib" scheme, update site._get_path() too. 'nt_user': { - 'stdlib': '{userbase}/Python{py_version_nodot_plat}', - 'platstdlib': '{userbase}/Python{py_version_nodot_plat}', - 'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'include': '{userbase}/Python{py_version_nodot_plat}/Include', - 'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts', + 'stdlib': '{userbase}/{implementation}{py_version_nodot_plat}', + 'platstdlib': '{userbase}/{implementation}{py_version_nodot_plat}', + 'purelib': '{userbase}/{implementation}{py_version_nodot_plat}/site-packages', + 'platlib': '{userbase}/{implementation}{py_version_nodot_plat}/site-packages', + 'include': '{userbase}/{implementation}{py_version_nodot_plat}/Include', + 'scripts': '{userbase}/{implementation}{py_version_nodot_plat}/Scripts', 'data': '{userbase}', }, 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}{abi_thread}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, 'osx_framework_user': { - 'stdlib': '{userbase}/lib/python', - 'platstdlib': '{userbase}/lib/python', - 'purelib': '{userbase}/lib/python/site-packages', - 'platlib': '{userbase}/lib/python/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/lib/{implementation_lower}', + 'platstdlib': '{userbase}/lib/{implementation_lower}', + 'purelib': '{userbase}/lib/{implementation_lower}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, @@ -170,19 +173,15 @@ def joinuser(*args): _PY_VERSION = sys.version.split()[0] _PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}' _PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}' -_PREFIX = os.path.normpath(sys.prefix) _BASE_PREFIX = os.path.normpath(sys.base_prefix) -_EXEC_PREFIX = os.path.normpath(sys.exec_prefix) _BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) +# Mutex guarding initialization of _CONFIG_VARS. +_CONFIG_VARS_LOCK = threading.RLock() _CONFIG_VARS = None +# True iff _CONFIG_VARS has been fully initialized. +_CONFIG_VARS_INITIALIZED = False _USER_BASE = None -# Regexes needed for parsing Makefile (and similar syntaxes, -# like old-style Setup files). -_variable_rx = r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)" -_findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" -_findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" - def _safe_realpath(path): try: @@ -221,8 +220,15 @@ def _safe_realpath(path): def is_python_build(check_home=None): if check_home is not None: import warnings - warnings.warn("check_home argument is deprecated and ignored.", - DeprecationWarning, stacklevel=2) + warnings.warn( + ( + 'The check_home argument of sysconfig.is_python_build is ' + 'deprecated and its value is ignored. ' + 'It will be removed in Python 3.15.' + ), + DeprecationWarning, + stacklevel=2, + ) for fn in ("Setup", "Setup.local"): if os.path.isfile(os.path.join(_PROJECT_BASE, "Modules", fn)): return True @@ -291,6 +297,7 @@ def _get_preferred_schemes(): 'home': 'posix_home', 'user': 'osx_framework_user', } + return { 'prefix': 'posix_prefix', 'home': 'posix_home', @@ -314,134 +321,6 @@ def get_default_scheme(): return get_preferred_scheme('prefix') -def _parse_makefile(filename, vars=None, keep_unresolved=True): - """Parse a Makefile-style file. - - A dictionary containing name/value pairs is returned. If an - optional dictionary is passed in as the second argument, it is - used instead of a new dictionary. - """ - import re - - if vars is None: - vars = {} - done = {} - notdone = {} - - with open(filename, encoding=sys.getfilesystemencoding(), - errors="surrogateescape") as f: - lines = f.readlines() - - for line in lines: - if line.startswith('#') or line.strip() == '': - continue - m = re.match(_variable_rx, line) - if m: - n, v = m.group(1, 2) - v = v.strip() - # `$$' is a literal `$' in make - tmpv = v.replace('$$', '') - - if "$" in tmpv: - notdone[n] = v - else: - try: - if n in _ALWAYS_STR: - raise ValueError - - v = int(v) - except ValueError: - # insert literal `$' - done[n] = v.replace('$$', '$') - else: - done[n] = v - - # do variable interpolation here - variables = list(notdone.keys()) - - # Variables with a 'PY_' prefix in the makefile. These need to - # be made available without that prefix through sysconfig. - # Special care is needed to ensure that variable expansion works, even - # if the expansion uses the name without a prefix. - renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS') - - while len(variables) > 0: - for name in tuple(variables): - value = notdone[name] - m1 = re.search(_findvar1_rx, value) - m2 = re.search(_findvar2_rx, value) - if m1 and m2: - m = m1 if m1.start() < m2.start() else m2 - else: - m = m1 if m1 else m2 - if m is not None: - n = m.group(1) - found = True - if n in done: - item = str(done[n]) - elif n in notdone: - # get it on a subsequent round - found = False - elif n in os.environ: - # do it like make: fall back to environment - item = os.environ[n] - - elif n in renamed_variables: - if (name.startswith('PY_') and - name[3:] in renamed_variables): - item = "" - - elif 'PY_' + n in notdone: - found = False - - else: - item = str(done['PY_' + n]) - - else: - done[n] = item = "" - - if found: - after = value[m.end():] - value = value[:m.start()] + item + after - if "$" in after: - notdone[name] = value - else: - try: - if name in _ALWAYS_STR: - raise ValueError - value = int(value) - except ValueError: - done[name] = value.strip() - else: - done[name] = value - variables.remove(name) - - if name.startswith('PY_') \ - and name[3:] in renamed_variables: - - name = name[3:] - if name not in done: - done[name] = value - - else: - # Adds unresolved variables to the done dict. - # This is disabled when called from distutils.sysconfig - if keep_unresolved: - done[name] = value - # bogus variable reference (e.g. "prefix=$/opt/python"); - # just drop it since we can't deal - variables.remove(name) - - # strip spurious spaces - for k, v in done.items(): - if isinstance(v, str): - done[k] = v.strip() - - # save the results in the global dictionary - vars.update(done) - return vars - - def get_makefile_filename(): """Return the path of the Makefile.""" if _PYTHON_BUILD: @@ -462,91 +341,44 @@ def _get_sysconfigdata_name(): f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}', ) - -def _generate_posix_vars(): - """Generate the Python module containing build-time variables.""" - import pprint - vars = {} - # load the installed Makefile: - makefile = get_makefile_filename() - try: - _parse_makefile(makefile, vars) - except OSError as e: - msg = f"invalid Python installation: unable to open {makefile}" - if hasattr(e, "strerror"): - msg = f"{msg} ({e.strerror})" - raise OSError(msg) - # load the installed pyconfig.h: - config_h = get_config_h_filename() - try: - with open(config_h, encoding="utf-8") as f: - parse_config_h(f, vars) - except OSError as e: - msg = f"invalid Python installation: unable to open {config_h}" - if hasattr(e, "strerror"): - msg = f"{msg} ({e.strerror})" - raise OSError(msg) - # On AIX, there are wrong paths to the linker scripts in the Makefile - # -- these paths are relative to the Python source, but when installed - # the scripts are in another directory. - if _PYTHON_BUILD: - vars['BLDSHARED'] = vars['LDSHARED'] - - # There's a chicken-and-egg situation on OS X with regards to the - # _sysconfigdata module after the changes introduced by #15298: - # get_config_vars() is called by get_platform() as part of the - # `make pybuilddir.txt` target -- which is a precursor to the - # _sysconfigdata.py module being constructed. Unfortunately, - # get_config_vars() eventually calls _init_posix(), which attempts - # to import _sysconfigdata, which we won't have built yet. In order - # for _init_posix() to work, if we're on Darwin, just mock up the - # _sysconfigdata module manually and populate it with the build vars. - # This is more than sufficient for ensuring the subsequent call to - # get_platform() succeeds. - name = _get_sysconfigdata_name() - if 'darwin' in sys.platform: - import types - module = types.ModuleType(name) - module.build_time_vars = vars - sys.modules[name] = module - - pybuilddir = f'build/lib.{get_platform()}-{_PY_VERSION_SHORT}' - if hasattr(sys, "gettotalrefcount"): - pybuilddir += '-pydebug' - os.makedirs(pybuilddir, exist_ok=True) - destfile = os.path.join(pybuilddir, name + '.py') - - with open(destfile, 'w', encoding='utf8') as f: - f.write('# system configuration generated and used by' - ' the sysconfig module\n') - f.write('build_time_vars = ') - pprint.pprint(vars, stream=f) - - # Create file used for sys.path fixup -- see Modules/getpath.c - with open('pybuilddir.txt', 'w', encoding='utf8') as f: - f.write(pybuilddir) - def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" # _sysconfigdata is generated at build time, see _generate_posix_vars() name = _get_sysconfigdata_name() - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) + + # For cross builds, the path to the target's sysconfigdata must be specified + # so it can be imported. It cannot be in PYTHONPATH, as foreign modules in + # sys.path can cause crashes when loaded by the host interpreter. + # Rely on truthiness as a valueless env variable is still an empty string. + # See OS X note in _generate_posix_vars re _sysconfigdata. + if (path := os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')): + from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES + from importlib.util import module_from_spec + spec = FileFinder(path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(name) + _temp = module_from_spec(spec) + spec.loader.exec_module(_temp) + else: + _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) build_time_vars = _temp.build_time_vars vars.update(build_time_vars) def _init_non_posix(vars): """Initialize the module as appropriate for NT""" # set basic install directories - import _imp + import _winapi + import _sysconfig vars['LIBDEST'] = get_path('stdlib') vars['BINLIBDEST'] = get_path('platstdlib') vars['INCLUDEPY'] = get_path('include') - try: - # GH-99201: _imp.extension_suffixes may be empty when - # HAVE_DYNAMIC_LOADING is not set. In this case, don't set EXT_SUFFIX. - vars['EXT_SUFFIX'] = _imp.extension_suffixes()[0] - except IndexError: - pass + + # Add EXT_SUFFIX, SOABI, and Py_GIL_DISABLED + vars.update(_sysconfig.config_vars()) + + vars['LIBDIR'] = _safe_realpath(os.path.join(get_config_var('installed_base'), 'libs')) + if hasattr(sys, 'dllhandle'): + dllhandle = _winapi.GetModuleFileName(sys.dllhandle) + vars['LIBRARY'] = os.path.basename(_safe_realpath(dllhandle)) + vars['LDLIBRARY'] = vars['LIBRARY'] vars['EXE'] = '.exe' vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) @@ -595,7 +427,7 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.join(_PROJECT_BASE, "PC") + inc_dir = os.path.dirname(sys._base_executable) else: inc_dir = _PROJECT_BASE else: @@ -633,6 +465,78 @@ def get_path(name, scheme=get_default_scheme(), vars=None, expand=True): return get_paths(scheme, vars, expand)[name] +def _init_config_vars(): + global _CONFIG_VARS + _CONFIG_VARS = {} + # Normalized versions of prefix and exec_prefix are handy to have; + # in fact, these are the standard versions used most places in the + # Distutils. + _PREFIX = os.path.normpath(sys.prefix) + _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) + _CONFIG_VARS['prefix'] = _PREFIX # FIXME: This gets overwriten by _init_posix. + _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX # FIXME: This gets overwriten by _init_posix. + _CONFIG_VARS['py_version'] = _PY_VERSION + _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT + _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT + _CONFIG_VARS['installed_base'] = _BASE_PREFIX + _CONFIG_VARS['base'] = _PREFIX + _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX + _CONFIG_VARS['platbase'] = _EXEC_PREFIX + _CONFIG_VARS['projectbase'] = _PROJECT_BASE + _CONFIG_VARS['platlibdir'] = sys.platlibdir + _CONFIG_VARS['implementation'] = _get_implementation() + _CONFIG_VARS['implementation_lower'] = _get_implementation().lower() + try: + _CONFIG_VARS['abiflags'] = sys.abiflags + except AttributeError: + # sys.abiflags may not be defined on all platforms. + _CONFIG_VARS['abiflags'] = '' + try: + _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') + except AttributeError: + _CONFIG_VARS['py_version_nodot_plat'] = '' + + if os.name == 'nt': + _init_non_posix(_CONFIG_VARS) + _CONFIG_VARS['VPATH'] = sys._vpath + if os.name == 'posix': + _init_posix(_CONFIG_VARS) + if _HAS_USER_BASE: + # Setting 'userbase' is done below the call to the + # init function to enable using 'get_config_var' in + # the init-function. + _CONFIG_VARS['userbase'] = _getuserbase() + + # e.g., 't' for free-threaded or '' for default build + _CONFIG_VARS['abi_thread'] = 't' if _CONFIG_VARS.get('Py_GIL_DISABLED') else '' + + # Always convert srcdir to an absolute path + srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) + if os.name == 'posix': + if _PYTHON_BUILD: + # If srcdir is a relative path (typically '.' or '..') + # then it should be interpreted relative to the directory + # containing Makefile. + base = os.path.dirname(get_makefile_filename()) + srcdir = os.path.join(base, srcdir) + else: + # srcdir is not meaningful since the installation is + # spread about the filesystem. We choose the + # directory containing the Makefile since we know it + # exists. + srcdir = os.path.dirname(get_makefile_filename()) + _CONFIG_VARS['srcdir'] = _safe_realpath(srcdir) + + # OS X platforms require special customization to handle + # multi-architecture, multi-os-version installers + if sys.platform == 'darwin': + import _osx_support + _osx_support.customize_config_vars(_CONFIG_VARS) + + global _CONFIG_VARS_INITIALIZED + _CONFIG_VARS_INITIALIZED = True + + def get_config_vars(*args): """With no arguments, return a dictionary of all configuration variables relevant for the current platform. @@ -643,66 +547,26 @@ def get_config_vars(*args): With arguments, return a list of values that result from looking up each argument in the configuration variable dictionary. """ - global _CONFIG_VARS - if _CONFIG_VARS is None: - _CONFIG_VARS = {} - # Normalized versions of prefix and exec_prefix are handy to have; - # in fact, these are the standard versions used most places in the - # Distutils. - _CONFIG_VARS['prefix'] = _PREFIX - _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX - _CONFIG_VARS['py_version'] = _PY_VERSION - _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT - _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT - _CONFIG_VARS['installed_base'] = _BASE_PREFIX - _CONFIG_VARS['base'] = _PREFIX - _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX - _CONFIG_VARS['platbase'] = _EXEC_PREFIX - _CONFIG_VARS['projectbase'] = _PROJECT_BASE - _CONFIG_VARS['platlibdir'] = sys.platlibdir - try: - _CONFIG_VARS['abiflags'] = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - _CONFIG_VARS['abiflags'] = '' - try: - _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') - except AttributeError: - _CONFIG_VARS['py_version_nodot_plat'] = '' - - if os.name == 'nt': - _init_non_posix(_CONFIG_VARS) - _CONFIG_VARS['VPATH'] = sys._vpath - if os.name == 'posix': - _init_posix(_CONFIG_VARS) - if _HAS_USER_BASE: - # Setting 'userbase' is done below the call to the - # init function to enable using 'get_config_var' in - # the init-function. - _CONFIG_VARS['userbase'] = _getuserbase() - - # Always convert srcdir to an absolute path - srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) - if os.name == 'posix': - if _PYTHON_BUILD: - # If srcdir is a relative path (typically '.' or '..') - # then it should be interpreted relative to the directory - # containing Makefile. - base = os.path.dirname(get_makefile_filename()) - srcdir = os.path.join(base, srcdir) - else: - # srcdir is not meaningful since the installation is - # spread about the filesystem. We choose the - # directory containing the Makefile since we know it - # exists. - srcdir = os.path.dirname(get_makefile_filename()) - _CONFIG_VARS['srcdir'] = _safe_realpath(srcdir) - - # OS X platforms require special customization to handle - # multi-architecture, multi-os-version installers - if sys.platform == 'darwin': - import _osx_support - _osx_support.customize_config_vars(_CONFIG_VARS) + global _CONFIG_VARS_INITIALIZED + + # Avoid claiming the lock once initialization is complete. + if not _CONFIG_VARS_INITIALIZED: + with _CONFIG_VARS_LOCK: + # Test again with the lock held to avoid races. Note that + # we test _CONFIG_VARS here, not _CONFIG_VARS_INITIALIZED, + # to ensure that recursive calls to get_config_vars() + # don't re-enter init_config_vars(). + if _CONFIG_VARS is None: + _init_config_vars() + else: + # If the site module initialization happened after _CONFIG_VARS was + # initialized, a virtual environment might have been activated, resulting in + # variables like sys.prefix changing their value, so we need to re-init the + # config vars (see GH-126789). + if _CONFIG_VARS['base'] != os.path.normpath(sys.prefix): + with _CONFIG_VARS_LOCK: + _CONFIG_VARS_INITIALIZED = False + _init_config_vars() if args: vals = [] @@ -737,7 +601,8 @@ def get_platform(): solaris-2.6-sun4u Windows will return one of: - win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) + win-amd64 (64-bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) + win-arm64 (64-bit Windows on ARM64 (aka AArch64) win32 (all others - specifically, sys.platform is returned) For other non-POSIX platforms, currently just returns 'sys.platform'. @@ -770,10 +635,22 @@ def get_platform(): machine = machine.replace('/', '-') if osname[:5] == "linux": - # At least on Linux/Intel, 'machine' is the processor -- - # i386, etc. - # XXX what about Alpha, SPARC, etc? - return f"{osname}-{machine}" + if sys.platform == "android": + osname = "android" + release = get_config_var("ANDROID_API_LEVEL") + + # Wheel tags use the ABI names from Android's own tools. + machine = { + "x86_64": "x86_64", + "i686": "x86", + "aarch64": "arm64_v8a", + "armv7l": "armeabi_v7a", + }[machine] + else: + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return f"{osname}-{machine}" elif osname[:5] == "sunos": if release[0] >= "5": # SunOS 5 == Solaris 2 osname = "solaris" @@ -795,10 +672,15 @@ def get_platform(): if m: release = m.group() elif osname[:6] == "darwin": - import _osx_support - osname, release, machine = _osx_support.get_platform_osx( - get_config_vars(), - osname, release, machine) + if sys.platform == "ios": + release = get_config_vars().get("IPHONEOS_DEPLOYMENT_TARGET", "13.0") + osname = sys.platform + machine = sys.implementation._multiarch + else: + import _osx_support + osname, release, machine = _osx_support.get_platform_osx( + get_config_vars(), + osname, release, machine) return f"{osname}-{release}-{machine}" @@ -807,6 +689,10 @@ def get_python_version(): return _PY_VERSION_SHORT +def _get_python_version_abi(): + return _PY_VERSION_SHORT + get_config_var("abi_thread") + + def expand_makefile_vars(s, vars): """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in 'string' according to 'vars' (a dictionary mapping variable names to @@ -817,6 +703,9 @@ def expand_makefile_vars(s, vars): """ import re + _findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" + _findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" + # This algorithm does multiple expansion, so if vars['foo'] contains # "${bar}", it will expand ${foo} to ${bar}, and then expand # ${bar}... and so forth. This is fine as long as 'vars' comes from @@ -831,28 +720,3 @@ def expand_makefile_vars(s, vars): else: break return s - - -def _print_dict(title, data): - for index, (key, value) in enumerate(sorted(data.items())): - if index == 0: - print(f'{title}: ') - print(f'\t{key} = "{value}"') - - -def _main(): - """Display all information sysconfig detains.""" - if '--generate-posix-vars' in sys.argv: - _generate_posix_vars() - return - print(f'Platform: "{get_platform()}"') - print(f'Python version: "{get_python_version()}"') - print(f'Current installation scheme: "{get_default_scheme()}"') - print() - _print_dict('Paths', get_paths()) - print() - _print_dict('Variables', get_config_vars()) - -if __name__ == '__main__': - _main() -'''.replace("Python", "RustPython").replace("/python", "/rustpython")) diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py new file mode 100644 index 0000000000..d7257b9d2d --- /dev/null +++ b/Lib/sysconfig/__main__.py @@ -0,0 +1,248 @@ +import os +import sys +from sysconfig import ( + _ALWAYS_STR, + _PYTHON_BUILD, + _get_sysconfigdata_name, + get_config_h_filename, + get_config_vars, + get_default_scheme, + get_makefile_filename, + get_paths, + get_platform, + get_python_version, + parse_config_h, +) + + +# Regexes needed for parsing Makefile (and similar syntaxes, +# like old-style Setup files). +_variable_rx = r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)" +_findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" +_findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" + + +def _parse_makefile(filename, vars=None, keep_unresolved=True): + """Parse a Makefile-style file. + + A dictionary containing name/value pairs is returned. If an + optional dictionary is passed in as the second argument, it is + used instead of a new dictionary. + """ + import re + + if vars is None: + vars = {} + done = {} + notdone = {} + + with open(filename, encoding=sys.getfilesystemencoding(), + errors="surrogateescape") as f: + lines = f.readlines() + + for line in lines: + if line.startswith('#') or line.strip() == '': + continue + m = re.match(_variable_rx, line) + if m: + n, v = m.group(1, 2) + v = v.strip() + # `$$' is a literal `$' in make + tmpv = v.replace('$$', '') + + if "$" in tmpv: + notdone[n] = v + else: + try: + if n in _ALWAYS_STR: + raise ValueError + + v = int(v) + except ValueError: + # insert literal `$' + done[n] = v.replace('$$', '$') + else: + done[n] = v + + # do variable interpolation here + variables = list(notdone.keys()) + + # Variables with a 'PY_' prefix in the makefile. These need to + # be made available without that prefix through sysconfig. + # Special care is needed to ensure that variable expansion works, even + # if the expansion uses the name without a prefix. + renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS') + + while len(variables) > 0: + for name in tuple(variables): + value = notdone[name] + m1 = re.search(_findvar1_rx, value) + m2 = re.search(_findvar2_rx, value) + if m1 and m2: + m = m1 if m1.start() < m2.start() else m2 + else: + m = m1 if m1 else m2 + if m is not None: + n = m.group(1) + found = True + if n in done: + item = str(done[n]) + elif n in notdone: + # get it on a subsequent round + found = False + elif n in os.environ: + # do it like make: fall back to environment + item = os.environ[n] + + elif n in renamed_variables: + if (name.startswith('PY_') and + name[3:] in renamed_variables): + item = "" + + elif 'PY_' + n in notdone: + found = False + + else: + item = str(done['PY_' + n]) + + else: + done[n] = item = "" + + if found: + after = value[m.end():] + value = value[:m.start()] + item + after + if "$" in after: + notdone[name] = value + else: + try: + if name in _ALWAYS_STR: + raise ValueError + value = int(value) + except ValueError: + done[name] = value.strip() + else: + done[name] = value + variables.remove(name) + + if name.startswith('PY_') \ + and name[3:] in renamed_variables: + + name = name[3:] + if name not in done: + done[name] = value + + else: + # Adds unresolved variables to the done dict. + # This is disabled when called from distutils.sysconfig + if keep_unresolved: + done[name] = value + # bogus variable reference (e.g. "prefix=$/opt/python"); + # just drop it since we can't deal + variables.remove(name) + + # strip spurious spaces + for k, v in done.items(): + if isinstance(v, str): + done[k] = v.strip() + + # save the results in the global dictionary + vars.update(done) + return vars + + +def _print_config_dict(d, stream): + print ("{", file=stream) + for k, v in sorted(d.items()): + print(f" {k!r}: {v!r},", file=stream) + print ("}", file=stream) + + +def _generate_posix_vars(): + """Generate the Python module containing build-time variables.""" + vars = {} + # load the installed Makefile: + makefile = get_makefile_filename() + try: + _parse_makefile(makefile, vars) + except OSError as e: + msg = f"invalid Python installation: unable to open {makefile}" + if hasattr(e, "strerror"): + msg = f"{msg} ({e.strerror})" + raise OSError(msg) + # load the installed pyconfig.h: + config_h = get_config_h_filename() + try: + with open(config_h, encoding="utf-8") as f: + parse_config_h(f, vars) + except OSError as e: + msg = f"invalid Python installation: unable to open {config_h}" + if hasattr(e, "strerror"): + msg = f"{msg} ({e.strerror})" + raise OSError(msg) + # On AIX, there are wrong paths to the linker scripts in the Makefile + # -- these paths are relative to the Python source, but when installed + # the scripts are in another directory. + if _PYTHON_BUILD: + vars['BLDSHARED'] = vars['LDSHARED'] + + # There's a chicken-and-egg situation on OS X with regards to the + # _sysconfigdata module after the changes introduced by #15298: + # get_config_vars() is called by get_platform() as part of the + # `make pybuilddir.txt` target -- which is a precursor to the + # _sysconfigdata.py module being constructed. Unfortunately, + # get_config_vars() eventually calls _init_posix(), which attempts + # to import _sysconfigdata, which we won't have built yet. In order + # for _init_posix() to work, if we're on Darwin, just mock up the + # _sysconfigdata module manually and populate it with the build vars. + # This is more than sufficient for ensuring the subsequent call to + # get_platform() succeeds. + name = _get_sysconfigdata_name() + if 'darwin' in sys.platform: + import types + module = types.ModuleType(name) + module.build_time_vars = vars + sys.modules[name] = module + + pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' + if hasattr(sys, "gettotalrefcount"): + pybuilddir += '-pydebug' + os.makedirs(pybuilddir, exist_ok=True) + destfile = os.path.join(pybuilddir, name + '.py') + + with open(destfile, 'w', encoding='utf8') as f: + f.write('# system configuration generated and used by' + ' the sysconfig module\n') + f.write('build_time_vars = ') + _print_config_dict(vars, stream=f) + + # Create file used for sys.path fixup -- see Modules/getpath.c + with open('pybuilddir.txt', 'w', encoding='utf8') as f: + f.write(pybuilddir) + + +def _print_dict(title, data): + for index, (key, value) in enumerate(sorted(data.items())): + if index == 0: + print(f'{title}: ') + print(f'\t{key} = "{value}"') + + +def _main(): + """Display all information sysconfig detains.""" + if '--generate-posix-vars' in sys.argv: + _generate_posix_vars() + return + print(f'Platform: "{get_platform()}"') + print(f'Python version: "{get_python_version()}"') + print(f'Current installation scheme: "{get_default_scheme()}"') + print() + _print_dict('Paths', get_paths()) + print() + _print_dict('Variables', get_config_vars()) + + +if __name__ == '__main__': + try: + _main() + except BrokenPipeError: + pass diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 4ff187674e..f5b8d42dba 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -7,6 +7,7 @@ import unittest import test.support from test import support +from test.support.script_helper import assert_python_ok from test.support import os_helper from test.support import socket_helper from test.support import captured_stderr @@ -330,14 +331,14 @@ def test_getsitepackages(self): self.assertEqual(len(dirs), 2) wanted = os.path.join('xoxo', sys.platlibdir, # XXX: RUSTPYTHON - 'rustpython%d.%d' % sys.version_info[:2], + f'rustpython{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[0], wanted) else: self.assertEqual(len(dirs), 1) wanted = os.path.join('xoxo', 'lib', # XXX: RUSTPYTHON - 'rustpython%d.%d' % sys.version_info[:2], + f'rustpython{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[-1], wanted) else: @@ -358,9 +359,7 @@ def test_no_home_directory(self): with EnvironmentVarGuard() as environ, \ mock.patch('os.path.expanduser', lambda path: path): - - del environ['PYTHONUSERBASE'] - del environ['APPDATA'] + environ.unset('PYTHONUSERBASE', 'APPDATA') user_base = site.getuserbase() self.assertTrue(user_base.startswith('~' + os.sep), @@ -382,6 +381,19 @@ def test_no_home_directory(self): mock_addsitedir.assert_not_called() self.assertFalse(known_paths) + def test_gethistoryfile(self): + filename = 'file' + rc, out, err = assert_python_ok('-c', + f'import site; assert site.gethistoryfile() == "{filename}"', + PYTHON_HISTORY=filename) + self.assertEqual(rc, 0) + + # Check that PYTHON_HISTORY is ignored in isolated mode. + rc, out, err = assert_python_ok('-I', '-c', + f'import site; assert site.gethistoryfile() != "{filename}"', + PYTHON_HISTORY=filename) + self.assertEqual(rc, 0) + def test_trace(self): message = "bla-bla-bla" for verbose, out in (True, message + "\n"), (False, ""): @@ -509,6 +521,44 @@ def test_sitecustomize_executed(self): else: self.fail("sitecustomize not imported automatically") + @support.requires_subprocess() + def test_customization_modules_on_startup(self): + mod_names = [ + 'sitecustomize' + ] + + if site.ENABLE_USER_SITE: + mod_names.append('usercustomize') + + temp_dir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, temp_dir) + + with EnvironmentVarGuard() as environ: + environ['PYTHONPATH'] = temp_dir + + for module_name in mod_names: + os_helper.rmtree(temp_dir) + os.mkdir(temp_dir) + + customize_path = os.path.join(temp_dir, f'{module_name}.py') + eyecatcher = f'EXECUTED_{module_name}' + + with open(customize_path, 'w') as f: + f.write(f'print("{eyecatcher}")') + + output = subprocess.check_output([sys.executable, '-c', '""']) + self.assertIn(eyecatcher, output.decode('utf-8')) + + # -S blocks any site-packages + output = subprocess.check_output([sys.executable, '-S', '-c', '""']) + self.assertNotIn(eyecatcher, output.decode('utf-8')) + + # -s blocks user site-packages + if 'usercustomize' == module_name: + output = subprocess.check_output([sys.executable, '-s', '-c', '""']) + self.assertNotIn(eyecatcher, output.decode('utf-8')) + + @unittest.skipUnless(hasattr(urllib.request, "HTTPSHandler"), 'need SSL support to download license') @test.support.requires_resource('network') diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 6db1442980..da242c7df7 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -1,24 +1,35 @@ +import platform +import re import unittest import sys import os import subprocess import shutil +import json +import textwrap from copy import copy from test.support import ( - captured_stdout, PythonSymlink, requires_subprocess, is_wasi + captured_stdout, + is_apple_mobile, + is_wasi, + PythonSymlink, + requires_subprocess, ) from test.support.import_helper import import_module from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink, change_cwd) -from test.support.warnings_helper import check_warnings +from test.support.venv import VirtualEnvironment import sysconfig from sysconfig import (get_paths, get_platform, get_config_vars, get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, - _expand_vars, _get_preferred_schemes, _main) + _expand_vars, _get_preferred_schemes) +from sysconfig.__main__ import _main, _parse_makefile +import _imp import _osx_support +import _sysconfig HAS_USER_BASE = sysconfig._HAS_USER_BASE @@ -41,6 +52,7 @@ def setUp(self): self.name = os.name self.platform = sys.platform self.version = sys.version + self._framework = sys._framework self.sep = os.sep self.join = os.path.join self.isabs = os.path.isabs @@ -64,6 +76,7 @@ def tearDown(self): os.name = self.name sys.platform = self.platform sys.version = self.version + sys._framework = self._framework os.sep = self.sep os.path.join = self.join os.path.isabs = self.isabs @@ -91,6 +104,12 @@ def _cleanup_testfn(self): elif os.path.isdir(path): shutil.rmtree(path) + def venv(self, **venv_create_args): + return VirtualEnvironment.from_tmpdir( + prefix=f'{self.id()}-venv-', + **venv_create_args, + ) + def test_get_path_names(self): self.assertEqual(get_path_names(), sysconfig._SCHEME_KEYS) @@ -137,35 +156,37 @@ def test_get_preferred_schemes(self): # Mac, framework build. os.name = 'posix' sys.platform = 'darwin' - sys._framework = True + sys._framework = "MyPython" self.assertIsInstance(schemes, dict) self.assertEqual(set(schemes), expected_schemes) - # NOTE: RUSTPYTHON this is hardcoded to 'python', we're set up for failure. - @unittest.expectedFailure def test_posix_venv_scheme(self): # The following directories were hardcoded in the venv module # before bpo-45413, here we assert the posix_venv scheme does not regress binpath = 'bin' incpath = 'include' libpath = os.path.join('lib', - 'python%d.%d' % sys.version_info[:2], + # XXX: RUSTPYTHON + f'rustpython{sysconfig._get_python_version_abi()}', 'site-packages') - # Resolve the paths in prefix - binpath = os.path.join(sys.prefix, binpath) - incpath = os.path.join(sys.prefix, incpath) - libpath = os.path.join(sys.prefix, libpath) + # Resolve the paths in an imaginary venv/ directory + binpath = os.path.join('venv', binpath) + incpath = os.path.join('venv', incpath) + libpath = os.path.join('venv', libpath) + + # Mimic the venv module, set all bases to the venv directory + bases = ('base', 'platbase', 'installed_base', 'installed_platbase') + vars = {base: 'venv' for base in bases} - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv')) + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv', vars=vars)) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv', vars=vars)) # The include directory on POSIX isn't exactly the same as before, # but it is "within" - sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv') + sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv', vars=vars) self.assertTrue(sysconfig_includedir.startswith(incpath + os.sep)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_nt_venv_scheme(self): # The following directories were hardcoded in the venv module # before bpo-45413, here we assert the posix_venv scheme does not regress @@ -173,14 +194,19 @@ def test_nt_venv_scheme(self): incpath = 'Include' libpath = os.path.join('Lib', 'site-packages') - # Resolve the paths in prefix - binpath = os.path.join(sys.prefix, binpath) - incpath = os.path.join(sys.prefix, incpath) - libpath = os.path.join(sys.prefix, libpath) + # Resolve the paths in an imaginary venv\ directory + venv = 'venv' + binpath = os.path.join(venv, binpath) + incpath = os.path.join(venv, incpath) + libpath = os.path.join(venv, libpath) - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv')) - self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv')) + # Mimic the venv module, set all bases to the venv directory + bases = ('base', 'platbase', 'installed_base', 'installed_platbase') + vars = {base: 'venv' for base in bases} + + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv', vars=vars)) + self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv', vars=vars)) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv', vars=vars)) def test_venv_scheme(self): if sys.platform == 'win32': @@ -216,6 +242,11 @@ def test_get_config_vars(self): self.assertTrue(cvars) def test_get_platform(self): + # Check the actual platform returns something reasonable. + actual_platform = get_platform() + self.assertIsInstance(actual_platform, str) + self.assertTrue(actual_platform) + # windows XP, 32bits os.name = 'nt' sys.version = ('2.4.4 (#71, Oct 18 2006, 08:34:43) ' @@ -331,11 +362,28 @@ def test_get_platform(self): self.assertEqual(get_platform(), 'linux-i686') + # Android + os.name = 'posix' + sys.platform = 'android' + get_config_vars()['ANDROID_API_LEVEL'] = 9 + for machine, abi in { + 'x86_64': 'x86_64', + 'i686': 'x86', + 'aarch64': 'arm64_v8a', + 'armv7l': 'armeabi_v7a', + }.items(): + with self.subTest(machine): + self._set_uname(('Linux', 'localhost', '3.18.91+', + '#1 Tue Jan 9 20:35:43 UTC 2018', machine)) + self.assertEqual(get_platform(), f'android-9-{abi}') + # XXX more platforms to tests here # TODO: RUSTPYTHON @unittest.expectedFailure @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't distribute header files in the runtime environment") def test_get_config_h_filename(self): config_h = sysconfig.get_config_h_filename() self.assertTrue(os.path.isfile(config_h), config_h) @@ -381,8 +429,8 @@ def test_user_similar(self): if name == 'platlib': # Replace "/lib64/python3.11/site-packages" suffix # with "/lib/python3.11/site-packages". - py_version_short = sysconfig.get_python_version() - suffix = f'python{py_version_short}/site-packages' + py_version_abi = sysconfig._get_python_version_abi() + suffix = f'python{py_version_abi}/site-packages' expected = expected.replace(f'/{sys.platlibdir}/{suffix}', f'/lib/{suffix}') self.assertEqual(user_path, expected) @@ -402,6 +450,32 @@ def test_ldshared_value(self): self.assertIn(ldflags, ldshared) + @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") + def test_soabi(self): + soabi = sysconfig.get_config_var('SOABI') + self.assertIn(soabi, _imp.extension_suffixes()[0]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_library(self): + library = sysconfig.get_config_var('LIBRARY') + ldlibrary = sysconfig.get_config_var('LDLIBRARY') + major, minor = sys.version_info[:2] + if sys.platform == 'win32': + self.assertTrue(library.startswith(f'python{major}{minor}')) + self.assertTrue(library.endswith('.dll')) + self.assertEqual(library, ldlibrary) + elif is_apple_mobile: + framework = sysconfig.get_config_var('PYTHONFRAMEWORK') + self.assertEqual(ldlibrary, f"{framework}.framework/{framework}") + else: + self.assertTrue(library.startswith(f'libpython{major}.{minor}')) + self.assertTrue(library.endswith('.a')) + if sys.platform == 'darwin' and sys._framework: + self.skipTest('gh-110824: skip LDLIBRARY test for framework build') + else: + self.assertTrue(ldlibrary.startswith(f'libpython{major}.{minor}')) + @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX") @requires_subprocess() def test_platform_in_subprocess(self): @@ -448,6 +522,8 @@ def test_platform_in_subprocess(self): @unittest.expectedFailureIf(sys.platform != "win32", "TODO: RUSTPYTHON") @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't include config folder at runtime") def test_srcdir(self): # See Issues #15322, #15364. srcdir = sysconfig.get_config_var('srcdir') @@ -460,11 +536,15 @@ def test_srcdir(self): # should be a full source checkout. Python_h = os.path.join(srcdir, 'Include', 'Python.h') self.assertTrue(os.path.exists(Python_h), Python_h) - # /PC/pyconfig.h always exists even if unused on POSIX. - pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h') + # /PC/pyconfig.h.in always exists even if unused + pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in) + if os.name == 'nt': + # /pyconfig.h exists on Windows in a build tree + pyconfig_h = os.path.join(sys.executable, '..', 'pyconfig.h') + self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) elif os.name == 'posix': makefile_dir = os.path.dirname(sysconfig.get_makefile_filename()) # Issue #19340: srcdir has been realpath'ed already @@ -479,23 +559,16 @@ def test_srcdir_independent_of_cwd(self): srcdir2 = sysconfig.get_config_var('srcdir') self.assertEqual(srcdir, srcdir2) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sysconfig.get_config_var('EXT_SUFFIX') is None, 'EXT_SUFFIX required for this test') + @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") def test_EXT_SUFFIX_in_vars(self): - import _imp - if not _imp.extension_suffixes(): - self.skipTest("stub loader has no suffixes") vars = sysconfig.get_config_vars() self.assertEqual(vars['EXT_SUFFIX'], _imp.extension_suffixes()[0]) - @unittest.skipUnless(sys.platform == 'linux' and - hasattr(sys.implementation, '_multiarch'), - 'multiarch-specific test') - def test_triplet_in_ext_suffix(self): + @unittest.skipUnless(sys.platform == 'linux', 'Linux-specific test') + def test_linux_ext_suffix(self): ctypes = import_module('ctypes') - import platform, re machine = platform.machine() suffix = sysconfig.get_config_var('EXT_SUFFIX') if re.match('(aarch64|arm|mips|ppc|powerpc|s390|sparc)', machine): @@ -510,11 +583,97 @@ def test_triplet_in_ext_suffix(self): # TODO: RUSTPYTHON @unittest.expectedFailure + @unittest.skipUnless(sys.platform == 'android', 'Android-specific test') + def test_android_ext_suffix(self): + machine = platform.machine() + suffix = sysconfig.get_config_var('EXT_SUFFIX') + expected_triplet = { + "x86_64": "x86_64-linux-android", + "i686": "i686-linux-android", + "aarch64": "aarch64-linux-android", + "armv7l": "arm-linux-androideabi", + }[machine] + self.assertTrue(suffix.endswith(f"-{expected_triplet}.so"), + f"{machine=}, {suffix=}") + + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(sys.platform == 'darwin', 'OS X-specific test') def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') self.assertTrue(suffix.endswith('-darwin.so'), suffix) + # TODO: RUSTPYTHON + @unittest.expectedFailure + @requires_subprocess() + def test_config_vars_depend_on_site_initialization(self): + script = textwrap.dedent(""" + import sysconfig + + config_vars = sysconfig.get_config_vars() + + import json + print(json.dumps(config_vars, indent=2)) + """) + + with self.venv() as venv: + site_config_vars = json.loads(venv.run('-c', script).stdout) + no_site_config_vars = json.loads(venv.run('-S', '-c', script).stdout) + + self.assertNotEqual(site_config_vars, no_site_config_vars) + # With the site initialization, the virtual environment should be enabled. + self.assertEqual(site_config_vars['base'], venv.prefix) + self.assertEqual(site_config_vars['platbase'], venv.prefix) + #self.assertEqual(site_config_vars['prefix'], venv.prefix) # # FIXME: prefix gets overwriten by _init_posix + # Without the site initialization, the virtual environment should be disabled. + self.assertEqual(no_site_config_vars['base'], site_config_vars['installed_base']) + self.assertEqual(no_site_config_vars['platbase'], site_config_vars['installed_platbase']) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + @requires_subprocess() + def test_config_vars_recalculation_after_site_initialization(self): + script = textwrap.dedent(""" + import sysconfig + + before = sysconfig.get_config_vars() + + import site + site.main() + + after = sysconfig.get_config_vars() + + import json + print(json.dumps({'before': before, 'after': after}, indent=2)) + """) + + with self.venv() as venv: + config_vars = json.loads(venv.run('-S', '-c', script).stdout) + + self.assertNotEqual(config_vars['before'], config_vars['after']) + self.assertEqual(config_vars['after']['base'], venv.prefix) + #self.assertEqual(config_vars['after']['prefix'], venv.prefix) # FIXME: prefix gets overwriten by _init_posix + #self.assertEqual(config_vars['after']['exec_prefix'], venv.prefix) # FIXME: exec_prefix gets overwriten by _init_posix + + # TODO: RUSTPYTHON + @unittest.expectedFailure + @requires_subprocess() + def test_paths_depend_on_site_initialization(self): + script = textwrap.dedent(""" + import sysconfig + + paths = sysconfig.get_paths() + + import json + print(json.dumps(paths, indent=2)) + """) + + with self.venv() as venv: + site_paths = json.loads(venv.run('-c', script).stdout) + no_site_paths = json.loads(venv.run('-S', '-c', script).stdout) + + self.assertNotEqual(site_paths, no_site_paths) + + class MakefileTests(unittest.TestCase): # TODO: RUSTPYTHON @@ -522,6 +681,8 @@ class MakefileTests(unittest.TestCase): @unittest.skipIf(sys.platform.startswith('win'), 'Test is not Windows compatible') @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't include config folder at runtime") def test_get_makefile_filename(self): makefile = sysconfig.get_makefile_filename() self.assertTrue(os.path.isfile(makefile), makefile) @@ -536,7 +697,7 @@ def test_parse_makefile(self): print("var5=dollar$$5", file=makefile) print("var6=${var3}/lib/python3.5/config-$(VAR2)$(var5)" "-x86_64-linux-gnu", file=makefile) - vars = sysconfig._parse_makefile(TESTFN) + vars = _parse_makefile(TESTFN) self.assertEqual(vars, { 'var1': 'ab42', 'VAR2': 'b42', @@ -547,5 +708,26 @@ def test_parse_makefile(self): }) +class DeprecationTests(unittest.TestCase): + def deprecated(self, removal_version, deprecation_msg=None, error=Exception, error_msg=None): + if sys.version_info >= removal_version: + return self.assertRaises(error, msg=error_msg) + else: + return self.assertWarns(DeprecationWarning, msg=deprecation_msg) + + def test_is_python_build_check_home(self): + with self.deprecated( + removal_version=(3, 15), + deprecation_msg=( + 'The check_home argument of sysconfig.is_python_build is ' + 'deprecated and its value is ignored. ' + 'It will be removed in Python 3.15.' + ), + error=TypeError, + error_msg="is_python_build() takes 0 positional arguments but 1 were given", + ): + sysconfig.is_python_build('foo') + + if __name__ == "__main__": unittest.main() diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 2c4442900b..9fae516fe0 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -18,6 +18,7 @@ mod stat; mod string; #[cfg(feature = "compiler")] mod symtable; +mod sysconfig; mod sysconfigdata; #[cfg(feature = "threading")] pub mod thread; @@ -97,6 +98,7 @@ pub fn get_module_inits() -> StdlibMap { "_signal" => signal::make_module, "_sre" => sre::make_module, "_stat" => stat::make_module, + "_sysconfig" => sysconfig::make_module, "_string" => string::make_module, "time" => time::make_module, "_typing" => typing::make_module, diff --git a/vm/src/stdlib/sysconfig.rs b/vm/src/stdlib/sysconfig.rs new file mode 100644 index 0000000000..5680198afe --- /dev/null +++ b/vm/src/stdlib/sysconfig.rs @@ -0,0 +1,14 @@ +pub(crate) use sysconfig::make_module; + +#[pymodule(name = "_sysconfig")] +pub(crate) mod sysconfig { + use crate::{VirtualMachine, builtins::PyDictRef, convert::ToPyObject}; + + #[pyfunction] + fn config_vars(vm: &VirtualMachine) -> PyDictRef { + let vars = vm.ctx.new_dict(); + vars.set_item("Py_GIL_DISABLED", false.to_pyobject(vm), vm) + .unwrap(); + vars + } +} From b47fc3ed2d09b5aa9d1069cca1cee4bbe44181e3 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 7 Sep 2025 11:39:34 +0300 Subject: [PATCH 2/4] Update vm/src/stdlib/sysconfig.rs Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- vm/src/stdlib/sysconfig.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/stdlib/sysconfig.rs b/vm/src/stdlib/sysconfig.rs index 5680198afe..2e0a8a51c7 100644 --- a/vm/src/stdlib/sysconfig.rs +++ b/vm/src/stdlib/sysconfig.rs @@ -7,7 +7,7 @@ pub(crate) mod sysconfig { #[pyfunction] fn config_vars(vm: &VirtualMachine) -> PyDictRef { let vars = vm.ctx.new_dict(); - vars.set_item("Py_GIL_DISABLED", false.to_pyobject(vm), vm) + vars.set_item("Py_GIL_DISABLED", true.to_pyobject(vm), vm) .unwrap(); vars } From 118fa87c286095a7bcb150e82303d13f179ba666 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 7 Sep 2025 12:20:48 +0300 Subject: [PATCH 3/4] Mark failing test --- Lib/test/test_faulthandler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index d7e2c6a1de..1225db997e 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -979,6 +979,7 @@ def test_cancel_later_without_dump_traceback_later(self): self.assertEqual(output, []) self.assertEqual(exitcode, 0) + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AttributeError: module 'msvcrt' has no attribute 'GetErrorMode'") @threading_helper.requires_working_threading() @unittest.skipUnless(support.Py_GIL_DISABLED, "only meaningful if the GIL is disabled") def test_free_threaded_dump_traceback(self): From 02f9f26df0f729990bd1b0075a26e83d0c21b954 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 7 Sep 2025 20:59:27 +0300 Subject: [PATCH 4/4] Trigger CI