From 99b82368c72550f39cdf9dea8d737b8ebf8b51b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 26 Nov 2024 17:42:03 +0000 Subject: [PATCH 1/7] GH-127178: install a _sysconfig_vars_(...).json in the stdlib directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/sysconfig/__main__.py | 37 ++++++++++++++----- Lib/test/test_sysconfig.py | 21 ++++++++++- Makefile.pre.in | 4 +- ...-11-26-17-42-00.gh-issue-127178.U8hxjc.rst | 4 ++ 4 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-26-17-42-00.gh-issue-127178.U8hxjc.rst diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index d7257b9d2d00db..5660a6c5105b9f 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -1,5 +1,7 @@ +import json import os import sys +import types from sysconfig import ( _ALWAYS_STR, _PYTHON_BUILD, @@ -157,6 +159,19 @@ def _print_config_dict(d, stream): print ("}", file=stream) +def _get_pybuilddir(): + pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' + if hasattr(sys, "gettotalrefcount"): + pybuilddir += '-pydebug' + return pybuilddir + + +def _get_json_data_name(): + name = _get_sysconfigdata_name() + assert name.startswith('_sysconfigdata') + return name.replace('_sysconfigdata', '_sysconfig_vars') + '.json' + + def _generate_posix_vars(): """Generate the Python module containing build-time variables.""" vars = {} @@ -185,6 +200,8 @@ def _generate_posix_vars(): if _PYTHON_BUILD: vars['BLDSHARED'] = vars['LDSHARED'] + name = _get_sysconfigdata_name() + # There's a chicken-and-egg situation on OS X with regards to the # _sysconfigdata module after the changes introduced by #15298: # get_config_vars() is called by get_platform() as part of the @@ -196,16 +213,13 @@ def _generate_posix_vars(): # _sysconfigdata module manually and populate it with the build vars. # This is more than sufficient for ensuring the subsequent call to # get_platform() succeeds. - name = _get_sysconfigdata_name() - if 'darwin' in sys.platform: - import types - module = types.ModuleType(name) - module.build_time_vars = vars - sys.modules[name] = module + # GH-127178: Since we started generating a .json file, we also need this to + # be able to run sysconfig.get_config_vars(). + module = types.ModuleType(name) + module.build_time_vars = vars + sys.modules[name] = module - pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' - if hasattr(sys, "gettotalrefcount"): - pybuilddir += '-pydebug' + pybuilddir = _get_pybuilddir() os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') @@ -215,6 +229,11 @@ def _generate_posix_vars(): f.write('build_time_vars = ') _print_config_dict(vars, stream=f) + # Write a JSON file with the output of sysconfig.get_config_vars + jsonfile = os.path.join(pybuilddir, _get_json_data_name()) + with open(jsonfile, 'w') as f: + json.dump(get_config_vars(), f, indent=2) + # Create file used for sys.path fixup -- see Modules/getpath.c with open('pybuilddir.txt', 'w', encoding='utf8') as f: f.write(pybuilddir) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index a705dd0cd89e4d..ae7310d380bc2f 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -25,8 +25,9 @@ from sysconfig import (get_paths, get_platform, get_config_vars, get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, - _expand_vars, _get_preferred_schemes) -from sysconfig.__main__ import _main, _parse_makefile + _expand_vars, _get_preferred_schemes, + is_python_build, _PROJECT_BASE) +from sysconfig.__main__ import _main, _parse_makefile, _get_pybuilddir, _get_json_data_name import _imp import _osx_support import _sysconfig @@ -625,6 +626,22 @@ def test_makefile_overwrites_config_vars(self): self.assertNotEqual(data['prefix'], data['base_prefix']) self.assertNotEqual(data['exec_prefix'], data['base_exec_prefix']) + def test_sysconfigdata_json(self): + if '_PYTHON_SYSCONFIGDATA_PATH' in os.environ: + data_dir = os.environ['_PYTHON_SYSCONFIGDATA_PATH'] + elif is_python_build(): + data_dir = os.path.join(_PROJECT_BASE, _get_pybuilddir()) + else: + data_dir = sys._stdlib_dir + + json_data_path = os.path.join(data_dir, _get_json_data_name()) + + with open(json_data_path) as f: + json_config_vars = json.load(f) + + self.assertEqual(get_config_vars(), json_config_vars) + + class MakefileTests(unittest.TestCase): @unittest.skipIf(sys.platform.startswith('win'), diff --git a/Makefile.pre.in b/Makefile.pre.in index 8d94ba361fd934..724354746b8d81 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2645,8 +2645,8 @@ libinstall: all $(srcdir)/Modules/xxmodule.c esac; \ done; \ done - $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py \ - $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfig_vars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt @ # If app store compliance has been configured, apply the patch to the @ # installed library code. The patch has been previously validated against diff --git a/Misc/NEWS.d/next/Library/2024-11-26-17-42-00.gh-issue-127178.U8hxjc.rst b/Misc/NEWS.d/next/Library/2024-11-26-17-42-00.gh-issue-127178.U8hxjc.rst new file mode 100644 index 00000000000000..b703b58ea8e1d9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-26-17-42-00.gh-issue-127178.U8hxjc.rst @@ -0,0 +1,4 @@ +A ``_sysconfig_vars_(...).json`` file is now shipped in the standard library +directory. It contains the output of :func:`sysconfig.get_config_vars` on +the default environment encoded as JSON data. This is an implementation +detail, and may change at any time. From 2a344f314debdf5550e3a0f666decc39ec2c6d59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 26 Nov 2024 18:00:42 +0000 Subject: [PATCH 2/7] Only run test on POSIX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/test/test_sysconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index ae7310d380bc2f..4cd5fdcdbfd5f1 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -626,6 +626,7 @@ def test_makefile_overwrites_config_vars(self): self.assertNotEqual(data['prefix'], data['base_prefix']) self.assertNotEqual(data['exec_prefix'], data['base_exec_prefix']) + @unittest.skipIf(os.name != 'posix', '_sysconfig-vars JSON file is only available on POSIX') def test_sysconfigdata_json(self): if '_PYTHON_SYSCONFIGDATA_PATH' in os.environ: data_dir = os.environ['_PYTHON_SYSCONFIGDATA_PATH'] From 8d351debc68e1ac19dc7001ff498dc37c8679f78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 26 Nov 2024 19:55:37 +0000 Subject: [PATCH 3/7] Skip test on WASI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/test/test_sysconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 4cd5fdcdbfd5f1..e53e8fa42bcf20 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -627,6 +627,7 @@ def test_makefile_overwrites_config_vars(self): self.assertNotEqual(data['exec_prefix'], data['base_exec_prefix']) @unittest.skipIf(os.name != 'posix', '_sysconfig-vars JSON file is only available on POSIX') + @unittest.skipIf(is_wasi, "_sysconfig-vars JSON file currently isn't available on WASI") def test_sysconfigdata_json(self): if '_PYTHON_SYSCONFIGDATA_PATH' in os.environ: data_dir = os.environ['_PYTHON_SYSCONFIGDATA_PATH'] From 22b431686fbb4d496c23e79329851b2930f5a769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 26 Nov 2024 21:46:33 +0000 Subject: [PATCH 4/7] Skip test on Android and iOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/test/test_sysconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index e53e8fa42bcf20..2096d35db70794 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -628,6 +628,7 @@ def test_makefile_overwrites_config_vars(self): @unittest.skipIf(os.name != 'posix', '_sysconfig-vars JSON file is only available on POSIX') @unittest.skipIf(is_wasi, "_sysconfig-vars JSON file currently isn't available on WASI") + @unittest.skipIf(is_android or is_apple_mobile, 'Android and iOS change the prefix') def test_sysconfigdata_json(self): if '_PYTHON_SYSCONFIGDATA_PATH' in os.environ: data_dir = os.environ['_PYTHON_SYSCONFIGDATA_PATH'] From 59bfcd35ec9bb464998b827ad23ad81df9af6fab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 26 Nov 2024 21:58:29 +0000 Subject: [PATCH 5/7] =?UTF-8?q?Add=20missing=20import=20=F0=9F=A4=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/test/test_sysconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 2096d35db70794..ba7be45a87ded4 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -11,6 +11,7 @@ from test.support import ( captured_stdout, + is_android, is_apple_mobile, is_wasi, PythonSymlink, From 2f5aa3567c5c7248a075c5e28ca951334594fa86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Wed, 27 Nov 2024 04:09:12 +0000 Subject: [PATCH 6/7] Show full diff in tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/test/test_sysconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index ba7be45a87ded4..9b0936ac122bc8 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -41,6 +41,7 @@ class TestSysConfig(unittest.TestCase): def setUp(self): super(TestSysConfig, self).setUp() + self.maxDiff = None self.sys_path = sys.path[:] # patching os.uname if hasattr(os, 'uname'): From feffa0ff30134135781c2e26c14ae41ec491fd1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Wed, 27 Nov 2024 05:01:42 +0000 Subject: [PATCH 7/7] Ignore projectbase and srcdir keys in test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/test/test_sysconfig.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 9b0936ac122bc8..0df1a67ea2b720 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -644,7 +644,14 @@ def test_sysconfigdata_json(self): with open(json_data_path) as f: json_config_vars = json.load(f) - self.assertEqual(get_config_vars(), json_config_vars) + system_config_vars = get_config_vars() + + # Ignore keys in the check + for key in ('projectbase', 'srcdir'): + json_config_vars.pop(key) + system_config_vars.pop(key) + + self.assertEqual(system_config_vars, json_config_vars) class MakefileTests(unittest.TestCase):