From 49f43ae981f01e0b7b164c48ba9918e044d7996f Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 13 May 2025 13:28:52 +0100 Subject: [PATCH 1/7] gh-133779: Revert Windows generation of pyconfig.h and go back to a static header. This means that extension builders will need to specify Py_GIL_DISABLED if they want to link to the free-threaded builds. --- ...-05-13-13-25-27.gh-issue-133779.-YcTBz.rst | 6 ++++ PC/{pyconfig.h.in => pyconfig.h} | 17 ++++++---- PCbuild/_freeze_module.vcxproj | 27 +--------------- PCbuild/pyproject.props | 8 ++--- PCbuild/pythoncore.vcxproj | 31 ++----------------- PCbuild/regen.targets | 14 ++++----- 6 files changed, 31 insertions(+), 72 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2025-05-13-13-25-27.gh-issue-133779.-YcTBz.rst rename PC/{pyconfig.h.in => pyconfig.h} (97%) diff --git a/Misc/NEWS.d/next/Windows/2025-05-13-13-25-27.gh-issue-133779.-YcTBz.rst b/Misc/NEWS.d/next/Windows/2025-05-13-13-25-27.gh-issue-133779.-YcTBz.rst new file mode 100644 index 00000000000000..550600d5eebf11 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2025-05-13-13-25-27.gh-issue-133779.-YcTBz.rst @@ -0,0 +1,6 @@ +Reverts the change to generate different :file:`pyconfig.h` files based on +compiler settings, as it was frequently causing extension builds to break. +In particular, the ``Py_GIL_DISABLED`` preprocessor variable must now always +be defined explicitly when compiling for the experimental free-threaded +runtime. The :func:`sysconfig.get_config_var` function can be used to +determine whether the current runtime was compiled with that flag or not. diff --git a/PC/pyconfig.h.in b/PC/pyconfig.h similarity index 97% rename from PC/pyconfig.h.in rename to PC/pyconfig.h index 1d659e7cee682b..ae2d014467b643 100644 --- a/PC/pyconfig.h.in +++ b/PC/pyconfig.h @@ -99,12 +99,17 @@ WIN32 is still required for the locale module. # define Py_DEBUG 1 #endif -/* Define to 1 if you want to disable the GIL */ -/* Uncomment the definition for free-threaded builds, or define it manually - * when compiling extension modules. Note that we test with #ifdef, so - * defining as 0 will still disable the GIL. */ -#ifndef Py_GIL_DISABLED -/* #define Py_GIL_DISABLED 1 */ +/* Define to 1 when compiling for experimental free-threaded builds */ +#ifdef Py_GIL_DISABLED +/* We undefine if it was set to zero because all later checks are #ifdef. + * Note that non-Windows builds do not do this, and so every effort should + * be made to avoid defining the variable at all when not desired. However, + * sysconfig.get_config_val always returns a 1 or a 0, and so it seems likely + * that a build backend will define it with the value. + */ +#if Py_GIL_DISABLED == 0 +#undef Py_GIL_DISABLED +#endif #endif /* Compiler specific defines */ diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 8c161835af9a8c..efff6a58d895cb 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -276,7 +276,7 @@ - + @@ -436,31 +436,6 @@ - - - - - - - - - - @(PyConfigH->'%(FullPath)', ';') - $([System.IO.File]::ReadAllText($(PyConfigH))) - $([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h')) - - - $(PyConfigHText.Replace('/* #define Py_GIL_DISABLED 1 */', '#define Py_GIL_DISABLED 1')) - - - - - diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 4e414dc913b9d5..7272542e13a5ca 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -10,10 +10,9 @@ $(MSBuildThisFileDirectory)obj\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\ $(IntDir.Replace(`\\`, `\`)) - - $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\pythoncore\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\zlib-ng\ + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_$(Configuration) $(ProjectName) $(TargetName)$(PyDebugExt) false @@ -49,11 +48,12 @@ <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64' and $(PlatformToolset) != 'ClangCL'">_M_X64;$(_PlatformPreprocessorDefinition) <_Py3NamePreprocessorDefinition>PY3_DLLNAME=L"$(Py3DllName)$(PyDebugExt)"; + <_FreeThreadedPreprocessorDefinition Condition="$(DisableGil) == 'true'">Py_GIL_DISABLED=1; - $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(GeneratedPyConfigDir);$(PySourcePath)PC;%(AdditionalIncludeDirectories) - WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PyStatsPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) + $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(PySourcePath)PC;%(AdditionalIncludeDirectories) + WIN32;$(_Py3NamePreprocessorDefinition)$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PyStatsPreprocessorDefinition)$(_PydPreprocessorDefinition)$(_FreeThreadedPreprocessorDefinition)%(PreprocessorDefinitions) _Py_USING_PGO=1;%(PreprocessorDefinitions) MaxSpeed diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 549d6284972afc..32a8f2dbad3d5e 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -102,6 +102,7 @@ /Zm200 %(AdditionalOptions) $(PySourcePath)Modules\_hacl;$(PySourcePath)Modules\_hacl\include;$(PySourcePath)Python;%(AdditionalIncludeDirectories) $(zlibNgDir);$(GeneratedZlibNgDir);%(AdditionalIncludeDirectories) + $(GeneratedJitStencilsDir);%(AdditionalIncludeDirectories) _USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";ZLIB_COMPAT;%(PreprocessorDefinitions) _Py_HAVE_ZLIB;%(PreprocessorDefinitions) _Py_JIT;%(PreprocessorDefinitions) @@ -409,7 +410,7 @@ - + @@ -688,34 +689,6 @@ - - - - - - - - - @(PyConfigH->'%(FullPath)', ';') - $([System.IO.File]::ReadAllText($(PyConfigH))) - $([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h')) - - - $(PyConfigHText.Replace('/* #define Py_GIL_DISABLED 1 */', '#define Py_GIL_DISABLED 1')) - - - - - - - - - - - git diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 3ad17737807235..21de614e71ddce 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -29,12 +29,12 @@ <_KeywordSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" /> <_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" /> - <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/> + <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(PySourcePath)PC\pyconfig.h;$(PySourcePath)Tools\jit\**"/> - <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/> - <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-aarch64-pc-windows-msvc.h" Condition="$(Platform) == 'ARM64'"/> - <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-i686-pc-windows-msvc.h" Condition="$(Platform) == 'Win32'"/> - <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-x86_64-pc-windows-msvc.h" Condition="$(Platform) == 'x64'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils.h"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-aarch64-pc-windows-msvc.h" Condition="$(Platform) == 'ARM64'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-i686-pc-windows-msvc.h" Condition="$(Platform) == 'Win32'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-x86_64-pc-windows-msvc.h" Condition="$(Platform) == 'x64'"/> <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" /> @@ -116,7 +116,7 @@ @@ -126,7 +126,7 @@ $(JITArgs) --debug + WorkingDirectory="$(GeneratedJitStencilsDir)"/> From ed89fa4600804a453b5738029f01a5d0eb49ac6f Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 13 May 2025 13:35:11 +0100 Subject: [PATCH 2/7] Add some docs --- Doc/howto/free-threading-extensions.rst | 8 ++++++++ Doc/whatsnew/3.14.rst | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 3f6ee517050bd8..5a3970f15d52d6 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -23,6 +23,14 @@ You can use it to enable code that only runs under the free-threaded build:: /* code that only runs in the free-threaded build */ #endif +.. note:: + + On Windows, this macro is not defined automatically, but must be specified + to the compiler when building. The :func:`sysconfig.get_config_var` function + can be used to determine whether the current running interpreter had the + macro defined. + + Module Initialization ===================== diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 11361289874c9d..8f39b99e38e3a5 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -829,6 +829,12 @@ Kumar Aditya, Edgar Margffoy, and many others. Some of these contributors are employed by Meta, which has continued to provide significant engineering resources to support this project. +From 3.14, when compiling extension modules for the free-threaded build of +CPython on Windows, the preprocessor variable ``Py_GIL_DISABLED`` now needs to +be specified by the build backend, as it will no longer be determined +automatically by the C compiler. For a running interpreter, the setting that +was used at compile time can be found using :func:`sysconfig.get_config_var`. + .. _whatsnew314-pyrepl-highlighting: From 0cb3cb81298df2def3058b713a49f1c0e881a40d Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 13 May 2025 13:46:27 +0100 Subject: [PATCH 3/7] Fix sysconfig --- Lib/sysconfig/__init__.py | 2 +- Lib/test/test_sysconfig.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 68890b45b82008..49e0986517ce97 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -457,7 +457,7 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.dirname(sys._base_executable) + inc_dir = os.path.join(_PROJECT_BASE, 'PC') else: inc_dir = _PROJECT_BASE else: diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 53e55383bf9c72..963cf753ce6178 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -531,13 +531,10 @@ def test_srcdir(self): Python_h = os.path.join(srcdir, 'Include', 'Python.h') self.assertTrue(os.path.exists(Python_h), Python_h) # /PC/pyconfig.h.in always exists even if unused - pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h.in') - self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in) if os.name == 'nt': - # /pyconfig.h exists on Windows in a build tree - pyconfig_h = os.path.join(sys.executable, '..', 'pyconfig.h') + pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h') self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) elif os.name == 'posix': makefile_dir = os.path.dirname(sysconfig.get_makefile_filename()) From 823eea18b84cfb05506c8f3185d77a8c0fd38e03 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 13 May 2025 15:59:40 +0100 Subject: [PATCH 4/7] Typo in comment --- PC/pyconfig.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PC/pyconfig.h b/PC/pyconfig.h index ae2d014467b643..0e8379387cd025 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -104,7 +104,7 @@ WIN32 is still required for the locale module. /* We undefine if it was set to zero because all later checks are #ifdef. * Note that non-Windows builds do not do this, and so every effort should * be made to avoid defining the variable at all when not desired. However, - * sysconfig.get_config_val always returns a 1 or a 0, and so it seems likely + * sysconfig.get_config_var always returns a 1 or a 0, and so it seems likely * that a build backend will define it with the value. */ #if Py_GIL_DISABLED == 0 From 7fbb48cda1144069190a3c92d2d23af916882e3a Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 14 May 2025 16:01:37 +0100 Subject: [PATCH 5/7] Fix MSI --- Tools/msi/dev/dev.wixproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/msi/dev/dev.wixproj b/Tools/msi/dev/dev.wixproj index 4052e4b5368273..544552592ce22e 100644 --- a/Tools/msi/dev/dev.wixproj +++ b/Tools/msi/dev/dev.wixproj @@ -16,7 +16,7 @@ + Exclude="$(PySourcePath)PC\pyconfig.h"> $(PySourcePath) !(bindpath.src) $(PySourcePath) From 142b60aa6ae750e9b6c0b161a97f3237ead59bcb Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 14 May 2025 19:35:30 +0100 Subject: [PATCH 6/7] Fix MSI properly --- Tools/msi/dev/dev.wixproj | 2 +- Tools/msi/dev/dev_files.wxs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/msi/dev/dev.wixproj b/Tools/msi/dev/dev.wixproj index 544552592ce22e..4052e4b5368273 100644 --- a/Tools/msi/dev/dev.wixproj +++ b/Tools/msi/dev/dev.wixproj @@ -16,7 +16,7 @@ + Exclude="$(PySourcePath)include\pyconfig.h"> $(PySourcePath) !(bindpath.src) $(PySourcePath) diff --git a/Tools/msi/dev/dev_files.wxs b/Tools/msi/dev/dev_files.wxs index 4357dc86d9d356..21f9c848cc6be5 100644 --- a/Tools/msi/dev/dev_files.wxs +++ b/Tools/msi/dev/dev_files.wxs @@ -3,7 +3,7 @@ - + From 7a8d6ab087937a33914173c75e349eb9449064e4 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Sat, 17 May 2025 17:38:38 +0100 Subject: [PATCH 7/7] Fix pegen build and suppress MSVC warning in our headers --- Include/Python.h | 13 +++++++++++++ Tools/peg_generator/pegen/build.py | 2 ++ 2 files changed, 15 insertions(+) diff --git a/Include/Python.h b/Include/Python.h index 64be80145890a3..f34d581f0b4c91 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -59,6 +59,14 @@ # include // __readgsqword() #endif +// Suppress known warnings in Python header files. +#if defined(_MSC_VER) +// Warning that alignas behaviour has changed. Doesn't affect us, because we +// never relied on the old behaviour. +#pragma warning(push) +#pragma warning(disable: 5274) +#endif + // Include Python header files #include "pyport.h" #include "pymacro.h" @@ -138,4 +146,9 @@ #include "cpython/pyfpe.h" #include "cpython/tracemalloc.h" +// Restore warning filter +#ifdef _MSC_VER +#pragma warning(pop) +#endif + #endif /* !Py_PYTHON_H */ diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 41338c29bdd9eb..be289c352de585 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -108,6 +108,8 @@ def compile_c_extension( extra_compile_args.append("-DPy_BUILD_CORE_MODULE") # Define _Py_TEST_PEGEN to not call PyAST_Validate() in Parser/pegen.c extra_compile_args.append("-D_Py_TEST_PEGEN") + if sys.platform == "win32" and sysconfig.get_config_var("Py_GIL_DISABLED"): + extra_compile_args.append("-DPy_GIL_DISABLED") extra_link_args = get_extra_flags("LDFLAGS", "PY_LDFLAGS_NODIST") if keep_asserts: extra_compile_args.append("-UNDEBUG")