Add Py_mod_gil module slot

python · colesbury · May 3, 2024 · Mar 12, 2024 · Mar 22, 2024 · Mar 27, 2024
commit fa92cb44e86cf8cf4bb8ce28d60c99560f88519e
@@ -411,6 +411,31 @@ The available slot types are:
 
    .. versionadded:: 3.12
 
+.. c:macro: Py_mod_gil
+
+   Specifies one of the following values:
+
+   .. c:macro:: Py_MOD_GIL_USED
+
+      The module assumes the presence of the global interpreter lock (GIL), and
+      may access global state without synchronization.
+
+   .. c:macro:: Py_MOD_GIL_NOT_USED
+
+      The module is safe to run without an active GIL.
+
+   This slot is only used by Python builds configured with
+   :option:`--disable-gil`, and determines whether or not importing this module
+   will cause the GIL to be automatically enabled. See :envvar:`PYTHON_GIL` and
+   :option:`-X gil <-X>` for more detail.
+
+   Multiple ``Py_mod_gil`` slots may not be specified in one module definition.
+
+   If ``Py_mod_gil`` is not specified, the import machinery defaults to
+   ``Py_MOD_GIL_USED``.
+
+   .. versionadded: 3.13
+
 See :PEP:`489` for more details on multi-phase initialization.
 
 Low-level module creation functions

diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
@@ -129,6 +129,15 @@ extern void _PyEval_FiniGIL(PyInterpreterState *interp);
 extern void _PyEval_AcquireLock(PyThreadState *tstate);
 extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *);
 
+#ifdef Py_GIL_DISABLED
+// Enable the GIL for the given thread's interpreter. This may affect other
+// interpreters, if the GIL is shared.
+//
+// Returns 1 if this call enabled the GIL, or 0 if it was already enabled. The
+// caller will hold the GIL upon return.
+extern int _PyEval_EnableGIL(PyThreadState *tstate);
+#endif
+
 extern void _PyEval_DeactivateOpCache(void);
 
 

@@ -76,9 +76,13 @@ struct PyModuleDef_Slot {
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000
 #  define Py_mod_multiple_interpreters 3
 #endif
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030e0000
+#  define Py_mod_gil 4
+#endif
+
 
 #ifndef Py_LIMITED_API
-#define _Py_mod_LAST_SLOT 3
+#define _Py_mod_LAST_SLOT 4
 #endif
 
 #endif /* New in 3.5 */
@@ -90,6 +94,12 @@ struct PyModuleDef_Slot {
 #  define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED ((void *)2)
 #endif
 
+/* for Py_mod_gil: */
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030e0000
+#  define Py_MOD_GIL_USED ((void *)0)
+#  define Py_MOD_GIL_NOT_USED ((void *)1)
+#endif
+
 struct PyModuleDef {
   PyModuleDef_Base m_base;
   const char* m_name;

diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-12-13-51-09.gh-issue-116322.q8TcDQ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-12-13-51-09.gh-issue-116322.q8TcDQ.rst
@@ -0,0 +1,5 @@
+Multi-phase init extension modules may indicate to the runtime that they can
+run without the GIL by providing ``Py_MOD_GIL_NOT_USED`` for the ``Py_mod_gil``
+slot. In ``--disable-gil`` builds, loading extensions that do not provide this
+slot will enable the GIL for the remainder of the current interpreter, unless
+the GIL was explicitly disabled by ``PYTHON_GIL=0`` or ``-Xgil=0``.
@@ -3,6 +3,8 @@
 
 #include "Python.h"
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
+#include "pycore_ceval.h"         // _PyEval_EnableGIL()
+#include "pycore_initconfig.h"    // _PyConfig_GIL_DEFAULT
 #include "pycore_interp.h"        // PyInterpreterState.importlib
 #include "pycore_modsupport.h"    // _PyModule_CreateInitialized()
 #include "pycore_moduleobject.h"  // _PyModule_GetDef()
@@ -249,6 +251,8 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
     PyObject *m = NULL;
     int has_multiple_interpreters_slot = 0;
     void *multiple_interpreters = (void *)0;
+    int has_gil_slot = 0;
+    void *gil_slot = (void *)Py_MOD_GIL_USED;
     int has_execution_slots = 0;
     const char *name;
     int ret;
@@ -303,6 +307,17 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
                 multiple_interpreters = cur_slot->value;
                 has_multiple_interpreters_slot = 1;
                 break;
+            case Py_mod_gil:
+                if (has_gil_slot) {
+                    PyErr_Format(
+                       PyExc_SystemError,
+                       "module %s has more than one 'gil' slot",
+                       name);
+                    goto error;
+                }
+                gil_slot = cur_slot->value;
+                has_gil_slot = 1;
+                break;
             default:
                 assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT);
                 PyErr_Format(
@@ -333,6 +348,27 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
         goto error;
     }
 
+#ifdef Py_GIL_DISABLED
+    PyThreadState *tstate = _PyThreadState_GET();
+    const PyConfig *config = _PyInterpreterState_GetConfig(interp);
+    if (gil_slot == Py_MOD_GIL_USED && config->enable_gil == _PyConfig_GIL_DEFAULT) {
+        if (_PyEval_EnableGIL(tstate)) {
+            PyErr_WarnFormat(
+                PyExc_RuntimeWarning,
+                1,
+                "The global interpreter lock (GIL) has been enabled to load "
+                "module '%s', which has not declared that it can run safely "
+                "without the GIL. To override this behavior and keep the GIL "
+                "disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0.",
+                name
+            );
+        }
+        if (config->verbose) {
+            PySys_FormatStderr("# loading module '%s', which requires the GIL\n", name);
+        }
+    }
+#endif
+
     if (create) {
         m = create(spec, def);
         if (m == NULL) {
@@ -458,6 +494,7 @@ PyModule_ExecDef(PyObject *module, PyModuleDef *def)
                 }
                 break;
             case Py_mod_multiple_interpreters:
+            case Py_mod_gil:
                 /* handled in PyModule_FromDefAndSpec2 */
                 break;
             default:

diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c
@@ -969,6 +969,51 @@ _PyEval_InitState(PyInterpreterState *interp)
     _gil_initialize(&interp->_gil);
 }
 
+#ifdef Py_GIL_DISABLED
+int
+_PyEval_EnableGIL(PyThreadState *tstate)
+{
+    struct _gil_runtime_state *gil = tstate->interp->ceval.gil;
+
+    // gil->enabled only transitions from 0 to 1, and only while the world is
+    // stopped, so this can be read without any synchronization.
+    if (gil->enabled) {
+        return 0;
+    }
+
+    // Enabling the GIL changes what it means to be an "attached" thread. To
+    // safely make this transition, we:
+    // 1. Detach the current thread.
+    // 2. Stop the world to detach (and suspend) all other threads.
+    // 3. Enable the GIL, if nobody else did between our check above and when
+    //    our stop-the-world begins.
+    // 4. Start the world.
+    // 5. Attach the current thread. Other threads may attach and hold the GIL
+    //    before this thread, which is harmless.
+    _PyThreadState_Detach(tstate);
+
+    // This could be an interpreter-local stop-the-world in situations where we
+    // know that this interpreter's GIL is not shared (and that it won't become
+    // shared before the stop-the-world begins). This operation will happen at
+    // most once per interpreter, though, so we always stop the whole world to
+    // keep things simpler.
+    _PyEval_StopTheWorldAll(&_PyRuntime);
+
+    int this_thread_enabled = 1;
+    if (gil->enabled) {
+        // A different thread enabled the GIL since our check above.
+        this_thread_enabled = 0;
+    } else {
+        gil->enabled = 1;
+    }
+
+    _PyEval_StartTheWorldAll(&_PyRuntime);
+    _PyThreadState_Attach(tstate);
+
+    return this_thread_enabled;
+}
+#endif
+
 
 /* Do periodic things, like check for signals and async I/0.
 * We need to do reasonably frequently, but not too frequently.