Skip to content

Commit ecd97ca

Browse files
miss-islingtonZeroIntensity
miss-islington
and
ZeroIntensity
authored
[3.14] gh-136421: Load _datetime static types during interpreter initialization (GH-136583) (GH-136943)
gh-136421: Load `_datetime` static types during interpreter initialization (GH-136583) `_datetime` is a special module, because it's the only non-builtin C extension that contains static types. As such, it would initialize static types in the module's execution function, which can run concurrently. Since static type initialization is not thread-safe, this caused crashes. This fixes it by moving the initialization of `_datetime`'s static types to interpreter startup (where all other static types are initialized), which is already properly protected through other locks. (cherry picked from commit a109606) Co-authored-by: Peter Bierma <zintensitydev@gmail.com>
1 parent 718fc5b commit ecd97ca

File tree

8 files changed

+111
-95
lines changed

8 files changed

+111
-95
lines changed

‎Include/internal/pycore_pylifecycle.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ extern PyStatus _Py_HashRandomization_Init(const PyConfig *);
4141

4242
extern PyStatus _PyGC_Init(PyInterpreterState *interp);
4343
extern PyStatus _PyAtExit_Init(PyInterpreterState *interp);
44+
extern PyStatus _PyDateTime_InitTypes(PyInterpreterState *interp);
4445

4546
/* Various internal finalizers */
4647

‎Lib/test/datetimetester.py

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7275,6 +7275,34 @@ def test_update_type_cache(self):
72757275
""")
72767276
script_helper.assert_python_ok('-c', script)
72777277

7278+
def test_concurrent_initialization_subinterpreter(self):
7279+
# gh-136421: Concurrent initialization of _datetime across multiple
7280+
# interpreters wasn't thread-safe due to its static types.
7281+
7282+
# Run in a subprocess to ensure we get a clean version of _datetime
7283+
script = """if True:
7284+
from concurrent.futures import InterpreterPoolExecutor
7285+
7286+
def func():
7287+
import _datetime
7288+
print('a', end='')
7289+
7290+
with InterpreterPoolExecutor() as executor:
7291+
for _ in range(8):
7292+
executor.submit(func)
7293+
"""
7294+
rc, out, err = script_helper.assert_python_ok("-c", script)
7295+
self.assertEqual(rc, 0)
7296+
self.assertEqual(out, b"a" * 8)
7297+
self.assertEqual(err, b"")
7298+
7299+
# Now test against concurrent reinitialization
7300+
script = "import _datetime\n" + script
7301+
rc, out, err = script_helper.assert_python_ok("-c", script)
7302+
self.assertEqual(rc, 0)
7303+
self.assertEqual(out, b"a" * 8)
7304+
self.assertEqual(err, b"")
7305+
72787306

72797307
def load_tests(loader, standard_tests, pattern):
72807308
standard_tests.addTest(ZoneInfoCompleteTest())

‎Misc/NEWS.d/next/Core_and_Builtins/2025-07-12-09-59-14.gh-issue-136421.ZD1rNj.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix crash when initializing :mod:`datetime` concurrently.

‎Modules/Setup.bootstrap.in

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ posix posixmodule.c
1212
_signal signalmodule.c
1313
_tracemalloc _tracemalloc.c
1414
_suggestions _suggestions.c
15+
# needs libm and on some platforms librt
16+
_datetime _datetimemodule.c
1517

1618
# modules used by importlib, deepfreeze, freeze, runpy, and sysconfig
1719
_codecs _codecsmodule.c

‎Modules/Setup.stdlib.in

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,6 @@
5656
@MODULE_CMATH_TRUE@cmath cmathmodule.c
5757
@MODULE__STATISTICS_TRUE@_statistics _statisticsmodule.c
5858

59-
# needs libm and on some platforms librt
60-
@MODULE__DATETIME_TRUE@_datetime _datetimemodule.c
61-
6259
# _decimal uses libmpdec
6360
# either static libmpdec.a from Modules/_decimal/libmpdec or libmpdec.so
6461
# with ./configure --with-system-libmpdec

‎Modules/_datetimemodule.c

Lines changed: 73 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include "pycore_object.h" // _PyObject_Init()
1515
#include "pycore_time.h" // _PyTime_ObjectToTime_t()
1616
#include "pycore_unicodeobject.h" // _PyUnicode_Copy()
17+
#include "pycore_initconfig.h" // _PyStatus_OK()
1718

1819
#include "datetime.h"
1920

@@ -124,10 +125,9 @@ get_module_state(PyObject *module)
124125
#define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module))
125126

126127
static PyObject *
127-
get_current_module(PyInterpreterState *interp, int *p_reloading)
128+
get_current_module(PyInterpreterState *interp)
128129
{
129130
PyObject *mod = NULL;
130-
int reloading = 0;
131131

132132
PyObject *dict = PyInterpreterState_GetDict(interp);
133133
if (dict == NULL) {
@@ -138,7 +138,6 @@ get_current_module(PyInterpreterState *interp, int *p_reloading)
138138
goto error;
139139
}
140140
if (ref != NULL) {
141-
reloading = 1;
142141
if (ref != Py_None) {
143142
(void)PyWeakref_GetRef(ref, &mod);
144143
if (mod == Py_None) {
@@ -147,9 +146,6 @@ get_current_module(PyInterpreterState *interp, int *p_reloading)
147146
Py_DECREF(ref);
148147
}
149148
}
150-
if (p_reloading != NULL) {
151-
*p_reloading = reloading;
152-
}
153149
return mod;
154150

155151
error:
@@ -163,7 +159,7 @@ static datetime_state *
163159
_get_current_state(PyObject **p_mod)
164160
{
165161
PyInterpreterState *interp = PyInterpreterState_Get();
166-
PyObject *mod = get_current_module(interp, NULL);
162+
PyObject *mod = get_current_module(interp);
167163
if (mod == NULL) {
168164
assert(!PyErr_Occurred());
169165
if (PyErr_Occurred()) {
@@ -4476,7 +4472,7 @@ static PyTypeObject PyDateTime_TimeZoneType = {
44764472
timezone_methods, /* tp_methods */
44774473
0, /* tp_members */
44784474
0, /* tp_getset */
4479-
0, /* tp_base; filled in PyInit__datetime */
4475+
&PyDateTime_TZInfoType, /* tp_base */
44804476
0, /* tp_dict */
44814477
0, /* tp_descr_get */
44824478
0, /* tp_descr_set */
@@ -7131,8 +7127,7 @@ static PyTypeObject PyDateTime_DateTimeType = {
71317127
datetime_methods, /* tp_methods */
71327128
0, /* tp_members */
71337129
datetime_getset, /* tp_getset */
7134-
0, /* tp_base; filled in
7135-
PyInit__datetime */
7130+
&PyDateTime_DateType, /* tp_base */
71367131
0, /* tp_dict */
71377132
0, /* tp_descr_get */
71387133
0, /* tp_descr_set */
@@ -7313,29 +7308,82 @@ clear_state(datetime_state *st)
73137308
}
73147309

73157310

7316-
static int
7317-
init_static_types(PyInterpreterState *interp, int reloading)
7311+
PyStatus
7312+
_PyDateTime_InitTypes(PyInterpreterState *interp)
73187313
{
7319-
if (reloading) {
7320-
return 0;
7321-
}
7322-
7323-
// `&...` is not a constant expression according to a strict reading
7324-
// of C standards. Fill tp_base at run-time rather than statically.
7325-
// See https://bugs.python.org/issue40777
7326-
PyDateTime_TimeZoneType.tp_base = &PyDateTime_TZInfoType;
7327-
PyDateTime_DateTimeType.tp_base = &PyDateTime_DateType;
7328-
73297314
/* Bases classes must be initialized before subclasses,
73307315
* so capi_types must have the types in the appropriate order. */
73317316
for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) {
73327317
PyTypeObject *type = capi_types[i];
73337318
if (_PyStaticType_InitForExtension(interp, type) < 0) {
7334-
return -1;
7319+
return _PyStatus_ERR("could not initialize static types");
73357320
}
73367321
}
73377322

7338-
return 0;
7323+
#define DATETIME_ADD_MACRO(dict, c, value_expr) \
7324+
do { \
7325+
assert(!PyErr_Occurred()); \
7326+
PyObject *value = (value_expr); \
7327+
if (value == NULL) { \
7328+
goto error; \
7329+
} \
7330+
if (PyDict_SetItemString(dict, c, value) < 0) { \
7331+
Py_DECREF(value); \
7332+
goto error; \
7333+
} \
7334+
Py_DECREF(value); \
7335+
} while(0)
7336+
7337+
/* timedelta values */
7338+
PyObject *d = _PyType_GetDict(&PyDateTime_DeltaType);
7339+
DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0));
7340+
DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0));
7341+
DATETIME_ADD_MACRO(d, "max",
7342+
new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0));
7343+
7344+
/* date values */
7345+
d = _PyType_GetDict(&PyDateTime_DateType);
7346+
DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1));
7347+
DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31));
7348+
DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0));
7349+
7350+
/* time values */
7351+
d = _PyType_GetDict(&PyDateTime_TimeType);
7352+
DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0));
7353+
DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0));
7354+
DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0));
7355+
7356+
/* datetime values */
7357+
d = _PyType_GetDict(&PyDateTime_DateTimeType);
7358+
DATETIME_ADD_MACRO(d, "min",
7359+
new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0));
7360+
DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59,
7361+
999999, Py_None, 0));
7362+
DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0));
7363+
7364+
/* timezone values */
7365+
d = _PyType_GetDict(&PyDateTime_TimeZoneType);
7366+
if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) {
7367+
goto error;
7368+
}
7369+
7370+
/* bpo-37642: These attributes are rounded to the nearest minute for backwards
7371+
* compatibility, even though the constructor will accept a wider range of
7372+
* values. This may change in the future.*/
7373+
7374+
/* -23:59 */
7375+
DATETIME_ADD_MACRO(d, "min", create_timezone_from_delta(-1, 60, 0, 1));
7376+
7377+
/* +23:59 */
7378+
DATETIME_ADD_MACRO(
7379+
d, "max", create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0));
7380+
7381+
#undef DATETIME_ADD_MACRO
7382+
7383+
return _PyStatus_OK();
7384+
7385+
error:
7386+
return _PyStatus_NO_MEMORY();
73397387
}
73407388

73417389

@@ -7353,20 +7401,15 @@ _datetime_exec(PyObject *module)
73537401
{
73547402
int rc = -1;
73557403
datetime_state *st = get_module_state(module);
7356-
int reloading = 0;
73577404

73587405
PyInterpreterState *interp = PyInterpreterState_Get();
7359-
PyObject *old_module = get_current_module(interp, &reloading);
7406+
PyObject *old_module = get_current_module(interp);
73607407
if (PyErr_Occurred()) {
73617408
assert(old_module == NULL);
73627409
goto error;
73637410
}
73647411
/* We actually set the "current" module right before a successful return. */
73657412

7366-
if (init_static_types(interp, reloading) < 0) {
7367-
goto error;
7368-
}
7369-
73707413
for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) {
73717414
PyTypeObject *type = capi_types[i];
73727415
const char *name = _PyType_Name(type);
@@ -7380,68 +7423,6 @@ _datetime_exec(PyObject *module)
73807423
goto error;
73817424
}
73827425

7383-
#define DATETIME_ADD_MACRO(dict, c, value_expr) \
7384-
do { \
7385-
assert(!PyErr_Occurred()); \
7386-
PyObject *value = (value_expr); \
7387-
if (value == NULL) { \
7388-
goto error; \
7389-
} \
7390-
if (PyDict_SetItemString(dict, c, value) < 0) { \
7391-
Py_DECREF(value); \
7392-
goto error; \
7393-
} \
7394-
Py_DECREF(value); \
7395-
} while(0)
7396-
7397-
if (!reloading) {
7398-
/* timedelta values */
7399-
PyObject *d = _PyType_GetDict(&PyDateTime_DeltaType);
7400-
DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0));
7401-
DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0));
7402-
DATETIME_ADD_MACRO(d, "max",
7403-
new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0));
7404-
7405-
/* date values */
7406-
d = _PyType_GetDict(&PyDateTime_DateType);
7407-
DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1));
7408-
DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31));
7409-
DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0));
7410-
7411-
/* time values */
7412-
d = _PyType_GetDict(&PyDateTime_TimeType);
7413-
DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0));
7414-
DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0));
7415-
DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0));
7416-
7417-
/* datetime values */
7418-
d = _PyType_GetDict(&PyDateTime_DateTimeType);
7419-
DATETIME_ADD_MACRO(d, "min",
7420-
new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0));
7421-
DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59,
7422-
999999, Py_None, 0));
7423-
DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0));
7424-
7425-
/* timezone values */
7426-
d = _PyType_GetDict(&PyDateTime_TimeZoneType);
7427-
if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) {
7428-
goto error;
7429-
}
7430-
7431-
/* bpo-37642: These attributes are rounded to the nearest minute for backwards
7432-
* compatibility, even though the constructor will accept a wider range of
7433-
* values. This may change in the future.*/
7434-
7435-
/* -23:59 */
7436-
DATETIME_ADD_MACRO(d, "min", create_timezone_from_delta(-1, 60, 0, 1));
7437-
7438-
/* +23:59 */
7439-
DATETIME_ADD_MACRO(
7440-
d, "max", create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0));
7441-
}
7442-
7443-
#undef DATETIME_ADD_MACRO
7444-
74457426
/* Add module level attributes */
74467427
if (PyModule_AddIntMacro(module, MINYEAR) < 0) {
74477428
goto error;

‎PCbuild/_freeze_module.vcxproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@
106106
</ItemGroup>
107107
<ItemGroup>
108108
<ClCompile Include="..\Modules\atexitmodule.c" />
109+
<ClCompile Include="..\Modules\_datetimemodule.c" />
109110
<ClCompile Include="..\Modules\faulthandler.c" />
110111
<ClCompile Include="..\Modules\gcmodule.c" />
111112
<ClCompile Include="..\Modules\getbuildinfo.c" />

‎Python/pylifecycle.c

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -760,6 +760,11 @@ pycore_init_types(PyInterpreterState *interp)
760760
return status;
761761
}
762762

763+
status = _PyDateTime_InitTypes(interp);
764+
if (_PyStatus_EXCEPTION(status)) {
765+
return status;
766+
}
767+
763768
return _PyStatus_OK();
764769
}
765770

0 commit comments

Comments
 (0)