Skip to content

gh-132413: Fix crash in _datetime when used at shutdown (alt) #132665

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 68 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
68 commits
Select commit Hold shift + click to select a range
f99a3d9
test
neonene Apr 18, 2025
f502244
Create a module at shutdown rather than import
neonene Apr 18, 2025
528882d
NEWS
neonene Apr 18, 2025
72991c2
Fix comment
neonene Apr 18, 2025
4abbc73
Remove unused code
neonene Apr 18, 2025
db2cb88
Remove PyDict_Contains()
neonene Apr 18, 2025
7225ec2
Add assertion in test
neonene Apr 18, 2025
e5900ef
Merge branch 'main' into 132413-take2
neonene Apr 18, 2025
22c4764
Correct PyModule_GetDict() usage
neonene Apr 18, 2025
b9eaee1
Fix comment
neonene Apr 18, 2025
97a6d0d
Set a spec later in set_current_module()
neonene Apr 18, 2025
c10d4db
Hold current module until interp-end
neonene Apr 20, 2025
029a032
Merge branch 'main' into 132413-take2
neonene Apr 20, 2025
b7d27ea
Fix decl
neonene Apr 20, 2025
3fba892
assert interp-dict has a key
neonene Apr 21, 2025
d5e1220
Reword
neonene Apr 21, 2025
c0a27eb
typo
neonene Apr 21, 2025
aabe338
Merge branch 'main' into 132413-take2
neonene Apr 21, 2025
bf3d238
Reword again
neonene Apr 21, 2025
0f59936
ditto
neonene Apr 22, 2025
78c762a
Check if interp-dict is empty
neonene Apr 22, 2025
8503986
Correct behavior
neonene Apr 23, 2025
4b92b19
Revert _datetimemodule.c to main
neonene Apr 23, 2025
3d29380
Fix crash (new attempt)
neonene Apr 23, 2025
26b8f51
Non-NULL check before PyType_Check()
neonene Apr 23, 2025
3e2ef0d
Faster _get_current_state() by 6%-
neonene Apr 23, 2025
2e1a129
Comment
neonene Apr 23, 2025
3c05989
assert(!_Py_IsInterpreterFinalizing())
neonene Apr 24, 2025
8eaae36
assign NULL to interp-state on error
neonene Apr 24, 2025
b285b46
Merge branch 'main' into 132413-take2
neonene Apr 24, 2025
c2758bd
Get module without interp-dict and weakref
neonene Apr 25, 2025
2f8535b
Merge branch 'main' into 132413-take2
neonene Apr 25, 2025
56f8a06
Add tests
neonene Apr 25, 2025
aa86807
Add tests
neonene Apr 25, 2025
6880f34
Update and pass tests
neonene Apr 26, 2025
022d4e8
Make tests realistic by using subinterp
neonene Apr 27, 2025
318888a
Nit
neonene Apr 27, 2025
838a2f2
Merge branch 'main' into 132413-take2
neonene Apr 27, 2025
6e6c328
Make tests generic
neonene Apr 27, 2025
351225e
Fix test_datetime_capi() for subinterps
neonene Apr 27, 2025
419500e
Use assert_python_failure()
neonene Apr 28, 2025
596007c
Add a failure test (crash)
neonene Apr 28, 2025
1dd9707
Correct the previous commit
neonene Apr 28, 2025
fb35703
Redirect to the CapiTest method
neonene Apr 28, 2025
c6cc066
Merge two tests (1)
neonene Apr 28, 2025
cc04c6f
Merge two tests (2)
neonene Apr 28, 2025
56267bd
Merge two tests (3)
neonene Apr 28, 2025
063f37d
Add test cases
neonene Apr 28, 2025
8a81b18
Merge branch 'main' into 132413-take2
neonene Apr 28, 2025
992fd0c
Move up a few tests
neonene Apr 30, 2025
f0f8fa0
Update tests
neonene Apr 30, 2025
7f31245
Move a repeat test to test_embed
neonene Apr 30, 2025
0b01f51
Merge branch 'main' into 132413-take2
neonene Apr 30, 2025
d7f80dd
Restore and keep _Py_ID for now
neonene Apr 30, 2025
23bdffd
Add a demonstration to test_embed
neonene Apr 30, 2025
68787de
Merge branch 'main' into 132413-take2
neonene Apr 30, 2025
ad7dfd2
Introduce test_datetime_capi_newinterp()
neonene Apr 30, 2025
37b2774
Cleanup
neonene May 1, 2025
f21f03f
Add a test to test_embed
neonene May 1, 2025
b009033
Merge branch 'main' into 132413-take2
neonene May 1, 2025
6a83ad7
Merge branch 'main' into 132413-take2
neonene May 8, 2025
2b6c12a
Decref the module explicitly on exec error
neonene May 10, 2025
8d94b2f
Focus on main interp's issue
neonene May 10, 2025
e1c9cdf
Tweak 2b6c12a
neonene May 11, 2025
5ed5e69
Smaller patch for tests
neonene May 13, 2025
d9121b1
Merge branch 'main' into 132413-take2
neonene May 13, 2025
f22edc1
minimize test
neonene Jun 23, 2025
7b417a3
Merge branch 'main' into 132413-take2
neonene Jun 23, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Include/internal/pycore_interp_structs.h
Original file line number Diff line number Diff line change
Expand Up @@ -963,6 +963,8 @@ struct _is {

Py_ssize_t _interactive_src_count;

void *datetime_module_state;

#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
uint64_t next_stackref;
_Py_hashtable_t *open_stackrefs_table;
Expand Down
19 changes: 19 additions & 0 deletions Lib/test/datetimetester.py
Original file line number Diff line number Diff line change
Expand Up @@ -7295,6 +7295,25 @@ def test_update_type_cache(self):
""")
script_helper.assert_python_ok('-c', script)

def test_static_type_at_shutdown(self):
# gh-132413
script = textwrap.dedent("""
import _datetime
timedelta = _datetime.timedelta

def gen():
try:
yield
finally:
# sys.modules is empty
_datetime.timedelta(days=1)
timedelta(days=1)

it = gen()
next(it)
""")
script_helper.assert_python_ok('-c', script)


def load_tests(loader, standard_tests, pattern):
standard_tests.addTest(ZoneInfoCompleteTest())
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix crash in C version of :mod:`datetime` when used during interpreter shutdown.
140 changes: 51 additions & 89 deletions Modules/_datetimemodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ static PyTypeObject PyDateTime_TimeZoneType;


typedef struct {
/* Corresponding module that is referenced via the interpreter state. */
PyObject *module;

/* Module heap types. */
PyTypeObject *isocalendar_date_type;

Expand Down Expand Up @@ -121,62 +124,40 @@ get_module_state(PyObject *module)
}


#define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module))
#define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module)) // unused

static PyObject *
get_current_module(PyInterpreterState *interp, int *p_reloading)
{
PyObject *mod = NULL;
int reloading = 0;

PyObject *dict = PyInterpreterState_GetDict(interp);
if (dict == NULL) {
goto error;
}
PyObject *ref = NULL;
if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) {
goto error;
}
if (ref != NULL) {
reloading = 1;
if (ref != Py_None) {
(void)PyWeakref_GetRef(ref, &mod);
if (mod == Py_None) {
Py_CLEAR(mod);
}
Py_DECREF(ref);
}
}
datetime_state *st = interp->datetime_module_state;
if (p_reloading != NULL) {
*p_reloading = reloading;
*p_reloading = st != NULL ? 1 : 0;
}
if (st != NULL && st != (void *)Py_None && st->module != NULL) {
assert(PyModule_CheckExact(st->module));
return Py_NewRef(st->module);
}
return mod;

error:
assert(PyErr_Occurred());
return NULL;
}

static PyModuleDef datetimemodule;

static datetime_state *
_get_current_state(PyObject **p_mod)
{
PyInterpreterState *interp = PyInterpreterState_Get();
PyObject *mod = get_current_module(interp, NULL);
datetime_state *st = interp->datetime_module_state;
if (st != NULL && st != (void *)Py_None && st->module != NULL) {
assert(PyModule_CheckExact(st->module));
*p_mod = Py_NewRef(st->module);
return st;
}

/* The static types can outlive the module,
* so we must re-import the module. */
PyObject *mod = PyImport_ImportModule("_datetime");
if (mod == NULL) {
assert(!PyErr_Occurred());
if (PyErr_Occurred()) {
return NULL;
}
/* The static types can outlive the module,
* so we must re-import the module. */
mod = PyImport_ImportModule("_datetime");
if (mod == NULL) {
return NULL;
}
return NULL;
}
datetime_state *st = get_module_state(mod);
st = get_module_state(mod);
*p_mod = mod;
return st;
}
Expand All @@ -190,61 +171,18 @@ static int
set_current_module(PyInterpreterState *interp, PyObject *mod)
{
assert(mod != NULL);
PyObject *dict = PyInterpreterState_GetDict(interp);
if (dict == NULL) {
return -1;
}
PyObject *ref = PyWeakref_NewRef(mod, NULL);
if (ref == NULL) {
return -1;
}
int rc = PyDict_SetItem(dict, INTERP_KEY, ref);
Py_DECREF(ref);
return rc;
interp->datetime_module_state = get_module_state(mod);
return 0;
}

static void
clear_current_module(PyInterpreterState *interp, PyObject *expected)
{
PyObject *exc = PyErr_GetRaisedException();

PyObject *dict = PyInterpreterState_GetDict(interp);
if (dict == NULL) {
goto error;
}

if (expected != NULL) {
PyObject *ref = NULL;
if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) {
goto error;
}
if (ref != NULL) {
PyObject *current = NULL;
int rc = PyWeakref_GetRef(ref, &current);
/* We only need "current" for pointer comparison. */
Py_XDECREF(current);
Py_DECREF(ref);
if (rc < 0) {
goto error;
}
if (current != expected) {
goto finally;
}
}
if (expected && get_module_state(expected) != interp->datetime_module_state) {
return;
}

/* We use None to identify that the module was previously loaded. */
if (PyDict_SetItem(dict, INTERP_KEY, Py_None) < 0) {
goto error;
}

goto finally;

error:
PyErr_FormatUnraisable("Exception ignored while clearing _datetime module");

finally:
PyErr_SetRaisedException(exc);
interp->datetime_module_state = Py_None;
}


Expand Down Expand Up @@ -2115,6 +2053,9 @@ delta_to_microseconds(PyDateTime_Delta *self)

PyObject *current_mod = NULL;
datetime_state *st = GET_CURRENT_STATE(current_mod);
if (current_mod == NULL) {
return NULL;
}

x1 = PyLong_FromLong(GET_TD_DAYS(self));
if (x1 == NULL)
Expand Down Expand Up @@ -2194,6 +2135,9 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type)

PyObject *current_mod = NULL;
datetime_state *st = GET_CURRENT_STATE(current_mod);
if (current_mod == NULL) {
return NULL;
}

tuple = checked_divmod(pyus, CONST_US_PER_SECOND(st));
if (tuple == NULL) {
Expand Down Expand Up @@ -2779,6 +2723,9 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw)

PyObject *current_mod = NULL;
datetime_state *st = GET_CURRENT_STATE(current_mod);
if (current_mod == NULL) {
return NULL;
}

/* Argument objects. */
PyObject *day = NULL;
Expand Down Expand Up @@ -2998,6 +2945,9 @@ delta_total_seconds(PyObject *op, PyObject *Py_UNUSED(dummy))

PyObject *current_mod = NULL;
datetime_state *st = GET_CURRENT_STATE(current_mod);
if (current_mod == NULL) {
return NULL;
}

total_seconds = PyNumber_TrueDivide(total_microseconds, CONST_US_PER_SECOND(st));

Expand Down Expand Up @@ -3781,6 +3731,9 @@ date_isocalendar(PyObject *self, PyObject *Py_UNUSED(dummy))

PyObject *current_mod = NULL;
datetime_state *st = GET_CURRENT_STATE(current_mod);
if (current_mod == NULL) {
return NULL;
}

PyObject *v = iso_calendar_date_new_impl(ISOCALENDAR_DATE_TYPE(st),
year, week + 1, day + 1);
Expand Down Expand Up @@ -6616,6 +6569,9 @@ local_timezone(PyDateTime_DateTime *utc_time)

PyObject *current_mod = NULL;
datetime_state *st = GET_CURRENT_STATE(current_mod);
if (current_mod == NULL) {
return NULL;
}

delta = datetime_subtract((PyObject *)utc_time, CONST_EPOCH(st));
RELEASE_CURRENT_STATE(st, current_mod);
Expand Down Expand Up @@ -6860,6 +6816,9 @@ datetime_timestamp(PyObject *op, PyObject *Py_UNUSED(dummy))
if (HASTZINFO(self) && self->tzinfo != Py_None) {
PyObject *current_mod = NULL;
datetime_state *st = GET_CURRENT_STATE(current_mod);
if (current_mod == NULL) {
return NULL;
}

PyObject *delta;
delta = datetime_subtract(op, CONST_EPOCH(st));
Expand Down Expand Up @@ -7501,6 +7460,7 @@ _datetime_exec(PyObject *module)
if (set_current_module(interp, module) < 0) {
goto error;
}
st->module = Py_NewRef(module);

rc = 0;
goto finally;
Expand All @@ -7524,6 +7484,7 @@ static int
module_traverse(PyObject *mod, visitproc visit, void *arg)
{
datetime_state *st = get_module_state(mod);
Py_VISIT(st->module);
traverse_state(st, visit, arg);
return 0;
}
Expand All @@ -7532,6 +7493,7 @@ static int
module_clear(PyObject *mod)
{
datetime_state *st = get_module_state(mod);
Py_CLEAR(st->module);
clear_state(st);

PyInterpreterState *interp = PyInterpreterState_Get();
Expand Down
Loading