From f99a3d976cf1e485531fa7f9820c9196806aaaf0 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 09:22:27 +0900 Subject: [PATCH 01/55] test --- Lib/test/datetimetester.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index ecb37250ceb6c4..200f8489feb020 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7269,7 +7269,28 @@ def test_update_type_cache(self): assert isinstance(_datetime.timezone.utc, _datetime.tzinfo) del sys.modules['_datetime'] """) - script_helper.assert_python_ok('-c', script) + res = script_helper.assert_python_ok('-c', script) + self.assertFalse(res.err) + + def test_module_state_at_shutdown(self): + # gh-132413 + script = textwrap.dedent(""" + import sys + import _datetime + + def gen(): + try: + yield + finally: + assert not sys.modules + td = _datetime.timedelta(days=1) # crash + assert td.days == 1 + + it = gen() + next(it) + """) + res = script_helper.assert_python_ok('-c', script) + self.assertFalse(res.err) def load_tests(loader, standard_tests, pattern): From f502244db3ae6c7aaabf1f822059b534e4d7a497 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 09:25:52 +0900 Subject: [PATCH 02/55] Create a module at shutdown rather than import --- Modules/_datetimemodule.c | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 9bba0e3354b26b..1e0b881f99cffe 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -158,6 +158,7 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) } static PyModuleDef datetimemodule; +static int set_current_module(PyInterpreterState *interp, PyObject *mod); static datetime_state * _get_current_state(PyObject **p_mod) @@ -173,7 +174,28 @@ _get_current_state(PyObject **p_mod) * so we must re-import the module. */ mod = PyImport_ImportModule("_datetime"); if (mod == NULL) { - return NULL; + PyErr_Clear(); + /* Create a module at shutdown */ + PyObject *dict = PyInterpreterState_GetDict(interp); + if (dict == NULL) { + return NULL; + } + PyObject *spec; + if (PyDict_GetItemStringRef(dict, "datetime_module_spec", &spec) != 1) { + return NULL; + } + mod = PyModule_FromDefAndSpec(&datetimemodule, spec); + if (mod == NULL) { + Py_DECREF(spec); + return NULL; + } + Py_DECREF(spec); + + /* The module will be held by heaptypes. Prefer + /* it not to be cached in the interp-dict. */ + if (PyModule_ExecDef(mod, &datetimemodule) < 0) { + return NULL; + } } } datetime_state *st = get_module_state(mod); @@ -198,6 +220,19 @@ set_current_module(PyInterpreterState *interp, PyObject *mod) if (ref == NULL) { return -1; } + + if (!PyDict_ContainsString(dict, "datetime_module_spec")) { + PyObject *spec; + if (PyDict_GetItemRef(PyModule_GetDict(mod), &_Py_ID(__spec__), &spec) != 1) { + return -1; + } + if (PyDict_SetItemString(dict, "datetime_module_spec", spec) < 0) { + Py_DECREF(spec); + return -1; + } + Py_DECREF(spec); + } + int rc = PyDict_SetItem(dict, INTERP_KEY, ref); Py_DECREF(ref); return rc; From 528882d1a04a47158b7a77306c58961a0aa27f35 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 09:27:54 +0900 Subject: [PATCH 03/55] NEWS --- .../next/Library/2025-04-16-14-34-04.gh-issue-132413.TvpIn2.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2025-04-16-14-34-04.gh-issue-132413.TvpIn2.rst diff --git a/Misc/NEWS.d/next/Library/2025-04-16-14-34-04.gh-issue-132413.TvpIn2.rst b/Misc/NEWS.d/next/Library/2025-04-16-14-34-04.gh-issue-132413.TvpIn2.rst new file mode 100644 index 00000000000000..3e778bf54f8e02 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-16-14-34-04.gh-issue-132413.TvpIn2.rst @@ -0,0 +1 @@ +Fix crash in C version of :mod:`datetime` when used during interpreter shutdown. From 72991c2c24f6b0537ab8c95a669dd5c5382f2fec Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 09:45:43 +0900 Subject: [PATCH 04/55] Fix comment --- Modules/_datetimemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 1e0b881f99cffe..d42b3490854bdf 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -192,7 +192,7 @@ _get_current_state(PyObject **p_mod) Py_DECREF(spec); /* The module will be held by heaptypes. Prefer - /* it not to be cached in the interp-dict. */ + * it not to be cached in the interp-dict. */ if (PyModule_ExecDef(mod, &datetimemodule) < 0) { return NULL; } From 4abbc7374f5030a5910461fe400cd5c482437b9d Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 10:13:38 +0900 Subject: [PATCH 05/55] Remove unused code --- Modules/_datetimemodule.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index d42b3490854bdf..e5dc2afe2528f2 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -158,7 +158,6 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) } static PyModuleDef datetimemodule; -static int set_current_module(PyInterpreterState *interp, PyObject *mod); static datetime_state * _get_current_state(PyObject **p_mod) @@ -192,7 +191,7 @@ _get_current_state(PyObject **p_mod) Py_DECREF(spec); /* The module will be held by heaptypes. Prefer - * it not to be cached in the interp-dict. */ + * it not to be stored in the interp-dict. */ if (PyModule_ExecDef(mod, &datetimemodule) < 0) { return NULL; } From db2cb883806b84b0116c9e6f2b38695cb39e417b Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 10:48:23 +0900 Subject: [PATCH 06/55] Remove PyDict_Contains() --- Modules/_datetimemodule.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index e5dc2afe2528f2..1b15baaf799099 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -219,18 +219,16 @@ set_current_module(PyInterpreterState *interp, PyObject *mod) if (ref == NULL) { return -1; } - - if (!PyDict_ContainsString(dict, "datetime_module_spec")) { - PyObject *spec; - if (PyDict_GetItemRef(PyModule_GetDict(mod), &_Py_ID(__spec__), &spec) != 1) { - return -1; - } - if (PyDict_SetItemString(dict, "datetime_module_spec", spec) < 0) { - Py_DECREF(spec); - return -1; - } + /* A module spec remains in the dict */ + PyObject *spec; + if (PyDict_GetItemRef(PyModule_GetDict(mod), &_Py_ID(__spec__), &spec) != 1) { + return -1; + } + if (PyDict_SetItemString(dict, "datetime_module_spec", spec) < 0) { Py_DECREF(spec); + return -1; } + Py_DECREF(spec); int rc = PyDict_SetItem(dict, INTERP_KEY, ref); Py_DECREF(ref); From 7225ec267517f23f4225a9c6d8e0324cd94c36df Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 18:20:26 +0900 Subject: [PATCH 07/55] Add assertion in test --- Lib/test/datetimetester.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 200f8489feb020..03b80607ee6713 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7285,6 +7285,7 @@ def gen(): assert not sys.modules td = _datetime.timedelta(days=1) # crash assert td.days == 1 + assert not sys.modules it = gen() next(it) From 22c4764584ee97d2ce3e8a0343ef41ccb95165b2 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 19:04:32 +0900 Subject: [PATCH 08/55] Correct PyModule_GetDict() usage --- Modules/_datetimemodule.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 1b15baaf799099..ed31c79ce6c4c6 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -191,7 +191,7 @@ _get_current_state(PyObject **p_mod) Py_DECREF(spec); /* The module will be held by heaptypes. Prefer - * it not to be stored in the interp-dict. */ + * it not to be stored in the interpreter's dict. */ if (PyModule_ExecDef(mod, &datetimemodule) < 0) { return NULL; } @@ -219,9 +219,13 @@ set_current_module(PyInterpreterState *interp, PyObject *mod) if (ref == NULL) { return -1; } - /* A module spec remains in the dict */ + /* A module spec remains in the interpreter's dict. */ + PyObject *mod_dict = PyModule_GetDict(mod); + if (mod_dict == NULL) { + return -1; + } PyObject *spec; - if (PyDict_GetItemRef(PyModule_GetDict(mod), &_Py_ID(__spec__), &spec) != 1) { + if (PyDict_GetItemRef(mod_dict, &_Py_ID(__spec__), &spec) != 1) { return -1; } if (PyDict_SetItemString(dict, "datetime_module_spec", spec) < 0) { From b9eaee159019eccd8e38ffac6168cda89f599a6a Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 19:46:38 +0900 Subject: [PATCH 09/55] Fix comment --- Modules/_datetimemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index ed31c79ce6c4c6..23bc5360de56d7 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -219,7 +219,7 @@ set_current_module(PyInterpreterState *interp, PyObject *mod) if (ref == NULL) { return -1; } - /* A module spec remains in the interpreter's dict. */ + /* Module spec remains in the interpreter's dict. */ PyObject *mod_dict = PyModule_GetDict(mod); if (mod_dict == NULL) { return -1; From 97a6d0d7824ba4c693fdddaa7fd16f7c77a2d865 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sat, 19 Apr 2025 04:23:51 +0900 Subject: [PATCH 10/55] Set a spec later in set_current_module() --- Modules/_datetimemodule.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 23bc5360de56d7..9736ed795196da 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -219,7 +219,14 @@ set_current_module(PyInterpreterState *interp, PyObject *mod) if (ref == NULL) { return -1; } - /* Module spec remains in the interpreter's dict. */ + if (PyDict_SetItem(dict, INTERP_KEY, ref) < 0) { + Py_DECREF(ref); + return -1; + } + Py_DECREF(ref); + + /* Make the module spec remain in the interpreter's dict. Not required, + * but reserve the new one for memory efficiency. */ PyObject *mod_dict = PyModule_GetDict(mod); if (mod_dict == NULL) { return -1; @@ -233,10 +240,7 @@ set_current_module(PyInterpreterState *interp, PyObject *mod) return -1; } Py_DECREF(spec); - - int rc = PyDict_SetItem(dict, INTERP_KEY, ref); - Py_DECREF(ref); - return rc; + return 0; } static void From c10d4dbb31e048c252e5e7eb022ed0a10f2f871b Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 21 Apr 2025 07:51:21 +0900 Subject: [PATCH 11/55] Hold current module until interp-end --- Include/internal/pycore_interp_structs.h | 2 + Modules/_datetimemodule.c | 179 +++++++---------------- 2 files changed, 55 insertions(+), 126 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 573b56a57e1d54..a0901386434440 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -947,6 +947,8 @@ struct _is { _Py_hashtable_t *closed_stackrefs_table; # endif #endif + + void *datetime_module_state; }; diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 9736ed795196da..81b5000a841e1c 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -49,6 +49,10 @@ typedef struct { /* The interned Unix epoch datetime instance */ PyObject *epoch; + + /* Reference to the interpreter's dict where the current module will be + * reserved even after the referent dict becomes NULL at shutdown. */ + PyObject *interp_dict; } datetime_state; /* The module has a fixed number of static objects, due to being exposed @@ -133,18 +137,13 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) if (dict == NULL) { goto error; } - PyObject *ref = NULL; - if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { + if (PyDict_GetItemRef(dict, INTERP_KEY, &mod) < 0) { goto error; } - if (ref != NULL) { + if (mod != NULL) { reloading = 1; - if (ref != Py_None) { - (void)PyWeakref_GetRef(ref, &mod); - if (mod == Py_None) { - Py_CLEAR(mod); - } - Py_DECREF(ref); + if (mod == Py_None) { + mod = NULL; } } if (p_reloading != NULL) { @@ -159,117 +158,51 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) static PyModuleDef datetimemodule; -static datetime_state * -_get_current_state(PyObject **p_mod) +static inline datetime_state * +get_current_state() { PyInterpreterState *interp = PyInterpreterState_Get(); - PyObject *mod = get_current_module(interp, NULL); - if (mod == NULL) { - assert(!PyErr_Occurred()); - if (PyErr_Occurred()) { - return NULL; - } - /* The static types can outlive the module, - * so we must re-import the module. */ - mod = PyImport_ImportModule("_datetime"); - if (mod == NULL) { - PyErr_Clear(); - /* Create a module at shutdown */ - PyObject *dict = PyInterpreterState_GetDict(interp); - if (dict == NULL) { - return NULL; - } - PyObject *spec; - if (PyDict_GetItemStringRef(dict, "datetime_module_spec", &spec) != 1) { - return NULL; - } - mod = PyModule_FromDefAndSpec(&datetimemodule, spec); - if (mod == NULL) { - Py_DECREF(spec); - return NULL; - } - Py_DECREF(spec); - - /* The module will be held by heaptypes. Prefer - * it not to be stored in the interpreter's dict. */ - if (PyModule_ExecDef(mod, &datetimemodule) < 0) { - return NULL; - } - } - } - datetime_state *st = get_module_state(mod); - *p_mod = mod; - return st; + void *state = interp->datetime_module_state; + assert(state != NULL); + return (datetime_state *)state; } -#define GET_CURRENT_STATE(MOD_VAR) \ - _get_current_state(&MOD_VAR) -#define RELEASE_CURRENT_STATE(ST_VAR, MOD_VAR) \ - Py_DECREF(MOD_VAR) - static int -set_current_module(PyInterpreterState *interp, PyObject *mod) +set_current_module(datetime_state *st, + PyInterpreterState *interp, PyObject *mod) { assert(mod != NULL); - PyObject *dict = PyInterpreterState_GetDict(interp); + PyObject *dict = st->interp_dict; if (dict == NULL) { return -1; } - PyObject *ref = PyWeakref_NewRef(mod, NULL); - if (ref == NULL) { - return -1; - } - if (PyDict_SetItem(dict, INTERP_KEY, ref) < 0) { - Py_DECREF(ref); - return -1; - } - Py_DECREF(ref); - - /* Make the module spec remain in the interpreter's dict. Not required, - * but reserve the new one for memory efficiency. */ - PyObject *mod_dict = PyModule_GetDict(mod); - if (mod_dict == NULL) { - return -1; - } - PyObject *spec; - if (PyDict_GetItemRef(mod_dict, &_Py_ID(__spec__), &spec) != 1) { - return -1; - } - if (PyDict_SetItemString(dict, "datetime_module_spec", spec) < 0) { - Py_DECREF(spec); + if (PyDict_SetItem(dict, INTERP_KEY, mod) < 0) { return -1; } - Py_DECREF(spec); + interp->datetime_module_state = st; return 0; } static void -clear_current_module(PyInterpreterState *interp, PyObject *expected) +clear_current_module(datetime_state *st, + PyInterpreterState *interp, PyObject *expected) { - PyObject *exc = PyErr_GetRaisedException(); - - PyObject *dict = PyInterpreterState_GetDict(interp); + PyObject *dict = st->interp_dict; if (dict == NULL) { - goto error; + return; /* Already cleared */ } + PyObject *exc = PyErr_GetRaisedException(); + if (expected != NULL) { - PyObject *ref = NULL; - if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { + PyObject *current; + if (PyDict_GetItemRef(dict, INTERP_KEY, ¤t) < 0) { goto error; } - if (ref != NULL) { - PyObject *current = NULL; - int rc = PyWeakref_GetRef(ref, ¤t); - /* We only need "current" for pointer comparison. */ - Py_XDECREF(current); - Py_DECREF(ref); - if (rc < 0) { - goto error; - } - if (current != expected) { - goto finally; - } + /* We only need "current" for pointer comparison. */ + Py_XDECREF(current); + if (current != expected) { + goto finally; } } @@ -277,6 +210,7 @@ clear_current_module(PyInterpreterState *interp, PyObject *expected) if (PyDict_SetItem(dict, INTERP_KEY, Py_None) < 0) { goto error; } + interp->datetime_module_state = NULL; goto finally; @@ -2147,8 +2081,7 @@ delta_to_microseconds(PyDateTime_Delta *self) PyObject *x3 = NULL; PyObject *result = NULL; - PyObject *current_mod = NULL; - datetime_state *st = GET_CURRENT_STATE(current_mod); + datetime_state *st = get_current_state(); x1 = PyLong_FromLong(GET_TD_DAYS(self)); if (x1 == NULL) @@ -2186,7 +2119,6 @@ delta_to_microseconds(PyDateTime_Delta *self) Py_XDECREF(x1); Py_XDECREF(x2); Py_XDECREF(x3); - RELEASE_CURRENT_STATE(st, current_mod); return result; } @@ -2226,8 +2158,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) PyObject *num = NULL; PyObject *result = NULL; - PyObject *current_mod = NULL; - datetime_state *st = GET_CURRENT_STATE(current_mod); + datetime_state *st = get_current_state(); tuple = checked_divmod(pyus, CONST_US_PER_SECOND(st)); if (tuple == NULL) { @@ -2272,7 +2203,6 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) Done: Py_XDECREF(tuple); Py_XDECREF(num); - RELEASE_CURRENT_STATE(st, current_mod); return result; BadDivmod: @@ -2811,8 +2741,7 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) { PyObject *self = NULL; - PyObject *current_mod = NULL; - datetime_state *st = GET_CURRENT_STATE(current_mod); + datetime_state *st = get_current_state(); /* Argument objects. */ PyObject *day = NULL; @@ -2917,7 +2846,6 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) Py_DECREF(x); Done: - RELEASE_CURRENT_STATE(st, current_mod); return self; #undef CLEANUP @@ -3030,12 +2958,10 @@ delta_total_seconds(PyObject *op, PyObject *Py_UNUSED(dummy)) if (total_microseconds == NULL) return NULL; - PyObject *current_mod = NULL; - datetime_state *st = GET_CURRENT_STATE(current_mod); + datetime_state *st = get_current_state(); total_seconds = PyNumber_TrueDivide(total_microseconds, CONST_US_PER_SECOND(st)); - RELEASE_CURRENT_STATE(st, current_mod); Py_DECREF(total_microseconds); return total_seconds; } @@ -3813,12 +3739,10 @@ date_isocalendar(PyObject *self, PyObject *Py_UNUSED(dummy)) week = 0; } - PyObject *current_mod = NULL; - datetime_state *st = GET_CURRENT_STATE(current_mod); + datetime_state *st = get_current_state(); PyObject *v = iso_calendar_date_new_impl(ISOCALENDAR_DATE_TYPE(st), year, week + 1, day + 1); - RELEASE_CURRENT_STATE(st, current_mod); if (v == NULL) { return NULL; } @@ -6638,11 +6562,9 @@ local_timezone(PyDateTime_DateTime *utc_time) PyObject *one_second; PyObject *seconds; - PyObject *current_mod = NULL; - datetime_state *st = GET_CURRENT_STATE(current_mod); + datetime_state *st = get_current_state(); delta = datetime_subtract((PyObject *)utc_time, CONST_EPOCH(st)); - RELEASE_CURRENT_STATE(st, current_mod); if (delta == NULL) return NULL; @@ -6882,12 +6804,10 @@ datetime_timestamp(PyObject *op, PyObject *Py_UNUSED(dummy)) PyObject *result; if (HASTZINFO(self) && self->tzinfo != Py_None) { - PyObject *current_mod = NULL; - datetime_state *st = GET_CURRENT_STATE(current_mod); + datetime_state *st = get_current_state(); PyObject *delta; delta = datetime_subtract(op, CONST_EPOCH(st)); - RELEASE_CURRENT_STATE(st, current_mod); if (delta == NULL) return NULL; result = delta_total_seconds(delta, NULL); @@ -7252,8 +7172,15 @@ create_timezone_from_delta(int days, int sec, int ms, int normalize) */ static int -init_state(datetime_state *st, PyObject *module, PyObject *old_module) +init_state(datetime_state *st, + PyInterpreterState *interp, PyObject *module, PyObject *old_module) { + PyObject *dict = PyInterpreterState_GetDict(interp); + if (dict == NULL) { + return -1; + } + st->interp_dict = Py_NewRef(dict); + /* Each module gets its own heap types. */ #define ADD_TYPE(FIELD, SPEC, BASE) \ do { \ @@ -7272,6 +7199,7 @@ init_state(datetime_state *st, PyObject *module, PyObject *old_module) assert(old_module != module); datetime_state *st_old = get_module_state(old_module); *st = (datetime_state){ + .interp_dict = st->interp_dict, .isocalendar_date_type = st->isocalendar_date_type, .us_per_ms = Py_NewRef(st_old->us_per_ms), .us_per_second = Py_NewRef(st_old->us_per_second), @@ -7331,9 +7259,8 @@ init_state(datetime_state *st, PyObject *module, PyObject *old_module) static int traverse_state(datetime_state *st, visitproc visit, void *arg) { - /* heap types */ Py_VISIT(st->isocalendar_date_type); - + Py_VISIT(st->interp_dict); return 0; } @@ -7349,6 +7276,7 @@ clear_state(datetime_state *st) Py_CLEAR(st->us_per_week); Py_CLEAR(st->seconds_per_day); Py_CLEAR(st->epoch); + Py_CLEAR(st->interp_dict); return 0; } @@ -7416,7 +7344,7 @@ _datetime_exec(PyObject *module) } } - if (init_state(st, module, old_module) < 0) { + if (init_state(st, interp, module, old_module) < 0) { goto error; } @@ -7522,7 +7450,7 @@ _datetime_exec(PyObject *module) static_assert(DI100Y == 25 * DI4Y - 1, "DI100Y"); assert(DI100Y == days_before_year(100+1)); - if (set_current_module(interp, module) < 0) { + if (set_current_module(st, interp, module) < 0) { goto error; } @@ -7556,10 +7484,9 @@ static int module_clear(PyObject *mod) { datetime_state *st = get_module_state(mod); - clear_state(st); - PyInterpreterState *interp = PyInterpreterState_Get(); - clear_current_module(interp, mod); + clear_current_module(st, interp, mod); + clear_state(st); // The runtime takes care of the static types for us. // See _PyTypes_FiniExtTypes().. From b7d27ead4dd82889fda34b160e96d1aec919f99d Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 21 Apr 2025 08:16:35 +0900 Subject: [PATCH 12/55] Fix decl --- Modules/_datetimemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 81b5000a841e1c..c278887d8200af 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -159,7 +159,7 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) static PyModuleDef datetimemodule; static inline datetime_state * -get_current_state() +get_current_state(void) { PyInterpreterState *interp = PyInterpreterState_Get(); void *state = interp->datetime_module_state; From 3fba89243ec6adc319383a85f666c9f658fc76d7 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 21 Apr 2025 19:39:28 +0900 Subject: [PATCH 13/55] assert interp-dict has a key --- Modules/_datetimemodule.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index c278887d8200af..0cfdd19b9eca60 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -162,9 +162,11 @@ static inline datetime_state * get_current_state(void) { PyInterpreterState *interp = PyInterpreterState_Get(); - void *state = interp->datetime_module_state; - assert(state != NULL); - return (datetime_state *)state; + datetime_state *st = interp->datetime_module_state; + assert(st != NULL); + assert(st->interp_dict != NULL); + assert(PyDict_Contains(st->interp_dict, INTERP_KEY) == 1); + return st; } static int From d5e12201ceb267814cf9a181bb75efda80e6eefa Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 22 Apr 2025 00:09:21 +0900 Subject: [PATCH 14/55] Reword --- Modules/_datetimemodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 0cfdd19b9eca60..06ad7949ab990a 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -50,8 +50,8 @@ typedef struct { /* The interned Unix epoch datetime instance */ PyObject *epoch; - /* Reference to the interpreter's dict where the current module will be - * reserved even after the referent dict becomes NULL at shutdown. */ + /* Extra reference to the interpreter's dict that will be decref'ed last + /* at shutdown. Use the dict to reserve the current module. */ PyObject *interp_dict; } datetime_state; From c0a27eb6b107304cd5bc16ed14a61646108858eb Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 22 Apr 2025 00:18:36 +0900 Subject: [PATCH 15/55] typo --- Modules/_datetimemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 06ad7949ab990a..3e659c1e3b5511 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -51,7 +51,7 @@ typedef struct { PyObject *epoch; /* Extra reference to the interpreter's dict that will be decref'ed last - /* at shutdown. Use the dict to reserve the current module. */ + * at shutdown. Use the dict to reserve the current module. */ PyObject *interp_dict; } datetime_state; From bf3d238d0ad4bbfe20622ddad60d5327985c8bc6 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 22 Apr 2025 02:52:51 +0900 Subject: [PATCH 16/55] Reword again --- Modules/_datetimemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 3e659c1e3b5511..a2d04350b3e247 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -51,7 +51,7 @@ typedef struct { PyObject *epoch; /* Extra reference to the interpreter's dict that will be decref'ed last - * at shutdown. Use the dict to reserve the current module. */ + * at shutdown. Keep the current module in the dict. */ PyObject *interp_dict; } datetime_state; From 0f5993654e3b90e9074118d1ac194fe866cf0443 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 22 Apr 2025 09:06:53 +0900 Subject: [PATCH 17/55] ditto --- Modules/_datetimemodule.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index a2d04350b3e247..d2a01ada50e386 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -50,8 +50,9 @@ typedef struct { /* The interned Unix epoch datetime instance */ PyObject *epoch; - /* Extra reference to the interpreter's dict that will be decref'ed last - * at shutdown. Keep the current module in the dict. */ + /* Extra reference to the interpreter's dict that will be decref'ed + * last at shutdown. We keep the current module in it, but don't rely + * on PyInterpreterState_GetDict() at the module's final phase. */ PyObject *interp_dict; } datetime_state; From 78c762a3ab6a94fd2936f2ef64fc8fdfccc899b4 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 23 Apr 2025 05:54:01 +0900 Subject: [PATCH 18/55] Check if interp-dict is empty --- Modules/_datetimemodule.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index d2a01ada50e386..3d6d3140bfec35 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -202,6 +202,9 @@ clear_current_module(datetime_state *st, if (PyDict_GetItemRef(dict, INTERP_KEY, ¤t) < 0) { goto error; } + if (current == NULL) { + interp->datetime_module_state = NULL; + } /* We only need "current" for pointer comparison. */ Py_XDECREF(current); if (current != expected) { From 8503986aef40e237205dbfb4964c3c2bab4c6be2 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 23 Apr 2025 13:30:19 +0900 Subject: [PATCH 19/55] Correct behavior --- Modules/_datetimemodule.c | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 3d6d3140bfec35..37d9165d67433f 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -159,14 +159,31 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) static PyModuleDef datetimemodule; -static inline datetime_state * +static datetime_state * get_current_state(void) { PyInterpreterState *interp = PyInterpreterState_Get(); datetime_state *st = interp->datetime_module_state; - assert(st != NULL); - assert(st->interp_dict != NULL); - assert(PyDict_Contains(st->interp_dict, INTERP_KEY) == 1); + if (st != NULL) { + assert(st->interp_dict != NULL); + assert(PyDict_Contains(st->interp_dict, INTERP_KEY) == 1); + return st; + } + PyObject *mod = get_current_module(interp, NULL); + if (mod == NULL) { + assert(!PyErr_Occurred()); + if (PyErr_Occurred()) { + return NULL; + } + /* The static types can outlive the module, + * so we must re-import the module. */ + mod = PyImport_ImportModule("_datetime"); + if (mod == NULL) { + return NULL; + } + } + st = get_module_state(mod); + Py_DECREF(mod); return st; } @@ -202,9 +219,6 @@ clear_current_module(datetime_state *st, if (PyDict_GetItemRef(dict, INTERP_KEY, ¤t) < 0) { goto error; } - if (current == NULL) { - interp->datetime_module_state = NULL; - } /* We only need "current" for pointer comparison. */ Py_XDECREF(current); if (current != expected) { @@ -216,7 +230,6 @@ clear_current_module(datetime_state *st, if (PyDict_SetItem(dict, INTERP_KEY, Py_None) < 0) { goto error; } - interp->datetime_module_state = NULL; goto finally; @@ -224,6 +237,9 @@ clear_current_module(datetime_state *st, PyErr_FormatUnraisable("Exception ignored while clearing _datetime module"); finally: + if (!expected || st == interp->datetime_module_state) { + interp->datetime_module_state = NULL; + } PyErr_SetRaisedException(exc); } From 4b92b193226e755e3479670a473ced62244d9be6 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 23 Apr 2025 13:41:05 +0900 Subject: [PATCH 20/55] Revert _datetimemodule.c to main --- Modules/_datetimemodule.c | 133 +++++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 61 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 37d9165d67433f..9bba0e3354b26b 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -49,11 +49,6 @@ typedef struct { /* The interned Unix epoch datetime instance */ PyObject *epoch; - - /* Extra reference to the interpreter's dict that will be decref'ed - * last at shutdown. We keep the current module in it, but don't rely - * on PyInterpreterState_GetDict() at the module's final phase. */ - PyObject *interp_dict; } datetime_state; /* The module has a fixed number of static objects, due to being exposed @@ -138,13 +133,18 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) if (dict == NULL) { goto error; } - if (PyDict_GetItemRef(dict, INTERP_KEY, &mod) < 0) { + PyObject *ref = NULL; + if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { goto error; } - if (mod != NULL) { + if (ref != NULL) { reloading = 1; - if (mod == Py_None) { - mod = NULL; + if (ref != Py_None) { + (void)PyWeakref_GetRef(ref, &mod); + if (mod == Py_None) { + Py_CLEAR(mod); + } + Py_DECREF(ref); } } if (p_reloading != NULL) { @@ -160,15 +160,9 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) static PyModuleDef datetimemodule; static datetime_state * -get_current_state(void) +_get_current_state(PyObject **p_mod) { PyInterpreterState *interp = PyInterpreterState_Get(); - datetime_state *st = interp->datetime_module_state; - if (st != NULL) { - assert(st->interp_dict != NULL); - assert(PyDict_Contains(st->interp_dict, INTERP_KEY) == 1); - return st; - } PyObject *mod = get_current_module(interp, NULL); if (mod == NULL) { assert(!PyErr_Occurred()); @@ -182,47 +176,60 @@ get_current_state(void) return NULL; } } - st = get_module_state(mod); - Py_DECREF(mod); + datetime_state *st = get_module_state(mod); + *p_mod = mod; return st; } +#define GET_CURRENT_STATE(MOD_VAR) \ + _get_current_state(&MOD_VAR) +#define RELEASE_CURRENT_STATE(ST_VAR, MOD_VAR) \ + Py_DECREF(MOD_VAR) + static int -set_current_module(datetime_state *st, - PyInterpreterState *interp, PyObject *mod) +set_current_module(PyInterpreterState *interp, PyObject *mod) { assert(mod != NULL); - PyObject *dict = st->interp_dict; + PyObject *dict = PyInterpreterState_GetDict(interp); if (dict == NULL) { return -1; } - if (PyDict_SetItem(dict, INTERP_KEY, mod) < 0) { + PyObject *ref = PyWeakref_NewRef(mod, NULL); + if (ref == NULL) { return -1; } - interp->datetime_module_state = st; - return 0; + int rc = PyDict_SetItem(dict, INTERP_KEY, ref); + Py_DECREF(ref); + return rc; } static void -clear_current_module(datetime_state *st, - PyInterpreterState *interp, PyObject *expected) +clear_current_module(PyInterpreterState *interp, PyObject *expected) { - PyObject *dict = st->interp_dict; + PyObject *exc = PyErr_GetRaisedException(); + + PyObject *dict = PyInterpreterState_GetDict(interp); if (dict == NULL) { - return; /* Already cleared */ + goto error; } - PyObject *exc = PyErr_GetRaisedException(); - if (expected != NULL) { - PyObject *current; - if (PyDict_GetItemRef(dict, INTERP_KEY, ¤t) < 0) { + PyObject *ref = NULL; + if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { goto error; } - /* We only need "current" for pointer comparison. */ - Py_XDECREF(current); - if (current != expected) { - goto finally; + if (ref != NULL) { + PyObject *current = NULL; + int rc = PyWeakref_GetRef(ref, ¤t); + /* We only need "current" for pointer comparison. */ + Py_XDECREF(current); + Py_DECREF(ref); + if (rc < 0) { + goto error; + } + if (current != expected) { + goto finally; + } } } @@ -237,9 +244,6 @@ clear_current_module(datetime_state *st, PyErr_FormatUnraisable("Exception ignored while clearing _datetime module"); finally: - if (!expected || st == interp->datetime_module_state) { - interp->datetime_module_state = NULL; - } PyErr_SetRaisedException(exc); } @@ -2103,7 +2107,8 @@ delta_to_microseconds(PyDateTime_Delta *self) PyObject *x3 = NULL; PyObject *result = NULL; - datetime_state *st = get_current_state(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); x1 = PyLong_FromLong(GET_TD_DAYS(self)); if (x1 == NULL) @@ -2141,6 +2146,7 @@ delta_to_microseconds(PyDateTime_Delta *self) Py_XDECREF(x1); Py_XDECREF(x2); Py_XDECREF(x3); + RELEASE_CURRENT_STATE(st, current_mod); return result; } @@ -2180,7 +2186,8 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) PyObject *num = NULL; PyObject *result = NULL; - datetime_state *st = get_current_state(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); tuple = checked_divmod(pyus, CONST_US_PER_SECOND(st)); if (tuple == NULL) { @@ -2225,6 +2232,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) Done: Py_XDECREF(tuple); Py_XDECREF(num); + RELEASE_CURRENT_STATE(st, current_mod); return result; BadDivmod: @@ -2763,7 +2771,8 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) { PyObject *self = NULL; - datetime_state *st = get_current_state(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); /* Argument objects. */ PyObject *day = NULL; @@ -2868,6 +2877,7 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) Py_DECREF(x); Done: + RELEASE_CURRENT_STATE(st, current_mod); return self; #undef CLEANUP @@ -2980,10 +2990,12 @@ delta_total_seconds(PyObject *op, PyObject *Py_UNUSED(dummy)) if (total_microseconds == NULL) return NULL; - datetime_state *st = get_current_state(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); total_seconds = PyNumber_TrueDivide(total_microseconds, CONST_US_PER_SECOND(st)); + RELEASE_CURRENT_STATE(st, current_mod); Py_DECREF(total_microseconds); return total_seconds; } @@ -3761,10 +3773,12 @@ date_isocalendar(PyObject *self, PyObject *Py_UNUSED(dummy)) week = 0; } - datetime_state *st = get_current_state(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); PyObject *v = iso_calendar_date_new_impl(ISOCALENDAR_DATE_TYPE(st), year, week + 1, day + 1); + RELEASE_CURRENT_STATE(st, current_mod); if (v == NULL) { return NULL; } @@ -6584,9 +6598,11 @@ local_timezone(PyDateTime_DateTime *utc_time) PyObject *one_second; PyObject *seconds; - datetime_state *st = get_current_state(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); delta = datetime_subtract((PyObject *)utc_time, CONST_EPOCH(st)); + RELEASE_CURRENT_STATE(st, current_mod); if (delta == NULL) return NULL; @@ -6826,10 +6842,12 @@ datetime_timestamp(PyObject *op, PyObject *Py_UNUSED(dummy)) PyObject *result; if (HASTZINFO(self) && self->tzinfo != Py_None) { - datetime_state *st = get_current_state(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); PyObject *delta; delta = datetime_subtract(op, CONST_EPOCH(st)); + RELEASE_CURRENT_STATE(st, current_mod); if (delta == NULL) return NULL; result = delta_total_seconds(delta, NULL); @@ -7194,15 +7212,8 @@ create_timezone_from_delta(int days, int sec, int ms, int normalize) */ static int -init_state(datetime_state *st, - PyInterpreterState *interp, PyObject *module, PyObject *old_module) +init_state(datetime_state *st, PyObject *module, PyObject *old_module) { - PyObject *dict = PyInterpreterState_GetDict(interp); - if (dict == NULL) { - return -1; - } - st->interp_dict = Py_NewRef(dict); - /* Each module gets its own heap types. */ #define ADD_TYPE(FIELD, SPEC, BASE) \ do { \ @@ -7221,7 +7232,6 @@ init_state(datetime_state *st, assert(old_module != module); datetime_state *st_old = get_module_state(old_module); *st = (datetime_state){ - .interp_dict = st->interp_dict, .isocalendar_date_type = st->isocalendar_date_type, .us_per_ms = Py_NewRef(st_old->us_per_ms), .us_per_second = Py_NewRef(st_old->us_per_second), @@ -7281,8 +7291,9 @@ init_state(datetime_state *st, static int traverse_state(datetime_state *st, visitproc visit, void *arg) { + /* heap types */ Py_VISIT(st->isocalendar_date_type); - Py_VISIT(st->interp_dict); + return 0; } @@ -7298,7 +7309,6 @@ clear_state(datetime_state *st) Py_CLEAR(st->us_per_week); Py_CLEAR(st->seconds_per_day); Py_CLEAR(st->epoch); - Py_CLEAR(st->interp_dict); return 0; } @@ -7366,7 +7376,7 @@ _datetime_exec(PyObject *module) } } - if (init_state(st, interp, module, old_module) < 0) { + if (init_state(st, module, old_module) < 0) { goto error; } @@ -7472,7 +7482,7 @@ _datetime_exec(PyObject *module) static_assert(DI100Y == 25 * DI4Y - 1, "DI100Y"); assert(DI100Y == days_before_year(100+1)); - if (set_current_module(st, interp, module) < 0) { + if (set_current_module(interp, module) < 0) { goto error; } @@ -7506,10 +7516,11 @@ static int module_clear(PyObject *mod) { datetime_state *st = get_module_state(mod); - PyInterpreterState *interp = PyInterpreterState_Get(); - clear_current_module(st, interp, mod); clear_state(st); + PyInterpreterState *interp = PyInterpreterState_Get(); + clear_current_module(interp, mod); + // The runtime takes care of the static types for us. // See _PyTypes_FiniExtTypes().. From 3d293800b390ff629dd891207873062e65d5f6b6 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 23 Apr 2025 13:59:45 +0900 Subject: [PATCH 21/55] Fix crash (new attempt) --- Modules/_datetimemodule.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 9bba0e3354b26b..0e515325d2bbc0 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -163,6 +163,13 @@ static datetime_state * _get_current_state(PyObject **p_mod) { PyInterpreterState *interp = PyInterpreterState_Get(); + datetime_state *st = interp->datetime_module_state; + if (st != NULL) { + PyObject *mod = PyType_GetModule(st->isocalendar_date_type); + assert(mod != NULL); + *p_mod = Py_NewRef(mod); + return st; + } PyObject *mod = get_current_module(interp, NULL); if (mod == NULL) { assert(!PyErr_Occurred()); @@ -176,7 +183,7 @@ _get_current_state(PyObject **p_mod) return NULL; } } - datetime_state *st = get_module_state(mod); + st = get_module_state(mod); *p_mod = mod; return st; } @@ -200,6 +207,9 @@ set_current_module(PyInterpreterState *interp, PyObject *mod) } int rc = PyDict_SetItem(dict, INTERP_KEY, ref); Py_DECREF(ref); + if (rc == 0) { + interp->datetime_module_state = get_module_state(mod); + } return rc; } @@ -244,6 +254,9 @@ clear_current_module(PyInterpreterState *interp, PyObject *expected) PyErr_FormatUnraisable("Exception ignored while clearing _datetime module"); finally: + if (!expected || get_module_state(expected) == interp->datetime_module_state) { + interp->datetime_module_state = NULL; + } PyErr_SetRaisedException(exc); } From 26b8f51709dafb5562fcd05038f51d5f5597739a Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 23 Apr 2025 19:26:22 +0900 Subject: [PATCH 22/55] Non-NULL check before PyType_Check() --- Modules/_datetimemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 0e515325d2bbc0..66e75e94b342cf 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -164,7 +164,7 @@ _get_current_state(PyObject **p_mod) { PyInterpreterState *interp = PyInterpreterState_Get(); datetime_state *st = interp->datetime_module_state; - if (st != NULL) { + if (st != NULL && st->isocalendar_date_type != NULL) { PyObject *mod = PyType_GetModule(st->isocalendar_date_type); assert(mod != NULL); *p_mod = Py_NewRef(mod); From 3e2ef0d9d62bf0ac5f217081829fbe2d2dff30c9 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 23 Apr 2025 21:44:13 +0900 Subject: [PATCH 23/55] Faster _get_current_state() by 6%- --- Modules/_datetimemodule.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 66e75e94b342cf..8c62c073cc335e 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -35,6 +35,9 @@ static PyTypeObject PyDateTime_TimeZoneType; typedef struct { + /* Corresponding module */ + PyObject *module; + /* Module heap types. */ PyTypeObject *isocalendar_date_type; @@ -164,12 +167,12 @@ _get_current_state(PyObject **p_mod) { PyInterpreterState *interp = PyInterpreterState_Get(); datetime_state *st = interp->datetime_module_state; - if (st != NULL && st->isocalendar_date_type != NULL) { - PyObject *mod = PyType_GetModule(st->isocalendar_date_type); - assert(mod != NULL); - *p_mod = Py_NewRef(mod); + if (st != NULL && st->module != NULL) { + assert(PyModule_CheckExact(st->module)); + *p_mod = Py_NewRef(st->module); return st; } + PyObject *mod = get_current_module(interp, NULL); if (mod == NULL) { assert(!PyErr_Occurred()); @@ -7227,6 +7230,8 @@ create_timezone_from_delta(int days, int sec, int ms, int normalize) static int init_state(datetime_state *st, PyObject *module, PyObject *old_module) { + st->module = Py_NewRef(module); + /* Each module gets its own heap types. */ #define ADD_TYPE(FIELD, SPEC, BASE) \ do { \ @@ -7245,6 +7250,7 @@ init_state(datetime_state *st, PyObject *module, PyObject *old_module) assert(old_module != module); datetime_state *st_old = get_module_state(old_module); *st = (datetime_state){ + .module = st->module, .isocalendar_date_type = st->isocalendar_date_type, .us_per_ms = Py_NewRef(st_old->us_per_ms), .us_per_second = Py_NewRef(st_old->us_per_second), @@ -7304,7 +7310,7 @@ init_state(datetime_state *st, PyObject *module, PyObject *old_module) static int traverse_state(datetime_state *st, visitproc visit, void *arg) { - /* heap types */ + Py_VISIT(st->module); Py_VISIT(st->isocalendar_date_type); return 0; @@ -7313,6 +7319,7 @@ traverse_state(datetime_state *st, visitproc visit, void *arg) static int clear_state(datetime_state *st) { + Py_CLEAR(st->module); /* Invalidate first */ Py_CLEAR(st->isocalendar_date_type); Py_CLEAR(st->us_per_ms); Py_CLEAR(st->us_per_second); From 2e1a1290a766aa300d6b50be45f38b22b5abe08d Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Thu, 24 Apr 2025 03:07:35 +0900 Subject: [PATCH 24/55] Comment --- Modules/_datetimemodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 8c62c073cc335e..5d34cb8638ed10 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -35,7 +35,8 @@ static PyTypeObject PyDateTime_TimeZoneType; typedef struct { - /* Corresponding module */ + /* Corresponding module that can be referred even after + * its weak reference stops working at shutdown. */ PyObject *module; /* Module heap types. */ From 3c059899552c57ac1a65c327ebffb527252fd4be Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Thu, 24 Apr 2025 11:15:46 +0900 Subject: [PATCH 25/55] assert(!_Py_IsInterpreterFinalizing()) --- Modules/_datetimemodule.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 5d34cb8638ed10..95f58d573efeff 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -12,6 +12,7 @@ #include "Python.h" #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyObject_Init() +#include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_time.h" // _PyTime_ObjectToTime_t() #include "pycore_unicodeobject.h" // _PyUnicode_Copy() @@ -180,6 +181,7 @@ _get_current_state(PyObject **p_mod) if (PyErr_Occurred()) { return NULL; } + assert(!_Py_IsInterpreterFinalizing(interp)); /* The static types can outlive the module, * so we must re-import the module. */ mod = PyImport_ImportModule("_datetime"); From 8eaae369fecfa5baca5fb9a69906b54770d1645a Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Thu, 24 Apr 2025 11:49:04 +0900 Subject: [PATCH 26/55] assign NULL to interp-state on error --- Modules/_datetimemodule.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 95f58d573efeff..3adfbab9a3825f 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -213,9 +213,7 @@ set_current_module(PyInterpreterState *interp, PyObject *mod) } int rc = PyDict_SetItem(dict, INTERP_KEY, ref); Py_DECREF(ref); - if (rc == 0) { - interp->datetime_module_state = get_module_state(mod); - } + interp->datetime_module_state = rc == 0 ? get_module_state(mod) : NULL; return rc; } From c2758bd610828c8b09a5ca8680398fbd4d74b70d Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 25 Apr 2025 12:16:57 +0900 Subject: [PATCH 27/55] Get module without interp-dict and weakref --- .../pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - .../internal/pycore_runtime_init_generated.h | 1 - .../internal/pycore_unicodeobject_generated.h | 4 - Modules/_datetimemodule.c | 115 +++--------------- 5 files changed, 18 insertions(+), 104 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 5485d0bd64f3f1..0203dd0e32ab7c 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -835,7 +835,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_call)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_return)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_datetime_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_statements)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cafile)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 3ce192511e3879..14befd2128a9f7 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -326,7 +326,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(c_call) STRUCT_FOR_ID(c_exception) STRUCT_FOR_ID(c_return) - STRUCT_FOR_ID(cached_datetime_module) STRUCT_FOR_ID(cached_statements) STRUCT_FOR_ID(cadata) STRUCT_FOR_ID(cafile) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 5c95d0feddecba..0c89372562df17 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -833,7 +833,6 @@ extern "C" { INIT_ID(c_call), \ INIT_ID(c_exception), \ INIT_ID(c_return), \ - INIT_ID(cached_datetime_module), \ INIT_ID(cached_statements), \ INIT_ID(cadata), \ INIT_ID(cafile), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index a1fc9736d66618..25d71867d49ef5 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1092,10 +1092,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(cached_datetime_module); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cached_statements); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 3adfbab9a3825f..946f6f8421778e 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -36,8 +36,7 @@ static PyTypeObject PyDateTime_TimeZoneType; typedef struct { - /* Corresponding module that can be referred even after - * its weak reference stops working at shutdown. */ + /* Corresponding module that is referenced via the interpreter state. */ PyObject *module; /* Module heap types. */ @@ -126,68 +125,37 @@ get_module_state(PyObject *module) } -#define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module)) - static PyObject * get_current_module(PyInterpreterState *interp, int *p_reloading) { - PyObject *mod = NULL; - int reloading = 0; - - PyObject *dict = PyInterpreterState_GetDict(interp); - if (dict == NULL) { - goto error; - } - PyObject *ref = NULL; - if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { - goto error; - } - if (ref != NULL) { - reloading = 1; - if (ref != Py_None) { - (void)PyWeakref_GetRef(ref, &mod); - if (mod == Py_None) { - Py_CLEAR(mod); - } - Py_DECREF(ref); - } - } + datetime_state *st = interp->datetime_module_state; if (p_reloading != NULL) { - *p_reloading = reloading; + *p_reloading = st != NULL ? 1 : 0; + } + if (st != NULL && st != (void *)Py_None && st->module != NULL) { + assert(PyModule_CheckExact(st->module)); + return Py_NewRef(st->module); } - return mod; - -error: - assert(PyErr_Occurred()); return NULL; } -static PyModuleDef datetimemodule; - static datetime_state * _get_current_state(PyObject **p_mod) { PyInterpreterState *interp = PyInterpreterState_Get(); datetime_state *st = interp->datetime_module_state; - if (st != NULL && st->module != NULL) { + if (st != NULL && st != (void *)Py_None && st->module != NULL) { assert(PyModule_CheckExact(st->module)); *p_mod = Py_NewRef(st->module); return st; } - PyObject *mod = get_current_module(interp, NULL); + assert(!_Py_IsInterpreterFinalizing(interp)); + /* The static types can outlive the module, + * so we must re-import the module. */ + PyObject *mod = PyImport_ImportModule("_datetime"); if (mod == NULL) { - assert(!PyErr_Occurred()); - if (PyErr_Occurred()) { - return NULL; - } - assert(!_Py_IsInterpreterFinalizing(interp)); - /* The static types can outlive the module, - * so we must re-import the module. */ - mod = PyImport_ImportModule("_datetime"); - if (mod == NULL) { - return NULL; - } + return NULL; } st = get_module_state(mod); *p_mod = mod; @@ -203,65 +171,18 @@ static int set_current_module(PyInterpreterState *interp, PyObject *mod) { assert(mod != NULL); - PyObject *dict = PyInterpreterState_GetDict(interp); - if (dict == NULL) { - return -1; - } - PyObject *ref = PyWeakref_NewRef(mod, NULL); - if (ref == NULL) { - return -1; - } - int rc = PyDict_SetItem(dict, INTERP_KEY, ref); - Py_DECREF(ref); - interp->datetime_module_state = rc == 0 ? get_module_state(mod) : NULL; - return rc; + interp->datetime_module_state = get_module_state(mod); + return 0; } static void clear_current_module(PyInterpreterState *interp, PyObject *expected) { - PyObject *exc = PyErr_GetRaisedException(); - - PyObject *dict = PyInterpreterState_GetDict(interp); - if (dict == NULL) { - goto error; - } - - if (expected != NULL) { - PyObject *ref = NULL; - if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { - goto error; - } - if (ref != NULL) { - PyObject *current = NULL; - int rc = PyWeakref_GetRef(ref, ¤t); - /* We only need "current" for pointer comparison. */ - Py_XDECREF(current); - Py_DECREF(ref); - if (rc < 0) { - goto error; - } - if (current != expected) { - goto finally; - } - } + if (expected && get_module_state(expected) != interp->datetime_module_state) { + return; } - /* We use None to identify that the module was previously loaded. */ - if (PyDict_SetItem(dict, INTERP_KEY, Py_None) < 0) { - goto error; - } - - goto finally; - -error: - PyErr_FormatUnraisable("Exception ignored while clearing _datetime module"); - -finally: - if (!expected || get_module_state(expected) == interp->datetime_module_state) { - interp->datetime_module_state = NULL; - } - PyErr_SetRaisedException(exc); + interp->datetime_module_state = Py_None; } From 56f8a067fda35e61b9364e6a2d0c9de00dfdd1d3 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sat, 26 Apr 2025 03:31:58 +0900 Subject: [PATCH 28/55] Add tests --- Lib/test/datetimetester.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 898389ef409c05..7d03beb1dccc68 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7294,6 +7294,32 @@ def gen(): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) + def test_module_free(self): + script = textwrap.dedent(""" + import sys + import gc + import weakref + ws = weakref.WeakSet() + for _ in range(3): + import _datetime + ws.add(_datetime) + del sys.modules["_datetime"] + del _datetime + gc.collect() + assert len(ws) == 0 + """) + res = script_helper.assert_python_ok('-c', script) + self.assertFalse(res.err) + + @unittest.skipIf(not support.Py_DEBUG, "Debug builds only") + def test_no_leak(self): + script = textwrap.dedent(""" + import datetime + datetime.datetime.strptime('20000101', '%Y%m%d').strftime('%Y%m%d') + """) + res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) + self.assertIn(b'[0 refs, 0 blocks]', res.err) + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) From aa868072bb94cd8839067c5be398d4c25d352d35 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sat, 26 Apr 2025 07:40:39 +0900 Subject: [PATCH 29/55] Add tests --- Lib/test/datetimetester.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 7d03beb1dccc68..73d9804a28d829 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7294,6 +7294,43 @@ def gen(): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) + def test_module_state_at_shutdown2(self): + script = textwrap.dedent(""" + import sys + import _datetime + timedelta = _datetime.timedelta + del _datetime + del sys.modules["_datetime"] + + def gen(): + try: + yield + finally: + td = timedelta(days=1) # crash + assert td.days == 1 + assert not sys.modules + + it = gen() + next(it) + """) + res = script_helper.assert_python_ok('-c', script) + self.assertFalse(res.err) + + def test_module_state_after_gc(self): + script = textwrap.dedent(""" + import sys + import gc + import _datetime + timedelta = _datetime.timedelta + del sys.modules['_datetime'] + del _datetime + gc.collect() + timedelta(days=1) + assert '_datetime' in sys.modules + """) + res = script_helper.assert_python_ok('-c', script) + self.assertFalse(res.err) + def test_module_free(self): script = textwrap.dedent(""" import sys From 6880f34529530164518823715a42cac3b9e76ecf Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sat, 26 Apr 2025 19:53:58 +0900 Subject: [PATCH 30/55] Update and pass tests --- Lib/test/datetimetester.py | 44 +++++++++++++++++++++++++++----------- Modules/_datetimemodule.c | 24 ++++++++++++++++++++- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 73d9804a28d829..20df2edb061103 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7273,7 +7273,7 @@ def test_update_type_cache(self): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) - def test_module_state_at_shutdown(self): + def test_static_type_at_shutdown1(self): # gh-132413 script = textwrap.dedent(""" import sys @@ -7284,7 +7284,7 @@ def gen(): yield finally: assert not sys.modules - td = _datetime.timedelta(days=1) # crash + td = _datetime.timedelta(days=1) assert td.days == 1 assert not sys.modules @@ -7294,19 +7294,17 @@ def gen(): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) - def test_module_state_at_shutdown2(self): + def test_static_type_at_shutdown2(self): script = textwrap.dedent(""" import sys - import _datetime - timedelta = _datetime.timedelta - del _datetime - del sys.modules["_datetime"] + from _datetime import timedelta def gen(): try: yield finally: - td = timedelta(days=1) # crash + assert not sys.modules + td = timedelta(days=1) assert td.days == 1 assert not sys.modules @@ -7316,15 +7314,34 @@ def gen(): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) - def test_module_state_after_gc(self): + def test_static_type_at_shutdown3(self): script = textwrap.dedent(""" + import gc import sys + from _datetime import timedelta + del sys.modules['_datetime'] + gc.collect() + + def gen(): + try: + yield + finally: + timedelta(days=1) + + it = gen() + next(it) + """) + res = script_helper.assert_python_ok('-c', script) + self.assertIn(b'ImportError: sys.meta_path is None', res.err) + + def test_static_type_before_shutdown(self): + script = textwrap.dedent(""" import gc - import _datetime - timedelta = _datetime.timedelta + import sys + from _datetime import timedelta del sys.modules['_datetime'] - del _datetime gc.collect() + timedelta(days=1) assert '_datetime' in sys.modules """) @@ -7339,8 +7356,9 @@ def test_module_free(self): ws = weakref.WeakSet() for _ in range(3): import _datetime + td = _datetime.timedelta ws.add(_datetime) - del sys.modules["_datetime"] + del sys.modules['_datetime'] del _datetime gc.collect() assert len(ws) == 0 diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 946f6f8421778e..9acda8238b67de 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -150,11 +150,12 @@ _get_current_state(PyObject **p_mod) return st; } - assert(!_Py_IsInterpreterFinalizing(interp)); /* The static types can outlive the module, * so we must re-import the module. */ PyObject *mod = PyImport_ImportModule("_datetime"); if (mod == NULL) { + assert(_Py_IsInterpreterFinalizing(interp)); + /* It is not preferable to reload the module implicitly here. */ return NULL; } st = get_module_state(mod); @@ -2047,6 +2048,9 @@ delta_to_microseconds(PyDateTime_Delta *self) PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (current_mod == NULL) { + return NULL; + } x1 = PyLong_FromLong(GET_TD_DAYS(self)); if (x1 == NULL) @@ -2126,6 +2130,9 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (current_mod == NULL) { + return NULL; + } tuple = checked_divmod(pyus, CONST_US_PER_SECOND(st)); if (tuple == NULL) { @@ -2711,6 +2718,9 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (current_mod == NULL) { + return NULL; + } /* Argument objects. */ PyObject *day = NULL; @@ -2930,6 +2940,9 @@ delta_total_seconds(PyObject *op, PyObject *Py_UNUSED(dummy)) PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (current_mod == NULL) { + return NULL; + } total_seconds = PyNumber_TrueDivide(total_microseconds, CONST_US_PER_SECOND(st)); @@ -3713,6 +3726,9 @@ date_isocalendar(PyObject *self, PyObject *Py_UNUSED(dummy)) PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (current_mod == NULL) { + return NULL; + } PyObject *v = iso_calendar_date_new_impl(ISOCALENDAR_DATE_TYPE(st), year, week + 1, day + 1); @@ -6538,6 +6554,9 @@ local_timezone(PyDateTime_DateTime *utc_time) PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (current_mod == NULL) { + return NULL; + } delta = datetime_subtract((PyObject *)utc_time, CONST_EPOCH(st)); RELEASE_CURRENT_STATE(st, current_mod); @@ -6782,6 +6801,9 @@ datetime_timestamp(PyObject *op, PyObject *Py_UNUSED(dummy)) if (HASTZINFO(self) && self->tzinfo != Py_None) { PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (current_mod == NULL) { + return NULL; + } PyObject *delta; delta = datetime_subtract(op, CONST_EPOCH(st)); From 022d4e8148bf1fdbcce9556b39d53a2547cb9607 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 27 Apr 2025 15:46:05 +0900 Subject: [PATCH 31/55] Make tests realistic by using subinterp --- Lib/test/datetimetester.py | 80 +++++++++++++++++++++++++++--------- Modules/_datetimemodule.c | 2 +- Modules/_testcapi/datetime.c | 11 +++-- 3 files changed, 69 insertions(+), 24 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 20df2edb061103..0438ec42a7eae4 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7176,6 +7176,7 @@ def test_type_check_in_subinterp(self): spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) + module.test_datetime_capi() def run(type_checker, obj): if not type_checker(obj, True): @@ -7314,38 +7315,77 @@ def gen(): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) + def run_script_132413(self, script): + # iOS requires the use of the custom framework loader, + # not the ExtensionFileLoader. + if sys.platform == "ios": + extension_loader = "AppleFrameworkLoader" + else: + extension_loader = "ExtensionFileLoader" + + main_interp_script = textwrap.dedent(f''' + import textwrap + from test import support + + sub_script = textwrap.dedent(""" + if {_interpreters is None}: + import _testcapi as module + else: + import importlib.machinery + import importlib.util + fullname = '_testcapi_datetime' + origin = importlib.util.find_spec('_testcapi').origin + loader = importlib.machinery.{extension_loader}(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + # Skip calling test_datetime_capi() + $REPLACE_ME$ + """) + + import _testcapi + _testcapi.test_datetime_capi() + + if {_interpreters is None}: + ret = support.run_in_subinterp(sub_script) + else: + import _interpreters + config = _interpreters.new_config('isolated').__dict__ + ret = support.run_in_subinterp_with_config(sub_script, **config) + + assert ret == 0 + + ''').replace('$REPLACE_ME$', textwrap.indent(script, '\x20'*4)) + + res = script_helper.assert_python_ok('-c', main_interp_script) + return res + def test_static_type_at_shutdown3(self): script = textwrap.dedent(""" - import gc - import sys - from _datetime import timedelta - del sys.modules['_datetime'] - gc.collect() + timedelta = module.get_delta_type() def gen(): - try: - yield - finally: - timedelta(days=1) + try: + yield + finally: + timedelta(days=1) it = gen() next(it) - """) - res = script_helper.assert_python_ok('-c', script) + """) + res = self.run_script_132413(script) self.assertIn(b'ImportError: sys.meta_path is None', res.err) def test_static_type_before_shutdown(self): - script = textwrap.dedent(""" - import gc + script = textwrap.dedent(f""" import sys - from _datetime import timedelta - del sys.modules['_datetime'] - gc.collect() - + assert '_datetime' not in sys.modules + timedelta = module.get_delta_type() timedelta(days=1) assert '_datetime' in sys.modules - """) - res = script_helper.assert_python_ok('-c', script) + """) + res = self.run_script_132413(script) self.assertFalse(res.err) def test_module_free(self): @@ -7356,7 +7396,7 @@ def test_module_free(self): ws = weakref.WeakSet() for _ in range(3): import _datetime - td = _datetime.timedelta + timedelta = _datetime.timedelta ws.add(_datetime) del sys.modules['_datetime'] del _datetime diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 9acda8238b67de..947bb901532c71 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -155,7 +155,7 @@ _get_current_state(PyObject **p_mod) PyObject *mod = PyImport_ImportModule("_datetime"); if (mod == NULL) { assert(_Py_IsInterpreterFinalizing(interp)); - /* It is not preferable to reload the module implicitly here. */ + /* We do not take care of the unlikely case. */ return NULL; } st = get_module_state(mod); diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index b800f9b8eb3473..daf0b8140a382a 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -453,6 +453,12 @@ test_PyDateTime_DELTA_GET(PyObject *self, PyObject *obj) return Py_BuildValue("(iii)", days, seconds, microseconds); } +static PyObject * +get_delta_type(PyObject *self, PyObject *args) +{ + return PyDateTimeAPI ? Py_NewRef(PyDateTimeAPI->DeltaType) : Py_None; +} + static PyMethodDef test_methods[] = { {"PyDateTime_DATE_GET", test_PyDateTime_DATE_GET, METH_O}, {"PyDateTime_DELTA_GET", test_PyDateTime_DELTA_GET, METH_O}, @@ -469,6 +475,7 @@ static PyMethodDef test_methods[] = { {"get_datetime_fromdateandtimeandfold", get_datetime_fromdateandtimeandfold, METH_VARARGS}, {"get_datetime_fromtimestamp", get_datetime_fromtimestamp, METH_VARARGS}, {"get_delta_fromdsu", get_delta_fromdsu, METH_VARARGS}, + {"get_delta_type", get_delta_type, METH_NOARGS}, {"get_time_fromtime", get_time_fromtime, METH_VARARGS}, {"get_time_fromtimeandfold", get_time_fromtimeandfold, METH_VARARGS}, {"get_timezone_utc_capi", get_timezone_utc_capi, METH_VARARGS}, @@ -495,9 +502,7 @@ _PyTestCapi_Init_DateTime(PyObject *mod) static int _testcapi_datetime_exec(PyObject *mod) { - if (test_datetime_capi(NULL, NULL) == NULL) { - return -1; - } + // Call test_datetime_capi() in each test. return 0; } From 318888ad07f4e4324bf07b8efee105aa093958c3 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 27 Apr 2025 16:24:12 +0900 Subject: [PATCH 32/55] Nit --- Lib/test/datetimetester.py | 16 ++++++++-------- Modules/_datetimemodule.c | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 0438ec42a7eae4..9a583f97df5bfd 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7291,7 +7291,7 @@ def gen(): it = gen() next(it) - """) + """) res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) @@ -7311,7 +7311,7 @@ def gen(): it = gen() next(it) - """) + """) res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) @@ -7366,10 +7366,10 @@ def test_static_type_at_shutdown3(self): timedelta = module.get_delta_type() def gen(): - try: - yield - finally: - timedelta(days=1) + try: + yield + finally: + timedelta(days=1) it = gen() next(it) @@ -7402,7 +7402,7 @@ def test_module_free(self): del _datetime gc.collect() assert len(ws) == 0 - """) + """) res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) @@ -7411,7 +7411,7 @@ def test_no_leak(self): script = textwrap.dedent(""" import datetime datetime.datetime.strptime('20000101', '%Y%m%d').strftime('%Y%m%d') - """) + """) res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) self.assertIn(b'[0 refs, 0 blocks]', res.err) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 947bb901532c71..cad082a1521ac1 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -155,7 +155,6 @@ _get_current_state(PyObject **p_mod) PyObject *mod = PyImport_ImportModule("_datetime"); if (mod == NULL) { assert(_Py_IsInterpreterFinalizing(interp)); - /* We do not take care of the unlikely case. */ return NULL; } st = get_module_state(mod); From 6e6c3288d5d7cd9b7e116ade9d756f9651b40c01 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 28 Apr 2025 02:44:01 +0900 Subject: [PATCH 33/55] Make tests generic --- Lib/test/datetimetester.py | 139 +++++++++++++++-------------------- Modules/_testcapi/datetime.c | 31 +++++++- 2 files changed, 87 insertions(+), 83 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 9a583f97df5bfd..825aad21ec5023 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7155,7 +7155,10 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) - def test_type_check_in_subinterp(self): + def assert_python_ok_in_subinterp(self, script, + setup='_testcapi.test_datetime_capi()', + mainsetup='_testcapi.test_datetime_capi()', + config='isolated'): # iOS requires the use of the custom framework loader, # not the ExtensionFileLoader. if sys.platform == "ios": @@ -7163,41 +7166,64 @@ def test_type_check_in_subinterp(self): else: extension_loader = "ExtensionFileLoader" - script = textwrap.dedent(f""" + maincode = textwrap.dedent(f''' + import textwrap + from test import support + + subcode = textwrap.dedent(""" + if {_interpreters is None}: + import _testcapi + else: + import importlib.machinery + import importlib.util + fullname = '_testcapi_datetime' + origin = importlib.util.find_spec('_testcapi').origin + loader = importlib.machinery.{extension_loader}(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + _testcapi = importlib.util.module_from_spec(spec) + spec.loader.exec_module(_testcapi) + + $SETUP$ + $SCRIPT$ + """) + + import _testcapi + $MAINSETUP$ + if {_interpreters is None}: - import _testcapi as module - module.test_datetime_capi() + ret = support.run_in_subinterp(subcode) else: - import importlib.machinery - import importlib.util - fullname = '_testcapi_datetime' - origin = importlib.util.find_spec('_testcapi').origin - loader = importlib.machinery.{extension_loader}(fullname, origin) - spec = importlib.util.spec_from_loader(fullname, loader) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - module.test_datetime_capi() + import _interpreters + config = _interpreters.new_config('{config}').__dict__ + ret = support.run_in_subinterp_with_config(subcode, **config) + + assert ret == 0 + + ''').replace('$MAINSETUP$', mainsetup) + maincode = maincode.replace('$SETUP$', textwrap.indent(setup, '\x20'*4)) + maincode = maincode.replace('$SCRIPT$', textwrap.indent(script, '\x20'*4)) + + res = script_helper.assert_python_ok('-c', maincode) + return res + def test_type_check_in_subinterp(self): + script = textwrap.dedent(f""" def run(type_checker, obj): if not type_checker(obj, True): raise TypeError(f'{{type(obj)}} is not C API type') import _datetime - run(module.datetime_check_date, _datetime.date.today()) - run(module.datetime_check_datetime, _datetime.datetime.now()) - run(module.datetime_check_time, _datetime.time(12, 30)) - run(module.datetime_check_delta, _datetime.timedelta(1)) - run(module.datetime_check_tzinfo, _datetime.tzinfo()) + run(_testcapi.datetime_check_date, _datetime.date.today()) + run(_testcapi.datetime_check_datetime, _datetime.datetime.now()) + run(_testcapi.datetime_check_time, _datetime.time(12, 30)) + run(_testcapi.datetime_check_delta, _datetime.timedelta(1)) + run(_testcapi.datetime_check_tzinfo, _datetime.tzinfo()) """) - if _interpreters is None: - ret = support.run_in_subinterp(script) - self.assertEqual(ret, 0) - else: - for name in ('isolated', 'legacy'): - with self.subTest(name): - config = _interpreters.new_config(name).__dict__ - ret = support.run_in_subinterp_with_config(script, **config) - self.assertEqual(ret, 0) + self.assert_python_ok_in_subinterp(script, mainsetup='') + if _interpreters is not None: + with self.subTest('legacy'): + self.assert_python_ok_in_subinterp(script, mainsetup='', + config='legacy') class ExtensionModuleTests(unittest.TestCase): @@ -7284,6 +7310,7 @@ def gen(): try: yield finally: + # Exceptions are ignored here assert not sys.modules td = _datetime.timedelta(days=1) assert td.days == 1 @@ -7315,55 +7342,9 @@ def gen(): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) - def run_script_132413(self, script): - # iOS requires the use of the custom framework loader, - # not the ExtensionFileLoader. - if sys.platform == "ios": - extension_loader = "AppleFrameworkLoader" - else: - extension_loader = "ExtensionFileLoader" - - main_interp_script = textwrap.dedent(f''' - import textwrap - from test import support - - sub_script = textwrap.dedent(""" - if {_interpreters is None}: - import _testcapi as module - else: - import importlib.machinery - import importlib.util - fullname = '_testcapi_datetime' - origin = importlib.util.find_spec('_testcapi').origin - loader = importlib.machinery.{extension_loader}(fullname, origin) - spec = importlib.util.spec_from_loader(fullname, loader) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - # Skip calling test_datetime_capi() - $REPLACE_ME$ - """) - - import _testcapi - _testcapi.test_datetime_capi() - - if {_interpreters is None}: - ret = support.run_in_subinterp(sub_script) - else: - import _interpreters - config = _interpreters.new_config('isolated').__dict__ - ret = support.run_in_subinterp_with_config(sub_script, **config) - - assert ret == 0 - - ''').replace('$REPLACE_ME$', textwrap.indent(script, '\x20'*4)) - - res = script_helper.assert_python_ok('-c', main_interp_script) - return res - def test_static_type_at_shutdown3(self): script = textwrap.dedent(""" - timedelta = module.get_delta_type() + timedelta = _testcapi.get_capi_types()['timedelta'] def gen(): try: @@ -7374,19 +7355,18 @@ def gen(): it = gen() next(it) """) - res = self.run_script_132413(script) + res = CapiTest.assert_python_ok_in_subinterp(self, script, setup='') self.assertIn(b'ImportError: sys.meta_path is None', res.err) def test_static_type_before_shutdown(self): script = textwrap.dedent(f""" import sys assert '_datetime' not in sys.modules - timedelta = module.get_delta_type() + timedelta = _testcapi.get_capi_types()['timedelta'] timedelta(days=1) assert '_datetime' in sys.modules """) - res = self.run_script_132413(script) - self.assertFalse(res.err) + CapiTest.assert_python_ok_in_subinterp(self, script, setup='') def test_module_free(self): script = textwrap.dedent(""" @@ -7403,8 +7383,7 @@ def test_module_free(self): gc.collect() assert len(ws) == 0 """) - res = script_helper.assert_python_ok('-c', script) - self.assertFalse(res.err) + script_helper.assert_python_ok('-c', script) @unittest.skipIf(not support.Py_DEBUG, "Debug builds only") def test_no_leak(self): diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index daf0b8140a382a..0b4a968d0a0d0c 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -454,9 +454,34 @@ test_PyDateTime_DELTA_GET(PyObject *self, PyObject *obj) } static PyObject * -get_delta_type(PyObject *self, PyObject *args) +get_capi_types(PyObject *self, PyObject *args) { - return PyDateTimeAPI ? Py_NewRef(PyDateTimeAPI->DeltaType) : Py_None; + if (PyDateTimeAPI == NULL) { + Py_RETURN_NONE; + } + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + if (PyDict_SetItemString(dict, "date", (PyObject *)PyDateTimeAPI->DateType) < 0) { + goto error; + } + if (PyDict_SetItemString(dict, "time", (PyObject *)PyDateTimeAPI->TimeType) < 0) { + goto error; + } + if (PyDict_SetItemString(dict, "datetime", (PyObject *)PyDateTimeAPI->DateTimeType) < 0) { + goto error; + } + if (PyDict_SetItemString(dict, "timedelta", (PyObject *)PyDateTimeAPI->DeltaType) < 0) { + goto error; + } + if (PyDict_SetItemString(dict, "tzinfo", (PyObject *)PyDateTimeAPI->TZInfoType) < 0) { + goto error; + } + return dict; +error: + Py_DECREF(dict); + return NULL; } static PyMethodDef test_methods[] = { @@ -475,11 +500,11 @@ static PyMethodDef test_methods[] = { {"get_datetime_fromdateandtimeandfold", get_datetime_fromdateandtimeandfold, METH_VARARGS}, {"get_datetime_fromtimestamp", get_datetime_fromtimestamp, METH_VARARGS}, {"get_delta_fromdsu", get_delta_fromdsu, METH_VARARGS}, - {"get_delta_type", get_delta_type, METH_NOARGS}, {"get_time_fromtime", get_time_fromtime, METH_VARARGS}, {"get_time_fromtimeandfold", get_time_fromtimeandfold, METH_VARARGS}, {"get_timezone_utc_capi", get_timezone_utc_capi, METH_VARARGS}, {"get_timezones_offset_zero", get_timezones_offset_zero, METH_NOARGS}, + {"get_capi_types", get_capi_types, METH_NOARGS}, {"make_timezones_capi", make_timezones_capi, METH_NOARGS}, {"test_datetime_capi", test_datetime_capi, METH_NOARGS}, {NULL}, From 351225e9ac0eb1818edfedccc331bfb6165b7e94 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 28 Apr 2025 07:56:39 +0900 Subject: [PATCH 34/55] Fix test_datetime_capi() for subinterps --- Lib/test/datetimetester.py | 40 ++++++++++++++++++++++-------------- Modules/_testcapi/datetime.c | 2 +- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 825aad21ec5023..1b1f7b57446132 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7157,7 +7157,6 @@ def test_datetime_from_timestamp(self): def assert_python_ok_in_subinterp(self, script, setup='_testcapi.test_datetime_capi()', - mainsetup='_testcapi.test_datetime_capi()', config='isolated'): # iOS requires the use of the custom framework loader, # not the ExtensionFileLoader. @@ -7166,7 +7165,7 @@ def assert_python_ok_in_subinterp(self, script, else: extension_loader = "ExtensionFileLoader" - maincode = textwrap.dedent(f''' + code = textwrap.dedent(f''' import textwrap from test import support @@ -7183,12 +7182,11 @@ def assert_python_ok_in_subinterp(self, script, _testcapi = importlib.util.module_from_spec(spec) spec.loader.exec_module(_testcapi) - $SETUP$ $SCRIPT$ """) import _testcapi - $MAINSETUP$ + $SETUP$ if {_interpreters is None}: ret = support.run_in_subinterp(subcode) @@ -7199,11 +7197,11 @@ def assert_python_ok_in_subinterp(self, script, assert ret == 0 - ''').replace('$MAINSETUP$', mainsetup) - maincode = maincode.replace('$SETUP$', textwrap.indent(setup, '\x20'*4)) - maincode = maincode.replace('$SCRIPT$', textwrap.indent(script, '\x20'*4)) + ''').rstrip() + code = code.replace('$SETUP$', setup) + code = code.replace('$SCRIPT$', textwrap.indent(script, '\x20'*4)) - res = script_helper.assert_python_ok('-c', maincode) + res = script_helper.assert_python_ok('-c', code) return res def test_type_check_in_subinterp(self): @@ -7212,6 +7210,7 @@ def run(type_checker, obj): if not type_checker(obj, True): raise TypeError(f'{{type(obj)}} is not C API type') + _testcapi.test_datetime_capi() import _datetime run(_testcapi.datetime_check_date, _datetime.date.today()) run(_testcapi.datetime_check_datetime, _datetime.datetime.now()) @@ -7219,11 +7218,10 @@ def run(type_checker, obj): run(_testcapi.datetime_check_delta, _datetime.timedelta(1)) run(_testcapi.datetime_check_tzinfo, _datetime.tzinfo()) """) - self.assert_python_ok_in_subinterp(script, mainsetup='') + self.assert_python_ok_in_subinterp(script, '') if _interpreters is not None: - with self.subTest('legacy'): - self.assert_python_ok_in_subinterp(script, mainsetup='', - config='legacy') + with self.subTest(name := 'legacy'): + self.assert_python_ok_in_subinterp(script, '', name) class ExtensionModuleTests(unittest.TestCase): @@ -7355,8 +7353,20 @@ def gen(): it = gen() next(it) """) - res = CapiTest.assert_python_ok_in_subinterp(self, script, setup='') - self.assertIn(b'ImportError: sys.meta_path is None', res.err) + + with self.subTest('PyDateTime_IMPORT by MainInterpreter'): + res = CapiTest.assert_python_ok_in_subinterp(self, script) + self.assertIn(b'ImportError: sys.meta_path is None', res.err) + + script2 = f'_testcapi.test_datetime_capi()\n{script}' + + with self.subTest('PyDateTime_IMPORT by Subinterpreter'): + res = CapiTest.assert_python_ok_in_subinterp(self, script2, '') + self.assertFalse(res.err) + + with self.subTest('PyDateTime_IMPORT by Main/Sub'): + res = CapiTest.assert_python_ok_in_subinterp(self, script2) + self.assertFalse(res.err) def test_static_type_before_shutdown(self): script = textwrap.dedent(f""" @@ -7366,7 +7376,7 @@ def test_static_type_before_shutdown(self): timedelta(days=1) assert '_datetime' in sys.modules """) - CapiTest.assert_python_ok_in_subinterp(self, script, setup='') + CapiTest.assert_python_ok_in_subinterp(self, script) def test_module_free(self): script = textwrap.dedent(""" diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index 0b4a968d0a0d0c..4ab18382c72a88 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -11,7 +11,7 @@ test_datetime_capi(PyObject *self, PyObject *args) if (PyDateTimeAPI) { if (test_run_counter) { /* Probably regrtest.py -R */ - Py_RETURN_NONE; + // Interpreters need their module, so call PyDateTime_IMPORT } else { PyErr_SetString(PyExc_AssertionError, From 419500eec2bbb62027261b175f9a5ad99f518807 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 28 Apr 2025 19:54:56 +0900 Subject: [PATCH 35/55] Use assert_python_failure() --- Lib/test/datetimetester.py | 39 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 1b1f7b57446132..3447ec86375d9f 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7155,9 +7155,9 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) - def assert_python_ok_in_subinterp(self, script, - setup='_testcapi.test_datetime_capi()', - config='isolated'): + def assert_python_in_subinterp(self, check_if_ok: bool, script, + setup='_testcapi.test_datetime_capi()', + config='isolated'): # iOS requires the use of the custom framework loader, # not the ExtensionFileLoader. if sys.platform == "ios": @@ -7201,7 +7201,10 @@ def assert_python_ok_in_subinterp(self, script, code = code.replace('$SETUP$', setup) code = code.replace('$SCRIPT$', textwrap.indent(script, '\x20'*4)) - res = script_helper.assert_python_ok('-c', code) + if check_if_ok: + res = script_helper.assert_python_ok('-c', code) + else: + res = script_helper.assert_python_failure('-c', code) return res def test_type_check_in_subinterp(self): @@ -7217,11 +7220,11 @@ def run(type_checker, obj): run(_testcapi.datetime_check_time, _datetime.time(12, 30)) run(_testcapi.datetime_check_delta, _datetime.timedelta(1)) run(_testcapi.datetime_check_tzinfo, _datetime.tzinfo()) - """) - self.assert_python_ok_in_subinterp(script, '') + """) + self.assert_python_in_subinterp(True, script, '') if _interpreters is not None: with self.subTest(name := 'legacy'): - self.assert_python_ok_in_subinterp(script, '', name) + self.assert_python_in_subinterp(True, script, '', name) class ExtensionModuleTests(unittest.TestCase): @@ -7316,7 +7319,7 @@ def gen(): it = gen() next(it) - """) + """) res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) @@ -7336,7 +7339,7 @@ def gen(): it = gen() next(it) - """) + """) res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) @@ -7352,20 +7355,18 @@ def gen(): it = gen() next(it) - """) - + """) with self.subTest('PyDateTime_IMPORT by MainInterpreter'): - res = CapiTest.assert_python_ok_in_subinterp(self, script) + res = CapiTest.assert_python_in_subinterp(self, True, script) self.assertIn(b'ImportError: sys.meta_path is None', res.err) script2 = f'_testcapi.test_datetime_capi()\n{script}' - with self.subTest('PyDateTime_IMPORT by Subinterpreter'): - res = CapiTest.assert_python_ok_in_subinterp(self, script2, '') + res = CapiTest.assert_python_in_subinterp(self, True, script2, '') self.assertFalse(res.err) with self.subTest('PyDateTime_IMPORT by Main/Sub'): - res = CapiTest.assert_python_ok_in_subinterp(self, script2) + res = CapiTest.assert_python_in_subinterp(self, True, script2, '') self.assertFalse(res.err) def test_static_type_before_shutdown(self): @@ -7375,8 +7376,8 @@ def test_static_type_before_shutdown(self): timedelta = _testcapi.get_capi_types()['timedelta'] timedelta(days=1) assert '_datetime' in sys.modules - """) - CapiTest.assert_python_ok_in_subinterp(self, script) + """) + CapiTest.assert_python_in_subinterp(self, True, script) def test_module_free(self): script = textwrap.dedent(""" @@ -7392,7 +7393,7 @@ def test_module_free(self): del _datetime gc.collect() assert len(ws) == 0 - """) + """) script_helper.assert_python_ok('-c', script) @unittest.skipIf(not support.Py_DEBUG, "Debug builds only") @@ -7400,7 +7401,7 @@ def test_no_leak(self): script = textwrap.dedent(""" import datetime datetime.datetime.strptime('20000101', '%Y%m%d').strftime('%Y%m%d') - """) + """) res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) self.assertIn(b'[0 refs, 0 blocks]', res.err) From 596007c00ce06c1d5761f580c49692d29289b9a2 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 28 Apr 2025 20:30:15 +0900 Subject: [PATCH 36/55] Add a failure test (crash) --- Lib/test/datetimetester.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 3447ec86375d9f..4f6668db4e4d96 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7301,6 +7301,26 @@ def test_update_type_cache(self): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) + def test_static_type_attr_on_subinterp(self): + script = textwrap.dedent(f""" + date = _testcapi.get_capi_types()['date'] + date.today + """) + # Fail before loaded + with self.subTest('[PyDateTime_IMPORT] main: yes sub: no'): + res = CapiTest.assert_python_in_subinterp(self, False, script) + self.assertIn(b'_PyType_CheckConsistency: Assertion failed', res.err) + self.assertIn(b'lookup_tp_dict(type) != ((void *)0)', res.err) + + # OK after loaded + with self.subTest('[PyDateTime_IMPORT] main: no sub: yes'): + script2 = f'_testcapi.test_datetime_capi()\n{script}' + CapiTest.assert_python_in_subinterp(self, True, script2) + + with self.subTest('Regular'): + script2 = f'import _datetime\n{script}' + CapiTest.assert_python_in_subinterp(self, True, script2) + def test_static_type_at_shutdown1(self): # gh-132413 script = textwrap.dedent(""" From 1dd9707405ec91aae1588aead94424c7d93a5d08 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 28 Apr 2025 21:12:51 +0900 Subject: [PATCH 37/55] Correct the previous commit --- Lib/test/datetimetester.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 4f6668db4e4d96..547fe885f12cf3 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7307,20 +7307,21 @@ def test_static_type_attr_on_subinterp(self): date.today """) # Fail before loaded - with self.subTest('[PyDateTime_IMPORT] main: yes sub: no'): - res = CapiTest.assert_python_in_subinterp(self, False, script) - self.assertIn(b'_PyType_CheckConsistency: Assertion failed', res.err) - self.assertIn(b'lookup_tp_dict(type) != ((void *)0)', res.err) + with self.subTest('[PyDateTime_IMPORT] main: yes, sub: no'): + CapiTest.assert_python_in_subinterp(self, False, script) # OK after loaded - with self.subTest('[PyDateTime_IMPORT] main: no sub: yes'): - script2 = f'_testcapi.test_datetime_capi()\n{script}' - CapiTest.assert_python_in_subinterp(self, True, script2) + script2 = f'_testcapi.test_datetime_capi()\n{script}' + with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): + CapiTest.assert_python_in_subinterp(self, True, script2, '') - with self.subTest('Regular'): - script2 = f'import _datetime\n{script}' + with self.subTest('[PyDateTime_IMPORT] main: yes, sub: yes'): CapiTest.assert_python_in_subinterp(self, True, script2) + script3 = f'import _datetime\n{script}' + with self.subTest('Regular import'): + CapiTest.assert_python_in_subinterp(self, True, script3) + def test_static_type_at_shutdown1(self): # gh-132413 script = textwrap.dedent(""" From fb35703cc11fc6fa7196465ebdfd34268a46cece Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 29 Apr 2025 03:00:49 +0900 Subject: [PATCH 38/55] Redirect to the CapiTest method --- Lib/test/datetimetester.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 547fe885f12cf3..9f824ca2306cbd 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7301,6 +7301,9 @@ def test_update_type_cache(self): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) + def assert_python_in_subinterp(self, *args, **kwargs): + return CapiTest.assert_python_in_subinterp(self, *args, **kwargs) + def test_static_type_attr_on_subinterp(self): script = textwrap.dedent(f""" date = _testcapi.get_capi_types()['date'] @@ -7308,19 +7311,20 @@ def test_static_type_attr_on_subinterp(self): """) # Fail before loaded with self.subTest('[PyDateTime_IMPORT] main: yes, sub: no'): - CapiTest.assert_python_in_subinterp(self, False, script) + self.assert_python_in_subinterp(False, script) # OK after loaded script2 = f'_testcapi.test_datetime_capi()\n{script}' with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): - CapiTest.assert_python_in_subinterp(self, True, script2, '') + self.assert_python_in_subinterp(True, script2, setup='') with self.subTest('[PyDateTime_IMPORT] main: yes, sub: yes'): - CapiTest.assert_python_in_subinterp(self, True, script2) + # Check if each test_datetime_capi() calls PyDateTime_IMPORT + self.assert_python_in_subinterp(True, script2) script3 = f'import _datetime\n{script}' with self.subTest('Regular import'): - CapiTest.assert_python_in_subinterp(self, True, script3) + self.assert_python_in_subinterp(True, script3) def test_static_type_at_shutdown1(self): # gh-132413 @@ -7377,17 +7381,18 @@ def gen(): it = gen() next(it) """) - with self.subTest('PyDateTime_IMPORT by MainInterpreter'): - res = CapiTest.assert_python_in_subinterp(self, True, script) + with self.subTest('[PyDateTime_IMPORT] main: yes, sub: no'): + res = self.assert_python_in_subinterp(True, script) self.assertIn(b'ImportError: sys.meta_path is None', res.err) script2 = f'_testcapi.test_datetime_capi()\n{script}' - with self.subTest('PyDateTime_IMPORT by Subinterpreter'): - res = CapiTest.assert_python_in_subinterp(self, True, script2, '') + with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): + res = self.assert_python_in_subinterp(True, script2, setup='') self.assertFalse(res.err) - with self.subTest('PyDateTime_IMPORT by Main/Sub'): - res = CapiTest.assert_python_in_subinterp(self, True, script2, '') + with self.subTest('[PyDateTime_IMPORT] main: yes, sub: yes'): + # Check if each test_datetime_capi() calls PyDateTime_IMPORT + res = self.assert_python_in_subinterp(True, script2) self.assertFalse(res.err) def test_static_type_before_shutdown(self): @@ -7398,7 +7403,7 @@ def test_static_type_before_shutdown(self): timedelta(days=1) assert '_datetime' in sys.modules """) - CapiTest.assert_python_in_subinterp(self, True, script) + self.assert_python_in_subinterp(True, script) def test_module_free(self): script = textwrap.dedent(""" From c6cc066d8bd2f283567b644f7091dc70255aad97 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 29 Apr 2025 03:02:57 +0900 Subject: [PATCH 39/55] Merge two tests (1) --- Lib/test/datetimetester.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 9f824ca2306cbd..a09cf6a1b2c5f2 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7304,7 +7304,7 @@ def test_update_type_cache(self): def assert_python_in_subinterp(self, *args, **kwargs): return CapiTest.assert_python_in_subinterp(self, *args, **kwargs) - def test_static_type_attr_on_subinterp(self): + def test_static_type_on_subinterp(self): script = textwrap.dedent(f""" date = _testcapi.get_capi_types()['date'] date.today @@ -7326,6 +7326,18 @@ def test_static_type_attr_on_subinterp(self): with self.subTest('Regular import'): self.assert_python_in_subinterp(True, script3) + script4 = textwrap.dedent(f""" + import sys + assert '_datetime' not in sys.modules + timedelta = _testcapi.get_capi_types()['timedelta'] + timedelta(days=1) + assert '_datetime' in sys.modules + date = _testcapi.get_capi_types()['date'] + date.today + """) + with self.subTest('Implicit import'): + self.assert_python_in_subinterp(True, script4) + def test_static_type_at_shutdown1(self): # gh-132413 script = textwrap.dedent(""" @@ -7395,16 +7407,6 @@ def gen(): res = self.assert_python_in_subinterp(True, script2) self.assertFalse(res.err) - def test_static_type_before_shutdown(self): - script = textwrap.dedent(f""" - import sys - assert '_datetime' not in sys.modules - timedelta = _testcapi.get_capi_types()['timedelta'] - timedelta(days=1) - assert '_datetime' in sys.modules - """) - self.assert_python_in_subinterp(True, script) - def test_module_free(self): script = textwrap.dedent(""" import sys From cc04c6f4bfc6489908839332d8b7782604f4c88e Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 29 Apr 2025 03:07:58 +0900 Subject: [PATCH 40/55] Merge two tests (2) --- Lib/test/datetimetester.py | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index a09cf6a1b2c5f2..070174e56db53a 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7338,11 +7338,12 @@ def test_static_type_on_subinterp(self): with self.subTest('Implicit import'): self.assert_python_in_subinterp(True, script4) - def test_static_type_at_shutdown1(self): + def test_static_type_at_shutdown(self): # gh-132413 script = textwrap.dedent(""" import sys import _datetime + timedelta = _datetime.timedelta def gen(): try: @@ -7352,24 +7353,6 @@ def gen(): assert not sys.modules td = _datetime.timedelta(days=1) assert td.days == 1 - assert not sys.modules - - it = gen() - next(it) - """) - res = script_helper.assert_python_ok('-c', script) - self.assertFalse(res.err) - - def test_static_type_at_shutdown2(self): - script = textwrap.dedent(""" - import sys - from _datetime import timedelta - - def gen(): - try: - yield - finally: - assert not sys.modules td = timedelta(days=1) assert td.days == 1 assert not sys.modules @@ -7377,8 +7360,12 @@ def gen(): it = gen() next(it) """) - res = script_helper.assert_python_ok('-c', script) - self.assertFalse(res.err) + with self.subTest('MainInterpreter'): + res = script_helper.assert_python_ok('-c', script) + self.assertFalse(res.err) + with self.subTest('Subinterpreter'): + res = self.assert_python_in_subinterp(True, script, setup='') + self.assertFalse(res.err) def test_static_type_at_shutdown3(self): script = textwrap.dedent(""" From 56267bdc968c1de5ca84e541540bb8bafcde326c Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 29 Apr 2025 03:10:04 +0900 Subject: [PATCH 41/55] Merge two tests (3) --- Lib/test/datetimetester.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 070174e56db53a..e6eeec4144d6ef 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7367,15 +7367,19 @@ def gen(): res = self.assert_python_in_subinterp(True, script, setup='') self.assertFalse(res.err) - def test_static_type_at_shutdown3(self): script = textwrap.dedent(""" + import sys timedelta = _testcapi.get_capi_types()['timedelta'] def gen(): try: yield finally: - timedelta(days=1) + # Exceptions are ignored here + assert not sys.modules + td = timedelta(days=1) + assert td.days == 1 + assert not sys.modules it = gen() next(it) From 063f37dd26ea675ec249f2c9cbb391e259d72171 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 29 Apr 2025 03:11:09 +0900 Subject: [PATCH 42/55] Add test cases --- Lib/test/datetimetester.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index e6eeec4144d6ef..9b33f4f700efa5 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7367,6 +7367,40 @@ def gen(): res = self.assert_python_in_subinterp(True, script, setup='') self.assertFalse(res.err) + script = textwrap.dedent(""" + import gc + import sys + import _testcapi + + def emulate_interp_restart(): + del sys.modules['_datetime'] + try: + del sys.modules['datetime'] + except KeyError: + pass + gc.collect() # unload + + _testcapi.test_datetime_capi() # only once + timedelta = _testcapi.get_capi_types()['timedelta'] + emulate_interp_restart() + timedelta(days=1) + emulate_interp_restart() + + def gen(): + try: + yield + finally: + # Exceptions are ignored here + assert not sys.modules + timedelta(days=1) + + it = gen() + next(it) + """) + with self.subTest('MainInterpreter Restart'): + res = script_helper.assert_python_ok('-c', script) + self.assertIn(b'ImportError: sys.meta_path is None', res.err) + script = textwrap.dedent(""" import sys timedelta = _testcapi.get_capi_types()['timedelta'] From 992fd0cf6039d27275c3c82a65805a5cf6b68671 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 30 Apr 2025 09:24:48 +0900 Subject: [PATCH 43/55] Move up a few tests --- Lib/test/datetimetester.py | 56 +++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 9b33f4f700efa5..5b1b280b3a4221 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7233,6 +7233,9 @@ def setUp(self): if self.__class__.__name__.endswith('Pure'): self.skipTest('Not relevant in pure Python') + def assert_python_in_subinterp(self, *args, **kwargs): + return CapiTest.assert_python_in_subinterp(self, *args, **kwargs) + @support.cpython_only def test_gh_120161(self): with self.subTest('simple'): @@ -7301,8 +7304,31 @@ def test_update_type_cache(self): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) - def assert_python_in_subinterp(self, *args, **kwargs): - return CapiTest.assert_python_in_subinterp(self, *args, **kwargs) + def test_module_free(self): + script = textwrap.dedent(""" + import sys + import gc + import weakref + ws = weakref.WeakSet() + for _ in range(3): + import _datetime + timedelta = _datetime.timedelta # static type + ws.add(_datetime) + del sys.modules['_datetime'] + del _datetime + gc.collect() + assert len(ws) == 0 + """) + script_helper.assert_python_ok('-c', script) + + @unittest.skipIf(not support.Py_DEBUG, "Debug builds only") + def test_no_leak(self): + script = textwrap.dedent(""" + import datetime + datetime.datetime.strptime('20000101', '%Y%m%d').strftime('%Y%m%d') + """) + res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) + self.assertIn(b'[0 refs, 0 blocks]', res.err) def test_static_type_on_subinterp(self): script = textwrap.dedent(f""" @@ -7432,32 +7458,6 @@ def gen(): res = self.assert_python_in_subinterp(True, script2) self.assertFalse(res.err) - def test_module_free(self): - script = textwrap.dedent(""" - import sys - import gc - import weakref - ws = weakref.WeakSet() - for _ in range(3): - import _datetime - timedelta = _datetime.timedelta - ws.add(_datetime) - del sys.modules['_datetime'] - del _datetime - gc.collect() - assert len(ws) == 0 - """) - script_helper.assert_python_ok('-c', script) - - @unittest.skipIf(not support.Py_DEBUG, "Debug builds only") - def test_no_leak(self): - script = textwrap.dedent(""" - import datetime - datetime.datetime.strptime('20000101', '%Y%m%d').strftime('%Y%m%d') - """) - res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) - self.assertIn(b'[0 refs, 0 blocks]', res.err) - def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) From f0f8fa0894bfc1a984cb665fa3d1aca428b139b9 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 30 Apr 2025 09:30:26 +0900 Subject: [PATCH 44/55] Update tests --- Lib/test/datetimetester.py | 119 ++++++++++++++++------------------- Modules/_testcapi/datetime.c | 5 +- 2 files changed, 57 insertions(+), 67 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 5b1b280b3a4221..2deeec663e5e01 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7155,9 +7155,8 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) - def assert_python_in_subinterp(self, check_if_ok: bool, script, - setup='_testcapi.test_datetime_capi()', - config='isolated'): + def assert_python_in_subinterp(self, check_if_ok, script, init='', + fini='', repeat=1, config='isolated'): # iOS requires the use of the custom framework loader, # not the ExtensionFileLoader. if sys.platform == "ios": @@ -7166,40 +7165,41 @@ def assert_python_in_subinterp(self, check_if_ok: bool, script, extension_loader = "ExtensionFileLoader" code = textwrap.dedent(f''' - import textwrap - from test import support - - subcode = textwrap.dedent(""" - if {_interpreters is None}: - import _testcapi - else: - import importlib.machinery - import importlib.util - fullname = '_testcapi_datetime' - origin = importlib.util.find_spec('_testcapi').origin - loader = importlib.machinery.{extension_loader}(fullname, origin) - spec = importlib.util.spec_from_loader(fullname, loader) - _testcapi = importlib.util.module_from_spec(spec) - spec.loader.exec_module(_testcapi) - - $SCRIPT$ - """) - - import _testcapi - $SETUP$ - + subinterp_code = """ if {_interpreters is None}: - ret = support.run_in_subinterp(subcode) + import _testcapi else: - import _interpreters - config = _interpreters.new_config('{config}').__dict__ - ret = support.run_in_subinterp_with_config(subcode, **config) + import importlib.machinery + import importlib.util + fullname = '_testcapi_datetime' + origin = importlib.util.find_spec('_testcapi').origin + loader = importlib.machinery.{extension_loader}(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + _testcapi = importlib.util.module_from_spec(spec) + spec.loader.exec_module(_testcapi) + INDEX = $INDEX$ + setup = _testcapi.test_datetime_capi # call it if needed + $SCRIPT$ + """ - assert ret == 0 + import _testcapi + from test import support + setup = _testcapi.test_datetime_capi + $INIT$ - ''').rstrip() - code = code.replace('$SETUP$', setup) - code = code.replace('$SCRIPT$', textwrap.indent(script, '\x20'*4)) + for idx in range({repeat}): + subcode = subinterp_code.replace('$INDEX$', str(idx)) + if {_interpreters is None}: + ret = support.run_in_subinterp(subcode) + else: + import _interpreters + config = _interpreters.new_config('{config}').__dict__ + ret = support.run_in_subinterp_with_config(subcode, **config) + assert ret == 0 + $FINI$ + ''') + code = code.replace('$INIT$', init).replace('$FINI$', fini) + code = code.replace('$SCRIPT$', script) if check_if_ok: res = script_helper.assert_python_ok('-c', code) @@ -7213,7 +7213,7 @@ def run(type_checker, obj): if not type_checker(obj, True): raise TypeError(f'{{type(obj)}} is not C API type') - _testcapi.test_datetime_capi() + setup() import _datetime run(_testcapi.datetime_check_date, _datetime.date.today()) run(_testcapi.datetime_check_datetime, _datetime.datetime.now()) @@ -7221,10 +7221,10 @@ def run(type_checker, obj): run(_testcapi.datetime_check_delta, _datetime.timedelta(1)) run(_testcapi.datetime_check_tzinfo, _datetime.tzinfo()) """) - self.assert_python_in_subinterp(True, script, '') + self.assert_python_in_subinterp(True, script) if _interpreters is not None: with self.subTest(name := 'legacy'): - self.assert_python_in_subinterp(True, script, '', name) + self.assert_python_in_subinterp(True, script, config=name) class ExtensionModuleTests(unittest.TestCase): @@ -7331,38 +7331,34 @@ def test_no_leak(self): self.assertIn(b'[0 refs, 0 blocks]', res.err) def test_static_type_on_subinterp(self): - script = textwrap.dedent(f""" + script = textwrap.dedent(""" date = _testcapi.get_capi_types()['date'] date.today """) - # Fail before loaded with self.subTest('[PyDateTime_IMPORT] main: yes, sub: no'): - self.assert_python_in_subinterp(False, script) + # FIXME: Segfault + self.assert_python_in_subinterp(False, script, 'setup()') - # OK after loaded - script2 = f'_testcapi.test_datetime_capi()\n{script}' + with_setup = 'setup()' + script with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): - self.assert_python_in_subinterp(True, script2, setup='') + self.assert_python_in_subinterp(True, with_setup) with self.subTest('[PyDateTime_IMPORT] main: yes, sub: yes'): - # Check if each test_datetime_capi() calls PyDateTime_IMPORT - self.assert_python_in_subinterp(True, script2) + # Check if PyDateTime_IMPORT is invoked not only once + self.assert_python_in_subinterp(True, with_setup, 'setup()') + self.assert_python_in_subinterp(True, 'setup()', fini=with_setup) + self.assert_python_in_subinterp(True, with_setup, repeat=2) - script3 = f'import _datetime\n{script}' - with self.subTest('Regular import'): - self.assert_python_in_subinterp(True, script3) + with_import = 'import _datetime' + script + with self.subTest('Explicit import'): + self.assert_python_in_subinterp(True, with_import, 'setup()') - script4 = textwrap.dedent(f""" - import sys - assert '_datetime' not in sys.modules + with_import = textwrap.dedent(""" timedelta = _testcapi.get_capi_types()['timedelta'] timedelta(days=1) - assert '_datetime' in sys.modules - date = _testcapi.get_capi_types()['date'] - date.today - """) + """) + script with self.subTest('Implicit import'): - self.assert_python_in_subinterp(True, script4) + self.assert_python_in_subinterp(True, with_import, 'setup()') def test_static_type_at_shutdown(self): # gh-132413 @@ -7390,7 +7386,7 @@ def gen(): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) with self.subTest('Subinterpreter'): - res = self.assert_python_in_subinterp(True, script, setup='') + res = self.assert_python_in_subinterp(True, script) self.assertFalse(res.err) script = textwrap.dedent(""" @@ -7445,17 +7441,12 @@ def gen(): next(it) """) with self.subTest('[PyDateTime_IMPORT] main: yes, sub: no'): - res = self.assert_python_in_subinterp(True, script) + res = self.assert_python_in_subinterp(True, script, 'setup()') self.assertIn(b'ImportError: sys.meta_path is None', res.err) - script2 = f'_testcapi.test_datetime_capi()\n{script}' + with_import = 'setup()' + script with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): - res = self.assert_python_in_subinterp(True, script2, setup='') - self.assertFalse(res.err) - - with self.subTest('[PyDateTime_IMPORT] main: yes, sub: yes'): - # Check if each test_datetime_capi() calls PyDateTime_IMPORT - res = self.assert_python_in_subinterp(True, script2) + res = self.assert_python_in_subinterp(True, with_import) self.assertFalse(res.err) diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index 4ab18382c72a88..42bce301a51277 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -11,7 +11,6 @@ test_datetime_capi(PyObject *self, PyObject *args) if (PyDateTimeAPI) { if (test_run_counter) { /* Probably regrtest.py -R */ - // Interpreters need their module, so call PyDateTime_IMPORT } else { PyErr_SetString(PyExc_AssertionError, @@ -20,7 +19,7 @@ test_datetime_capi(PyObject *self, PyObject *args) } } test_run_counter++; - PyDateTime_IMPORT; + PyDateTime_IMPORT; // Ensure interpreters individually import a module if (PyDateTimeAPI == NULL) { return NULL; @@ -527,7 +526,7 @@ _PyTestCapi_Init_DateTime(PyObject *mod) static int _testcapi_datetime_exec(PyObject *mod) { - // Call test_datetime_capi() in each test. + // The execution does not invoke test_datetime_capi() return 0; } From 7f31245634aafe3e7042346afdff5e9f6dfcf106 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 30 Apr 2025 09:31:14 +0900 Subject: [PATCH 45/55] Move a repeat test to test_embed --- Lib/test/datetimetester.py | 34 ---------------------------------- Lib/test/test_embed.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 2deeec663e5e01..9f21a23524b2da 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7389,40 +7389,6 @@ def gen(): res = self.assert_python_in_subinterp(True, script) self.assertFalse(res.err) - script = textwrap.dedent(""" - import gc - import sys - import _testcapi - - def emulate_interp_restart(): - del sys.modules['_datetime'] - try: - del sys.modules['datetime'] - except KeyError: - pass - gc.collect() # unload - - _testcapi.test_datetime_capi() # only once - timedelta = _testcapi.get_capi_types()['timedelta'] - emulate_interp_restart() - timedelta(days=1) - emulate_interp_restart() - - def gen(): - try: - yield - finally: - # Exceptions are ignored here - assert not sys.modules - timedelta(days=1) - - it = gen() - next(it) - """) - with self.subTest('MainInterpreter Restart'): - res = script_helper.assert_python_ok('-c', script) - self.assertIn(b'ImportError: sys.meta_path is None', res.err) - script = textwrap.dedent(""" import sys timedelta = _testcapi.get_capi_types()['timedelta'] diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index e06e684408ca6b..afe2c2286b67ad 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -440,6 +440,37 @@ def test_datetime_reset_strptime(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '20000101\n' * INIT_LOOPS) + def test_datetime_capi_at_shutdown(self): + # gh-120782: Test the case where PyDateTime_IMPORT is called only once + code = textwrap.dedent(""" + import sys + import _testcapi + if not _testcapi.get_capi_types(): + _testcapi.test_datetime_capi() + assert '_datetime' in sys.modules + else: + assert '_datetime' not in sys.modules + timedelta = _testcapi.get_capi_types()['timedelta'] + + def gen(): + try: + yield + finally: + assert not sys.modules + res = 1 + try: + timedelta(days=1) + except ImportError as e: + res = 2 if 'sys.meta_path is None' in e.msg else 3 + assert not sys.modules + print(res) + + it = gen() + next(it) + """) + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(out, '1\n' + '2\n' * (INIT_LOOPS - 1)) + def test_static_types_inherited_slots(self): script = textwrap.dedent(""" import test.support From d7f80dd5f43407c720e45af83b403391a14284c8 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 30 Apr 2025 13:12:30 +0900 Subject: [PATCH 46/55] Restore and keep _Py_ID for now --- Include/internal/pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + Include/internal/pycore_runtime_init_generated.h | 1 + Include/internal/pycore_unicodeobject_generated.h | 4 ++++ Modules/_datetimemodule.c | 2 ++ 5 files changed, 9 insertions(+) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 53e7593e1584e4..e412db1de68f8b 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -835,6 +835,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_call)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_return)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_datetime_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_statements)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cafile)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index df8cae8c7efec3..2a6c2065af6bb9 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -326,6 +326,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(c_call) STRUCT_FOR_ID(c_exception) STRUCT_FOR_ID(c_return) + STRUCT_FOR_ID(cached_datetime_module) STRUCT_FOR_ID(cached_statements) STRUCT_FOR_ID(cadata) STRUCT_FOR_ID(cafile) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index e083c858b4f5ea..2368157a4fd18b 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -833,6 +833,7 @@ extern "C" { INIT_ID(c_call), \ INIT_ID(c_exception), \ INIT_ID(c_return), \ + INIT_ID(cached_datetime_module), \ INIT_ID(cached_statements), \ INIT_ID(cadata), \ INIT_ID(cafile), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index f5cf6ed5b4c6ac..72c3346328a552 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1092,6 +1092,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(cached_datetime_module); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cached_statements); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index cad082a1521ac1..ef19b1bca664b6 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -125,6 +125,8 @@ get_module_state(PyObject *module) } +#define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module)) // unused + static PyObject * get_current_module(PyInterpreterState *interp, int *p_reloading) { From 23bdffd606c31eca8a2fa5c9d2b29c0f5ddf452c Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 30 Apr 2025 13:13:20 +0900 Subject: [PATCH 47/55] Add a demonstration to test_embed --- Lib/test/test_embed.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index afe2c2286b67ad..23199d50ac3d21 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -441,7 +441,7 @@ def test_datetime_reset_strptime(self): self.assertEqual(out, '20000101\n' * INIT_LOOPS) def test_datetime_capi_at_shutdown(self): - # gh-120782: Test the case where PyDateTime_IMPORT is called only once + # gh-120782: Current datetime test calls PyDateTime_IMPORT only once code = textwrap.dedent(""" import sys import _testcapi @@ -471,6 +471,34 @@ def gen(): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '1\n' + '2\n' * (INIT_LOOPS - 1)) + def test_datetime_capi_at_shutdown2(self): + # gh-120782: This PR allows PyDateTime_IMPORT to be called on restart + code = textwrap.dedent(""" + import sys + import _testcapi + _testcapi.test_datetime_capi() + assert '_datetime' in sys.modules + timedelta = _testcapi.get_capi_types()['timedelta'] + + def gen(): + try: + yield + finally: + assert not sys.modules + res = 1 + try: + timedelta(days=1) + except ImportError as e: + res = 2 if 'sys.meta_path is None' in e.msg else 3 + assert not sys.modules + print(res) + + it = gen() + next(it) + """) + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(out, '1\n' * INIT_LOOPS) + def test_static_types_inherited_slots(self): script = textwrap.dedent(""" import test.support From ad7dfd2dfb012ba6c7a68f1996ab0057954f5cb0 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Thu, 1 May 2025 01:20:58 +0900 Subject: [PATCH 48/55] Introduce test_datetime_capi_newinterp() --- Lib/test/datetimetester.py | 11 ++++----- Lib/test/test_embed.py | 43 +++++------------------------------- Modules/_testcapi/datetime.c | 29 ++++++++++++++++++++++-- 3 files changed, 37 insertions(+), 46 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 9f21a23524b2da..d38168f567b3e1 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7178,13 +7178,13 @@ def assert_python_in_subinterp(self, check_if_ok, script, init='', _testcapi = importlib.util.module_from_spec(spec) spec.loader.exec_module(_testcapi) INDEX = $INDEX$ - setup = _testcapi.test_datetime_capi # call it if needed + setup = _testcapi.test_datetime_capi_newinterp # call it if needed $SCRIPT$ """ import _testcapi from test import support - setup = _testcapi.test_datetime_capi + setup = _testcapi.test_datetime_capi_newinterp $INIT$ for idx in range({repeat}): @@ -7335,16 +7335,13 @@ def test_static_type_on_subinterp(self): date = _testcapi.get_capi_types()['date'] date.today """) - with self.subTest('[PyDateTime_IMPORT] main: yes, sub: no'): - # FIXME: Segfault - self.assert_python_in_subinterp(False, script, 'setup()') - with_setup = 'setup()' + script with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): self.assert_python_in_subinterp(True, with_setup) with self.subTest('[PyDateTime_IMPORT] main: yes, sub: yes'): - # Check if PyDateTime_IMPORT is invoked not only once + # Fails if the setup() means test_datetime_capi() rather than + # test_datetime_capi_newinterp() self.assert_python_in_subinterp(True, with_setup, 'setup()') self.assert_python_in_subinterp(True, 'setup()', fini=with_setup) self.assert_python_in_subinterp(True, with_setup, repeat=2) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 23199d50ac3d21..e859fa78d11de8 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -441,15 +441,12 @@ def test_datetime_reset_strptime(self): self.assertEqual(out, '20000101\n' * INIT_LOOPS) def test_datetime_capi_at_shutdown(self): - # gh-120782: Current datetime test calls PyDateTime_IMPORT only once + # gh-132413: datetime module is currently tested in an interp's life. + # PyDateTime_IMPORT needs to be called at least once after the restart. code = textwrap.dedent(""" import sys import _testcapi - if not _testcapi.get_capi_types(): - _testcapi.test_datetime_capi() - assert '_datetime' in sys.modules - else: - assert '_datetime' not in sys.modules + _testcapi.test_datetime_capi_newinterp() timedelta = _testcapi.get_capi_types()['timedelta'] def gen(): @@ -457,40 +454,12 @@ def gen(): yield finally: assert not sys.modules - res = 1 + res = 0 try: timedelta(days=1) + res = 1 except ImportError as e: - res = 2 if 'sys.meta_path is None' in e.msg else 3 - assert not sys.modules - print(res) - - it = gen() - next(it) - """) - out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) - self.assertEqual(out, '1\n' + '2\n' * (INIT_LOOPS - 1)) - - def test_datetime_capi_at_shutdown2(self): - # gh-120782: This PR allows PyDateTime_IMPORT to be called on restart - code = textwrap.dedent(""" - import sys - import _testcapi - _testcapi.test_datetime_capi() - assert '_datetime' in sys.modules - timedelta = _testcapi.get_capi_types()['timedelta'] - - def gen(): - try: - yield - finally: - assert not sys.modules - res = 1 - try: - timedelta(days=1) - except ImportError as e: - res = 2 if 'sys.meta_path is None' in e.msg else 3 - assert not sys.modules + res = 2 print(res) it = gen() diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index 42bce301a51277..375196e28fb727 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -11,6 +11,7 @@ test_datetime_capi(PyObject *self, PyObject *args) if (PyDateTimeAPI) { if (test_run_counter) { /* Probably regrtest.py -R */ + Py_RETURN_NONE; } else { PyErr_SetString(PyExc_AssertionError, @@ -19,7 +20,7 @@ test_datetime_capi(PyObject *self, PyObject *args) } } test_run_counter++; - PyDateTime_IMPORT; // Ensure interpreters individually import a module + PyDateTime_IMPORT; if (PyDateTimeAPI == NULL) { return NULL; @@ -34,6 +35,29 @@ test_datetime_capi(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +test_datetime_capi_newinterp(PyObject *self, PyObject *args) +{ + // Call PyDateTime_IMPORT at least once in each interpreter's life + if (PyDateTimeAPI != NULL && test_run_counter == 0) { + PyErr_SetString(PyExc_AssertionError, + "PyDateTime_CAPI somehow initialized"); + return NULL; + } + test_run_counter++; + PyDateTime_IMPORT; + + if (PyDateTimeAPI == NULL) { + return NULL; + } + assert(!PyType_HasFeature(PyDateTimeAPI->DateType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->TimeType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->DateTimeType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->DeltaType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->TZInfoType, Py_TPFLAGS_HEAPTYPE)); + Py_RETURN_NONE; +} + /* Functions exposing the C API type checking for testing */ #define MAKE_DATETIME_CHECK_FUNC(check_method, exact_method) \ do { \ @@ -506,6 +530,7 @@ static PyMethodDef test_methods[] = { {"get_capi_types", get_capi_types, METH_NOARGS}, {"make_timezones_capi", make_timezones_capi, METH_NOARGS}, {"test_datetime_capi", test_datetime_capi, METH_NOARGS}, + {"test_datetime_capi_newinterp",test_datetime_capi_newinterp, METH_NOARGS}, {NULL}, }; @@ -526,7 +551,7 @@ _PyTestCapi_Init_DateTime(PyObject *mod) static int _testcapi_datetime_exec(PyObject *mod) { - // The execution does not invoke test_datetime_capi() + // The execution does not invoke PyDateTime_IMPORT return 0; } From 37b27747e7464a6a83377843fc07ee6ca63ace31 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Thu, 1 May 2025 09:22:48 +0900 Subject: [PATCH 49/55] Cleanup --- Lib/test/datetimetester.py | 43 +++++++++++++++++--------------------- Lib/test/test_embed.py | 2 +- Modules/_datetimemodule.c | 2 -- 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index d38168f567b3e1..88c153f8379025 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7155,8 +7155,8 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) - def assert_python_in_subinterp(self, check_if_ok, script, init='', - fini='', repeat=1, config='isolated'): + def assert_python_ok_in_subinterp(self, script, init='', fini='', + repeat=1, config='isolated'): # iOS requires the use of the custom framework loader, # not the ExtensionFileLoader. if sys.platform == "ios": @@ -7177,7 +7177,7 @@ def assert_python_in_subinterp(self, check_if_ok, script, init='', spec = importlib.util.spec_from_loader(fullname, loader) _testcapi = importlib.util.module_from_spec(spec) spec.loader.exec_module(_testcapi) - INDEX = $INDEX$ + run_counter = $RUN_COUNTER$ setup = _testcapi.test_datetime_capi_newinterp # call it if needed $SCRIPT$ """ @@ -7187,8 +7187,8 @@ def assert_python_in_subinterp(self, check_if_ok, script, init='', setup = _testcapi.test_datetime_capi_newinterp $INIT$ - for idx in range({repeat}): - subcode = subinterp_code.replace('$INDEX$', str(idx)) + for i in range(1, {1 + repeat}): + subcode = subinterp_code.replace('$RUN_COUNTER$', str(i)) if {_interpreters is None}: ret = support.run_in_subinterp(subcode) else: @@ -7200,12 +7200,7 @@ def assert_python_in_subinterp(self, check_if_ok, script, init='', ''') code = code.replace('$INIT$', init).replace('$FINI$', fini) code = code.replace('$SCRIPT$', script) - - if check_if_ok: - res = script_helper.assert_python_ok('-c', code) - else: - res = script_helper.assert_python_failure('-c', code) - return res + return script_helper.assert_python_ok('-c', code) def test_type_check_in_subinterp(self): script = textwrap.dedent(f""" @@ -7221,10 +7216,10 @@ def run(type_checker, obj): run(_testcapi.datetime_check_delta, _datetime.timedelta(1)) run(_testcapi.datetime_check_tzinfo, _datetime.tzinfo()) """) - self.assert_python_in_subinterp(True, script) + self.assert_python_ok_in_subinterp(script) if _interpreters is not None: with self.subTest(name := 'legacy'): - self.assert_python_in_subinterp(True, script, config=name) + self.assert_python_ok_in_subinterp(script, config=name) class ExtensionModuleTests(unittest.TestCase): @@ -7233,8 +7228,8 @@ def setUp(self): if self.__class__.__name__.endswith('Pure'): self.skipTest('Not relevant in pure Python') - def assert_python_in_subinterp(self, *args, **kwargs): - return CapiTest.assert_python_in_subinterp(self, *args, **kwargs) + def assert_python_ok_in_subinterp(self, *args, **kwargs): + return CapiTest.assert_python_ok_in_subinterp(self, *args, **kwargs) @support.cpython_only def test_gh_120161(self): @@ -7337,25 +7332,25 @@ def test_static_type_on_subinterp(self): """) with_setup = 'setup()' + script with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): - self.assert_python_in_subinterp(True, with_setup) + self.assert_python_ok_in_subinterp(with_setup) with self.subTest('[PyDateTime_IMPORT] main: yes, sub: yes'): # Fails if the setup() means test_datetime_capi() rather than # test_datetime_capi_newinterp() - self.assert_python_in_subinterp(True, with_setup, 'setup()') - self.assert_python_in_subinterp(True, 'setup()', fini=with_setup) - self.assert_python_in_subinterp(True, with_setup, repeat=2) + self.assert_python_ok_in_subinterp(with_setup, 'setup()') + self.assert_python_ok_in_subinterp('setup()', fini=with_setup) + self.assert_python_ok_in_subinterp(with_setup, repeat=2) with_import = 'import _datetime' + script with self.subTest('Explicit import'): - self.assert_python_in_subinterp(True, with_import, 'setup()') + self.assert_python_ok_in_subinterp(with_import, 'setup()') with_import = textwrap.dedent(""" timedelta = _testcapi.get_capi_types()['timedelta'] timedelta(days=1) """) + script with self.subTest('Implicit import'): - self.assert_python_in_subinterp(True, with_import, 'setup()') + self.assert_python_ok_in_subinterp(with_import, 'setup()') def test_static_type_at_shutdown(self): # gh-132413 @@ -7383,7 +7378,7 @@ def gen(): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) with self.subTest('Subinterpreter'): - res = self.assert_python_in_subinterp(True, script) + res = self.assert_python_ok_in_subinterp(script) self.assertFalse(res.err) script = textwrap.dedent(""" @@ -7404,12 +7399,12 @@ def gen(): next(it) """) with self.subTest('[PyDateTime_IMPORT] main: yes, sub: no'): - res = self.assert_python_in_subinterp(True, script, 'setup()') + res = self.assert_python_ok_in_subinterp(script, 'setup()') self.assertIn(b'ImportError: sys.meta_path is None', res.err) with_import = 'setup()' + script with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): - res = self.assert_python_in_subinterp(True, with_import) + res = self.assert_python_ok_in_subinterp(with_import) self.assertFalse(res.err) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index e859fa78d11de8..d272e106d66b10 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -458,7 +458,7 @@ def gen(): try: timedelta(days=1) res = 1 - except ImportError as e: + except ImportError: res = 2 print(res) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index ef19b1bca664b6..08759fd03b39ba 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -12,7 +12,6 @@ #include "Python.h" #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyObject_Init() -#include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_time.h" // _PyTime_ObjectToTime_t() #include "pycore_unicodeobject.h" // _PyUnicode_Copy() @@ -156,7 +155,6 @@ _get_current_state(PyObject **p_mod) * so we must re-import the module. */ PyObject *mod = PyImport_ImportModule("_datetime"); if (mod == NULL) { - assert(_Py_IsInterpreterFinalizing(interp)); return NULL; } st = get_module_state(mod); From f21f03f850844e53c7281e4db8188e1629662d1e Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Thu, 1 May 2025 21:46:14 +0900 Subject: [PATCH 50/55] Add a test to test_embed --- Lib/test/test_embed.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index d272e106d66b10..6cfa9ee43b3e4b 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -440,6 +440,21 @@ def test_datetime_reset_strptime(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '20000101\n' * INIT_LOOPS) + def test_datetime_capi_type_address(self): + # Check if the C-API types keep their addresses until runtime shutdown + code = textwrap.dedent(""" + import _datetime as d + print( + f'{id(d.date)}' + f'{id(d.time)}' + f'{id(d.datetime)}' + f'{id(d.timedelta)}' + f'{id(d.tzinfo)}' + ) + """) + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(len(set(out.splitlines())), 1) + def test_datetime_capi_at_shutdown(self): # gh-132413: datetime module is currently tested in an interp's life. # PyDateTime_IMPORT needs to be called at least once after the restart. From 2b6c12a145f40a91678fcfd0137d7d896f9390e9 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 11 May 2025 03:41:00 +0900 Subject: [PATCH 51/55] Decref the module explicitly on exec error --- Modules/_datetimemodule.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 08759fd03b39ba..b2d4ff07766316 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -7173,8 +7173,6 @@ create_timezone_from_delta(int days, int sec, int ms, int normalize) static int init_state(datetime_state *st, PyObject *module, PyObject *old_module) { - st->module = Py_NewRef(module); - /* Each module gets its own heap types. */ #define ADD_TYPE(FIELD, SPEC, BASE) \ do { \ @@ -7253,7 +7251,6 @@ init_state(datetime_state *st, PyObject *module, PyObject *old_module) static int traverse_state(datetime_state *st, visitproc visit, void *arg) { - Py_VISIT(st->module); Py_VISIT(st->isocalendar_date_type); return 0; @@ -7262,7 +7259,6 @@ traverse_state(datetime_state *st, visitproc visit, void *arg) static int clear_state(datetime_state *st) { - Py_CLEAR(st->module); /* Invalidate first */ Py_CLEAR(st->isocalendar_date_type); Py_CLEAR(st->us_per_ms); Py_CLEAR(st->us_per_second); @@ -7339,6 +7335,7 @@ _datetime_exec(PyObject *module) } } + st->module = Py_NewRef(module); if (init_state(st, module, old_module) < 0) { goto error; } @@ -7453,6 +7450,7 @@ _datetime_exec(PyObject *module) goto finally; error: + Py_CLEAR(st->module); clear_state(st); finally: @@ -7471,6 +7469,7 @@ static int module_traverse(PyObject *mod, visitproc visit, void *arg) { datetime_state *st = get_module_state(mod); + Py_VISIT(st->module); traverse_state(st, visit, arg); return 0; } @@ -7479,6 +7478,7 @@ static int module_clear(PyObject *mod) { datetime_state *st = get_module_state(mod); + Py_CLEAR(st->module); clear_state(st); PyInterpreterState *interp = PyInterpreterState_Get(); From 8d94b2f3df600ce39ee5ee9aabd1fadc6ccac21f Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 11 May 2025 06:52:37 +0900 Subject: [PATCH 52/55] Focus on main interp's issue --- Lib/test/datetimetester.py | 132 +++++++---------------------------- Lib/test/test_embed.py | 8 +-- Modules/_datetimemodule.c | 1 + Modules/_testcapi/datetime.c | 28 +------- 4 files changed, 35 insertions(+), 134 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 88c153f8379025..2406a1f7ad13bb 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7155,8 +7155,7 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) - def assert_python_ok_in_subinterp(self, script, init='', fini='', - repeat=1, config='isolated'): + def test_type_check_in_subinterp(self): # iOS requires the use of the custom framework loader, # not the ExtensionFileLoader. if sys.platform == "ios": @@ -7164,10 +7163,10 @@ def assert_python_ok_in_subinterp(self, script, init='', fini='', else: extension_loader = "ExtensionFileLoader" - code = textwrap.dedent(f''' - subinterp_code = """ + script = textwrap.dedent(f""" if {_interpreters is None}: - import _testcapi + import _testcapi as module + module.test_datetime_capi() else: import importlib.machinery import importlib.util @@ -7175,51 +7174,29 @@ def assert_python_ok_in_subinterp(self, script, init='', fini='', origin = importlib.util.find_spec('_testcapi').origin loader = importlib.machinery.{extension_loader}(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) - _testcapi = importlib.util.module_from_spec(spec) - spec.loader.exec_module(_testcapi) - run_counter = $RUN_COUNTER$ - setup = _testcapi.test_datetime_capi_newinterp # call it if needed - $SCRIPT$ - """ - - import _testcapi - from test import support - setup = _testcapi.test_datetime_capi_newinterp - $INIT$ + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) - for i in range(1, {1 + repeat}): - subcode = subinterp_code.replace('$RUN_COUNTER$', str(i)) - if {_interpreters is None}: - ret = support.run_in_subinterp(subcode) - else: - import _interpreters - config = _interpreters.new_config('{config}').__dict__ - ret = support.run_in_subinterp_with_config(subcode, **config) - assert ret == 0 - $FINI$ - ''') - code = code.replace('$INIT$', init).replace('$FINI$', fini) - code = code.replace('$SCRIPT$', script) - return script_helper.assert_python_ok('-c', code) - - def test_type_check_in_subinterp(self): - script = textwrap.dedent(f""" def run(type_checker, obj): if not type_checker(obj, True): raise TypeError(f'{{type(obj)}} is not C API type') - setup() import _datetime - run(_testcapi.datetime_check_date, _datetime.date.today()) - run(_testcapi.datetime_check_datetime, _datetime.datetime.now()) - run(_testcapi.datetime_check_time, _datetime.time(12, 30)) - run(_testcapi.datetime_check_delta, _datetime.timedelta(1)) - run(_testcapi.datetime_check_tzinfo, _datetime.tzinfo()) - """) - self.assert_python_ok_in_subinterp(script) - if _interpreters is not None: - with self.subTest(name := 'legacy'): - self.assert_python_ok_in_subinterp(script, config=name) + run(module.datetime_check_date, _datetime.date.today()) + run(module.datetime_check_datetime, _datetime.datetime.now()) + run(module.datetime_check_time, _datetime.time(12, 30)) + run(module.datetime_check_delta, _datetime.timedelta(1)) + run(module.datetime_check_tzinfo, _datetime.tzinfo()) + """) + if _interpreters is None: + ret = support.run_in_subinterp(script) + self.assertEqual(ret, 0) + else: + for name in ('isolated', 'legacy'): + with self.subTest(name): + config = _interpreters.new_config(name).__dict__ + ret = support.run_in_subinterp_with_config(script, **config) + self.assertEqual(ret, 0) class ExtensionModuleTests(unittest.TestCase): @@ -7228,9 +7205,6 @@ def setUp(self): if self.__class__.__name__.endswith('Pure'): self.skipTest('Not relevant in pure Python') - def assert_python_ok_in_subinterp(self, *args, **kwargs): - return CapiTest.assert_python_ok_in_subinterp(self, *args, **kwargs) - @support.cpython_only def test_gh_120161(self): with self.subTest('simple'): @@ -7325,33 +7299,6 @@ def test_no_leak(self): res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) self.assertIn(b'[0 refs, 0 blocks]', res.err) - def test_static_type_on_subinterp(self): - script = textwrap.dedent(""" - date = _testcapi.get_capi_types()['date'] - date.today - """) - with_setup = 'setup()' + script - with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): - self.assert_python_ok_in_subinterp(with_setup) - - with self.subTest('[PyDateTime_IMPORT] main: yes, sub: yes'): - # Fails if the setup() means test_datetime_capi() rather than - # test_datetime_capi_newinterp() - self.assert_python_ok_in_subinterp(with_setup, 'setup()') - self.assert_python_ok_in_subinterp('setup()', fini=with_setup) - self.assert_python_ok_in_subinterp(with_setup, repeat=2) - - with_import = 'import _datetime' + script - with self.subTest('Explicit import'): - self.assert_python_ok_in_subinterp(with_import, 'setup()') - - with_import = textwrap.dedent(""" - timedelta = _testcapi.get_capi_types()['timedelta'] - timedelta(days=1) - """) + script - with self.subTest('Implicit import'): - self.assert_python_ok_in_subinterp(with_import, 'setup()') - def test_static_type_at_shutdown(self): # gh-132413 script = textwrap.dedent(""" @@ -7374,38 +7321,13 @@ def gen(): it = gen() next(it) """) - with self.subTest('MainInterpreter'): - res = script_helper.assert_python_ok('-c', script) - self.assertFalse(res.err) - with self.subTest('Subinterpreter'): - res = self.assert_python_ok_in_subinterp(script) - self.assertFalse(res.err) - - script = textwrap.dedent(""" - import sys - timedelta = _testcapi.get_capi_types()['timedelta'] - - def gen(): - try: - yield - finally: - # Exceptions are ignored here - assert not sys.modules - td = timedelta(days=1) - assert td.days == 1 - assert not sys.modules + res = script_helper.assert_python_ok('-c', script) + self.assertFalse(res.err) - it = gen() - next(it) - """) - with self.subTest('[PyDateTime_IMPORT] main: yes, sub: no'): - res = self.assert_python_ok_in_subinterp(script, 'setup()') - self.assertIn(b'ImportError: sys.meta_path is None', res.err) - - with_import = 'setup()' + script - with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): - res = self.assert_python_ok_in_subinterp(with_import) - self.assertFalse(res.err) + if support.Py_DEBUG: + with self.subTest('Refleak'): + res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) + self.assertIn(b'[0 refs, 0 blocks]', res.err) def load_tests(loader, standard_tests, pattern): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 28bd60ed9b5928..8908f0ede6616c 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -456,12 +456,12 @@ def test_datetime_capi_type_address(self): self.assertEqual(len(set(out.splitlines())), 1) def test_datetime_capi_at_shutdown(self): - # gh-132413: datetime module is currently tested in an interp's life. - # PyDateTime_IMPORT needs to be called at least once after the restart. + # gh-132413: Users need to call PyDateTime_IMPORT every time + # after starting an interpreter. code = textwrap.dedent(""" import sys import _testcapi - _testcapi.test_datetime_capi_newinterp() + _testcapi.test_datetime_capi() # PyDateTime_IMPORT only once timedelta = _testcapi.get_capi_types()['timedelta'] def gen(): @@ -481,7 +481,7 @@ def gen(): next(it) """) out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) - self.assertEqual(out, '1\n' * INIT_LOOPS) + self.assertEqual(out, '1\n' + '2\n' * (INIT_LOOPS - 1)) def test_static_types_inherited_slots(self): script = textwrap.dedent(""" diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index b2d4ff07766316..a603952af07c45 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -7251,6 +7251,7 @@ init_state(datetime_state *st, PyObject *module, PyObject *old_module) static int traverse_state(datetime_state *st, visitproc visit, void *arg) { + /* heap types */ Py_VISIT(st->isocalendar_date_type); return 0; diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index 375196e28fb727..36fd9e50ca154c 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -35,29 +35,6 @@ test_datetime_capi(PyObject *self, PyObject *args) Py_RETURN_NONE; } -static PyObject * -test_datetime_capi_newinterp(PyObject *self, PyObject *args) -{ - // Call PyDateTime_IMPORT at least once in each interpreter's life - if (PyDateTimeAPI != NULL && test_run_counter == 0) { - PyErr_SetString(PyExc_AssertionError, - "PyDateTime_CAPI somehow initialized"); - return NULL; - } - test_run_counter++; - PyDateTime_IMPORT; - - if (PyDateTimeAPI == NULL) { - return NULL; - } - assert(!PyType_HasFeature(PyDateTimeAPI->DateType, Py_TPFLAGS_HEAPTYPE)); - assert(!PyType_HasFeature(PyDateTimeAPI->TimeType, Py_TPFLAGS_HEAPTYPE)); - assert(!PyType_HasFeature(PyDateTimeAPI->DateTimeType, Py_TPFLAGS_HEAPTYPE)); - assert(!PyType_HasFeature(PyDateTimeAPI->DeltaType, Py_TPFLAGS_HEAPTYPE)); - assert(!PyType_HasFeature(PyDateTimeAPI->TZInfoType, Py_TPFLAGS_HEAPTYPE)); - Py_RETURN_NONE; -} - /* Functions exposing the C API type checking for testing */ #define MAKE_DATETIME_CHECK_FUNC(check_method, exact_method) \ do { \ @@ -530,7 +507,6 @@ static PyMethodDef test_methods[] = { {"get_capi_types", get_capi_types, METH_NOARGS}, {"make_timezones_capi", make_timezones_capi, METH_NOARGS}, {"test_datetime_capi", test_datetime_capi, METH_NOARGS}, - {"test_datetime_capi_newinterp",test_datetime_capi_newinterp, METH_NOARGS}, {NULL}, }; @@ -551,7 +527,9 @@ _PyTestCapi_Init_DateTime(PyObject *mod) static int _testcapi_datetime_exec(PyObject *mod) { - // The execution does not invoke PyDateTime_IMPORT + if (test_datetime_capi(NULL, NULL) == NULL) { + return -1; + } return 0; } From e1c9cdfbffd1bbb93a4dcd1559b5b63815ebcc9a Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 11 May 2025 09:26:37 +0900 Subject: [PATCH 53/55] Tweak 2b6c12a --- Modules/_datetimemodule.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index a603952af07c45..9b01f44703d979 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -7191,7 +7191,6 @@ init_state(datetime_state *st, PyObject *module, PyObject *old_module) assert(old_module != module); datetime_state *st_old = get_module_state(old_module); *st = (datetime_state){ - .module = st->module, .isocalendar_date_type = st->isocalendar_date_type, .us_per_ms = Py_NewRef(st_old->us_per_ms), .us_per_second = Py_NewRef(st_old->us_per_second), @@ -7336,7 +7335,6 @@ _datetime_exec(PyObject *module) } } - st->module = Py_NewRef(module); if (init_state(st, module, old_module) < 0) { goto error; } @@ -7446,12 +7444,12 @@ _datetime_exec(PyObject *module) if (set_current_module(interp, module) < 0) { goto error; } + st->module = Py_NewRef(module); rc = 0; goto finally; error: - Py_CLEAR(st->module); clear_state(st); finally: From 5ed5e69b8e82f339aebc997d77a2bc3fb0443633 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 13 May 2025 19:41:33 +0900 Subject: [PATCH 54/55] Smaller patch for tests --- Lib/test/datetimetester.py | 3 +-- Lib/test/test_embed.py | 2 +- Modules/_testcapi/datetime.c | 32 -------------------------------- 3 files changed, 2 insertions(+), 35 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 2406a1f7ad13bb..b2025131924bab 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7270,8 +7270,7 @@ def test_update_type_cache(self): assert isinstance(_datetime.timezone.utc, _datetime.tzinfo) del sys.modules['_datetime'] """) - res = script_helper.assert_python_ok('-c', script) - self.assertFalse(res.err) + script_helper.assert_python_ok('-c', script) def test_module_free(self): script = textwrap.dedent(""" diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 8908f0ede6616c..cf0a7f2d88d5ee 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -462,7 +462,7 @@ def test_datetime_capi_at_shutdown(self): import sys import _testcapi _testcapi.test_datetime_capi() # PyDateTime_IMPORT only once - timedelta = _testcapi.get_capi_types()['timedelta'] + timedelta = type(_testcapi.get_delta_fromdsu(False, 1, 0, 0)) def gen(): try: diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index 36fd9e50ca154c..b800f9b8eb3473 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -453,37 +453,6 @@ test_PyDateTime_DELTA_GET(PyObject *self, PyObject *obj) return Py_BuildValue("(iii)", days, seconds, microseconds); } -static PyObject * -get_capi_types(PyObject *self, PyObject *args) -{ - if (PyDateTimeAPI == NULL) { - Py_RETURN_NONE; - } - PyObject *dict = PyDict_New(); - if (dict == NULL) { - return NULL; - } - if (PyDict_SetItemString(dict, "date", (PyObject *)PyDateTimeAPI->DateType) < 0) { - goto error; - } - if (PyDict_SetItemString(dict, "time", (PyObject *)PyDateTimeAPI->TimeType) < 0) { - goto error; - } - if (PyDict_SetItemString(dict, "datetime", (PyObject *)PyDateTimeAPI->DateTimeType) < 0) { - goto error; - } - if (PyDict_SetItemString(dict, "timedelta", (PyObject *)PyDateTimeAPI->DeltaType) < 0) { - goto error; - } - if (PyDict_SetItemString(dict, "tzinfo", (PyObject *)PyDateTimeAPI->TZInfoType) < 0) { - goto error; - } - return dict; -error: - Py_DECREF(dict); - return NULL; -} - static PyMethodDef test_methods[] = { {"PyDateTime_DATE_GET", test_PyDateTime_DATE_GET, METH_O}, {"PyDateTime_DELTA_GET", test_PyDateTime_DELTA_GET, METH_O}, @@ -504,7 +473,6 @@ static PyMethodDef test_methods[] = { {"get_time_fromtimeandfold", get_time_fromtimeandfold, METH_VARARGS}, {"get_timezone_utc_capi", get_timezone_utc_capi, METH_VARARGS}, {"get_timezones_offset_zero", get_timezones_offset_zero, METH_NOARGS}, - {"get_capi_types", get_capi_types, METH_NOARGS}, {"make_timezones_capi", make_timezones_capi, METH_NOARGS}, {"test_datetime_capi", test_datetime_capi, METH_NOARGS}, {NULL}, From f22edc1e32721d5dcd6c6b297991fcb567b8acce Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 24 Jun 2025 05:01:30 +0900 Subject: [PATCH 55/55] minimize test --- Lib/test/datetimetester.py | 45 ++++---------------------------------- Lib/test/test_embed.py | 43 ------------------------------------ 2 files changed, 4 insertions(+), 84 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 3b96cd856d853e..49e78f6f9a74cd 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7275,36 +7275,9 @@ def test_update_type_cache(self): """) script_helper.assert_python_ok('-c', script) - def test_module_free(self): - script = textwrap.dedent(""" - import sys - import gc - import weakref - ws = weakref.WeakSet() - for _ in range(3): - import _datetime - timedelta = _datetime.timedelta # static type - ws.add(_datetime) - del sys.modules['_datetime'] - del _datetime - gc.collect() - assert len(ws) == 0 - """) - script_helper.assert_python_ok('-c', script) - - @unittest.skipIf(not support.Py_DEBUG, "Debug builds only") - def test_no_leak(self): - script = textwrap.dedent(""" - import datetime - datetime.datetime.strptime('20000101', '%Y%m%d').strftime('%Y%m%d') - """) - res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) - self.assertIn(b'[0 refs, 0 blocks]', res.err) - def test_static_type_at_shutdown(self): # gh-132413 script = textwrap.dedent(""" - import sys import _datetime timedelta = _datetime.timedelta @@ -7312,24 +7285,14 @@ def gen(): try: yield finally: - # Exceptions are ignored here - assert not sys.modules - td = _datetime.timedelta(days=1) - assert td.days == 1 - td = timedelta(days=1) - assert td.days == 1 - assert not sys.modules + # sys.modules is empty + _datetime.timedelta(days=1) + timedelta(days=1) it = gen() next(it) """) - res = script_helper.assert_python_ok('-c', script) - self.assertFalse(res.err) - - if support.Py_DEBUG: - with self.subTest('Refleak'): - res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) - self.assertIn(b'[0 refs, 0 blocks]', res.err) + script_helper.assert_python_ok('-c', script) def load_tests(loader, standard_tests, pattern): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 8717541de57dce..46222e521aead8 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -440,49 +440,6 @@ def test_datetime_reset_strptime(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '20000101\n' * INIT_LOOPS) - def test_datetime_capi_type_address(self): - # Check if the C-API types keep their addresses until runtime shutdown - code = textwrap.dedent(""" - import _datetime as d - print( - f'{id(d.date)}' - f'{id(d.time)}' - f'{id(d.datetime)}' - f'{id(d.timedelta)}' - f'{id(d.tzinfo)}' - ) - """) - out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) - self.assertEqual(len(set(out.splitlines())), 1) - - def test_datetime_capi_at_shutdown(self): - # gh-132413: Users need to call PyDateTime_IMPORT every time - # after starting an interpreter. - code = textwrap.dedent(""" - import sys - import _testcapi - _testcapi.test_datetime_capi() # PyDateTime_IMPORT only once - timedelta = type(_testcapi.get_delta_fromdsu(False, 1, 0, 0)) - - def gen(): - try: - yield - finally: - assert not sys.modules - res = 0 - try: - timedelta(days=1) - res = 1 - except ImportError: - res = 2 - print(res) - - it = gen() - next(it) - """) - out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) - self.assertEqual(out, '1\n' + '2\n' * (INIT_LOOPS - 1)) - def test_static_types_inherited_slots(self): script = textwrap.dedent(""" import test.support