From 53165608262b2946d4bdfd7c8c7e7fc6da5cccb6 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 14 Feb 2024 15:41:57 -0700 Subject: [PATCH 1/8] Fix the channels module. --- Lib/test/support/interpreters/channels.py | 1 - Modules/_xxinterpchannelsmodule.c | 179 ++++++++++------------ 2 files changed, 83 insertions(+), 97 deletions(-) diff --git a/Lib/test/support/interpreters/channels.py b/Lib/test/support/interpreters/channels.py index 75a5a60f54f926..7bc9436eb5af54 100644 --- a/Lib/test/support/interpreters/channels.py +++ b/Lib/test/support/interpreters/channels.py @@ -167,5 +167,4 @@ def close(self): _channels.close(self._id, send=True) -# XXX This is causing leaks (gh-110318): _channels._register_end_types(SendChannel, RecvChannel) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index a2974aced12ca0..a5035458fa6433 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -93,35 +93,35 @@ API.. The module does not create any objects that are shared globally. PyMem_RawFree(VAR) -struct xid_class_registry { - size_t count; -#define MAX_XID_CLASSES 5 - struct { - PyTypeObject *cls; - } added[MAX_XID_CLASSES]; +struct xid_types { + /* Added at runtime by interpreters module. */ + PyTypeObject *SendChannel; + PyTypeObject *RecvChannel; + + /* heap types */ + PyTypeObject *ChannelID; }; -static int -register_xid_class(PyTypeObject *cls, crossinterpdatafunc shared, - struct xid_class_registry *classes) +static void +clear_xid_types(struct xid_types *types) { - int res = ensure_xid_class(cls, shared); - if (res == 0) { - assert(classes->count < MAX_XID_CLASSES); - // The class has refs elsewhere, so we need to incref here. - classes->added[classes->count].cls = cls; - classes->count += 1; + /* external types */ + if (types->SendChannel != NULL) { + (void)_PyCrossInterpreterData_UnregisterClass( + types->SendChannel); + Py_CLEAR(types->SendChannel); + } + if (types->RecvChannel != NULL) { + (void)_PyCrossInterpreterData_UnregisterClass( + types->RecvChannel); + Py_CLEAR(types->RecvChannel); } - return res; -} -static void -clear_xid_class_registry(struct xid_class_registry *classes) -{ - while (classes->count > 0) { - classes->count -= 1; - PyTypeObject *cls = classes->added[classes->count].cls; - _PyCrossInterpreterData_UnregisterClass(cls); + /* heap types */ + if (types->ChannelID != NULL) { + (void)_PyCrossInterpreterData_UnregisterClass( + types->ChannelID); + Py_CLEAR(types->ChannelID); } } @@ -223,28 +223,6 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base) #define ADD_NEW_EXCEPTION(MOD, NAME, BASE) \ add_new_exception(MOD, MODULE_NAME_STR "." Py_STRINGIFY(NAME), BASE) -static PyTypeObject * -add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared, - struct xid_class_registry *classes) -{ - PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec( - mod, spec, NULL); - if (cls == NULL) { - return NULL; - } - if (PyModule_AddType(mod, cls) < 0) { - Py_DECREF(cls); - return NULL; - } - if (shared != NULL) { - if (register_xid_class(cls, shared, classes)) { - Py_DECREF(cls); - return NULL; - } - } - return cls; -} - static int wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) { @@ -269,15 +247,10 @@ wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) /* module state *************************************************************/ typedef struct { - struct xid_class_registry xid_classes; - - /* Added at runtime by interpreters module. */ - PyTypeObject *send_channel_type; - PyTypeObject *recv_channel_type; + struct xid_types xid_types; /* heap types */ PyTypeObject *ChannelInfoType; - PyTypeObject *ChannelIDType; /* exceptions */ PyObject *ChannelError; @@ -315,12 +288,12 @@ static int traverse_module_state(module_state *state, visitproc visit, void *arg) { /* external types */ - Py_VISIT(state->send_channel_type); - Py_VISIT(state->recv_channel_type); + Py_VISIT(state->xid_types.SendChannel); + Py_VISIT(state->xid_types.RecvChannel); /* heap types */ Py_VISIT(state->ChannelInfoType); - Py_VISIT(state->ChannelIDType); + Py_VISIT(state->xid_types.ChannelID); /* exceptions */ Py_VISIT(state->ChannelError); @@ -335,16 +308,10 @@ traverse_module_state(module_state *state, visitproc visit, void *arg) static int clear_module_state(module_state *state) { - /* external types */ - Py_CLEAR(state->send_channel_type); - Py_CLEAR(state->recv_channel_type); + clear_xid_types(&state->xid_types); /* heap types */ Py_CLEAR(state->ChannelInfoType); - if (state->ChannelIDType != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType); - } - Py_CLEAR(state->ChannelIDType); /* exceptions */ Py_CLEAR(state->ChannelError); @@ -2210,7 +2177,7 @@ channel_id_converter(PyObject *arg, void *ptr) struct channel_id_converter_data *data = ptr; module_state *state = get_module_state(data->module); assert(state != NULL); - if (PyObject_TypeCheck(arg, state->ChannelIDType)) { + if (PyObject_TypeCheck(arg, state->xid_types.ChannelID)) { cid = ((channelid *)arg)->cid; end = ((channelid *)arg)->end; } @@ -2406,14 +2373,14 @@ channelid_richcompare(PyObject *self, PyObject *other, int op) goto done; } - if (!PyObject_TypeCheck(self, state->ChannelIDType)) { + if (!PyObject_TypeCheck(self, state->xid_types.ChannelID)) { res = Py_NewRef(Py_NotImplemented); goto done; } channelid *cidobj = (channelid *)self; int equal; - if (PyObject_TypeCheck(other, state->ChannelIDType)) { + if (PyObject_TypeCheck(other, state->xid_types.ChannelID)) { channelid *othercidobj = (channelid *)other; equal = (cidobj->end == othercidobj->end) && (cidobj->cid == othercidobj->cid); } @@ -2494,7 +2461,7 @@ _channelid_from_xid(_PyCrossInterpreterData *data) // Note that we do not preserve the "resolve" flag. PyObject *cidobj = NULL; - int err = newchannelid(state->ChannelIDType, xid->cid, xid->end, + int err = newchannelid(state->xid_types.ChannelID, xid->cid, xid->end, _global_channels(), 0, 0, (channelid **)&cidobj); if (err != 0) { @@ -2614,6 +2581,26 @@ static PyType_Spec channelid_typespec = { .slots = channelid_typeslots, }; +static PyTypeObject * +add_channelid_type(PyObject *mod) +{ + PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec( + mod, &channelid_typespec, NULL); + if (cls == NULL) { + return NULL; + } + if (ensure_xid_class(cls, _channelid_shared)) { + Py_DECREF(cls); + return NULL; + } + if (PyModule_AddType(mod, cls) < 0) { + (void)_PyCrossInterpreterData_UnregisterClass(cls); + Py_DECREF(cls); + return NULL; + } + return cls; +} + /* SendChannel and RecvChannel classes */ @@ -2628,11 +2615,11 @@ _get_current_channelend_type(int end) } PyTypeObject *cls; if (end == CHANNEL_SEND) { - cls = state->send_channel_type; + cls = state->xid_types.SendChannel; } else { assert(end == CHANNEL_RECV); - cls = state->recv_channel_type; + cls = state->xid_types.RecvChannel; } if (cls == NULL) { // Force the module to be loaded, to register the type. @@ -2646,10 +2633,10 @@ _get_current_channelend_type(int end) } Py_DECREF(highlevel); if (end == CHANNEL_SEND) { - cls = state->send_channel_type; + cls = state->xid_types.SendChannel; } else { - cls = state->recv_channel_type; + cls = state->xid_types.RecvChannel; } assert(cls != NULL); } @@ -2697,23 +2684,30 @@ set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) if (state == NULL) { return -1; } - struct xid_class_registry *xid_classes = &state->xid_classes; + struct xid_types *types = &state->xid_types; - if (state->send_channel_type != NULL - || state->recv_channel_type != NULL) - { - PyErr_SetString(PyExc_TypeError, "already registered"); - return -1; + // Clear the old values if the .py module was reloaded. + if (types->SendChannel != NULL) { + (void)_PyCrossInterpreterData_UnregisterClass( + types->SendChannel); + Py_CLEAR(types->SendChannel); + } + if (types->RecvChannel != NULL) { + (void)_PyCrossInterpreterData_UnregisterClass( + types->RecvChannel); + Py_CLEAR(types->RecvChannel); } - state->send_channel_type = (PyTypeObject *)Py_NewRef(send); - state->recv_channel_type = (PyTypeObject *)Py_NewRef(recv); - if (register_xid_class(send, _channelend_shared, xid_classes)) { + // Add and register the new types. + if (ensure_xid_class(send, _channelend_shared) < 0) { return -1; } - if (register_xid_class(recv, _channelend_shared, xid_classes)) { + if (ensure_xid_class(recv, _channelend_shared) < 0) { + (void)_PyCrossInterpreterData_UnregisterClass(send); return -1; } + types->SendChannel = (PyTypeObject *)Py_NewRef(send); + types->RecvChannel = (PyTypeObject *)Py_NewRef(recv); return 0; } @@ -2792,7 +2786,7 @@ channelsmod_create(PyObject *self, PyObject *Py_UNUSED(ignored)) return NULL; } PyObject *cidobj = NULL; - int err = newchannelid(state->ChannelIDType, cid, 0, + int err = newchannelid(state->xid_types.ChannelID, cid, 0, &_globals.channels, 0, 0, (channelid **)&cidobj); if (handle_channel_error(err, self, cid)) { @@ -2864,7 +2858,7 @@ channelsmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) int64_t *cur = cids; for (int64_t i=0; i < count; cur++, i++) { PyObject *cidobj = NULL; - int err = newchannelid(state->ChannelIDType, *cur, 0, + int err = newchannelid(state->xid_types.ChannelID, *cur, 0, &_globals.channels, 0, 0, (channelid **)&cidobj); if (handle_channel_error(err, self, *cur)) { @@ -3214,7 +3208,7 @@ channelsmod__channel_id(PyObject *self, PyObject *args, PyObject *kwds) if (state == NULL) { return NULL; } - PyTypeObject *cls = state->ChannelIDType; + PyTypeObject *cls = state->xid_types.ChannelID; PyObject *mod = get_module_from_owned_type(cls); assert(mod == self); @@ -3294,13 +3288,13 @@ module_exec(PyObject *mod) if (_globals_init() != 0) { return -1; } - struct xid_class_registry *xid_classes = NULL; + struct xid_types *xid_types = NULL; module_state *state = get_module_state(mod); if (state == NULL) { goto error; } - xid_classes = &state->xid_classes; + xid_types = &state->xid_types; /* Add exception types */ if (exceptions_init(mod) != 0) { @@ -3319,9 +3313,8 @@ module_exec(PyObject *mod) } // ChannelID - state->ChannelIDType = add_new_type( - mod, &channelid_typespec, _channelid_shared, xid_classes); - if (state->ChannelIDType == NULL) { + xid_types->ChannelID = add_channelid_type(mod); + if (xid_types->ChannelID == NULL) { goto error; } @@ -3332,8 +3325,8 @@ module_exec(PyObject *mod) return 0; error: - if (xid_classes != NULL) { - clear_xid_class_registry(xid_classes); + if (xid_types != NULL) { + clear_xid_types(xid_types); } _globals_fini(); return -1; @@ -3360,9 +3353,6 @@ module_clear(PyObject *mod) module_state *state = get_module_state(mod); assert(state != NULL); - // Before clearing anything, we unregister the various XID types. */ - clear_xid_class_registry(&state->xid_classes); - // Now we clear the module state. clear_module_state(state); return 0; @@ -3374,9 +3364,6 @@ module_free(void *mod) module_state *state = get_module_state(mod); assert(state != NULL); - // Before clearing anything, we unregister the various XID types. */ - clear_xid_class_registry(&state->xid_classes); - // Now we clear the module state. clear_module_state(state); From 55866852f19cd01a915fbf9ab571826ada515ec3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 14 Feb 2024 15:44:33 -0700 Subject: [PATCH 2/8] Fix the queues module. --- Modules/_xxinterpqueuesmodule.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index 7d8c67f49fefb8..df0aca717e5f0b 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -169,6 +169,9 @@ static int clear_module_state(module_state *state) { /* external types */ + if (state->queue_type != NULL) { + (void)_PyCrossInterpreterData_UnregisterClass(state->queue_type); + } Py_CLEAR(state->queue_type); /* QueueError */ @@ -1060,15 +1063,18 @@ set_external_queue_type(PyObject *module, PyTypeObject *queue_type) { module_state *state = get_module_state(module); + // Clear the old value if the .py module was reloaded. if (state->queue_type != NULL) { - PyErr_SetString(PyExc_TypeError, "already registered"); - return -1; + (void)_PyCrossInterpreterData_UnregisterClass( + state->queue_type); + Py_CLEAR(state->queue_type); } - state->queue_type = (PyTypeObject *)Py_NewRef(queue_type); + // Add and register the new type. if (ensure_xid_class(queue_type, _queueobj_shared) < 0) { return -1; } + state->queue_type = (PyTypeObject *)Py_NewRef(queue_type); return 0; } @@ -1652,10 +1658,6 @@ module_clear(PyObject *mod) { module_state *state = get_module_state(mod); - if (state->queue_type != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->queue_type); - } - // Now we clear the module state. clear_module_state(state); return 0; From 18fa5d4868e028ab28b6952c32dfc8f744ca09d0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 15 Feb 2024 11:12:44 -0700 Subject: [PATCH 3/8] Revert "Fix the channels module." This reverts commit 53165608262b2946d4bdfd7c8c7e7fc6da5cccb6. --- Lib/test/support/interpreters/channels.py | 1 + Modules/_xxinterpchannelsmodule.c | 179 ++++++++++++---------- 2 files changed, 97 insertions(+), 83 deletions(-) diff --git a/Lib/test/support/interpreters/channels.py b/Lib/test/support/interpreters/channels.py index 7bc9436eb5af54..75a5a60f54f926 100644 --- a/Lib/test/support/interpreters/channels.py +++ b/Lib/test/support/interpreters/channels.py @@ -167,4 +167,5 @@ def close(self): _channels.close(self._id, send=True) +# XXX This is causing leaks (gh-110318): _channels._register_end_types(SendChannel, RecvChannel) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index a5035458fa6433..a2974aced12ca0 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -93,35 +93,35 @@ API.. The module does not create any objects that are shared globally. PyMem_RawFree(VAR) -struct xid_types { - /* Added at runtime by interpreters module. */ - PyTypeObject *SendChannel; - PyTypeObject *RecvChannel; - - /* heap types */ - PyTypeObject *ChannelID; +struct xid_class_registry { + size_t count; +#define MAX_XID_CLASSES 5 + struct { + PyTypeObject *cls; + } added[MAX_XID_CLASSES]; }; -static void -clear_xid_types(struct xid_types *types) +static int +register_xid_class(PyTypeObject *cls, crossinterpdatafunc shared, + struct xid_class_registry *classes) { - /* external types */ - if (types->SendChannel != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass( - types->SendChannel); - Py_CLEAR(types->SendChannel); - } - if (types->RecvChannel != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass( - types->RecvChannel); - Py_CLEAR(types->RecvChannel); + int res = ensure_xid_class(cls, shared); + if (res == 0) { + assert(classes->count < MAX_XID_CLASSES); + // The class has refs elsewhere, so we need to incref here. + classes->added[classes->count].cls = cls; + classes->count += 1; } + return res; +} - /* heap types */ - if (types->ChannelID != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass( - types->ChannelID); - Py_CLEAR(types->ChannelID); +static void +clear_xid_class_registry(struct xid_class_registry *classes) +{ + while (classes->count > 0) { + classes->count -= 1; + PyTypeObject *cls = classes->added[classes->count].cls; + _PyCrossInterpreterData_UnregisterClass(cls); } } @@ -223,6 +223,28 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base) #define ADD_NEW_EXCEPTION(MOD, NAME, BASE) \ add_new_exception(MOD, MODULE_NAME_STR "." Py_STRINGIFY(NAME), BASE) +static PyTypeObject * +add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared, + struct xid_class_registry *classes) +{ + PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec( + mod, spec, NULL); + if (cls == NULL) { + return NULL; + } + if (PyModule_AddType(mod, cls) < 0) { + Py_DECREF(cls); + return NULL; + } + if (shared != NULL) { + if (register_xid_class(cls, shared, classes)) { + Py_DECREF(cls); + return NULL; + } + } + return cls; +} + static int wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) { @@ -247,10 +269,15 @@ wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) /* module state *************************************************************/ typedef struct { - struct xid_types xid_types; + struct xid_class_registry xid_classes; + + /* Added at runtime by interpreters module. */ + PyTypeObject *send_channel_type; + PyTypeObject *recv_channel_type; /* heap types */ PyTypeObject *ChannelInfoType; + PyTypeObject *ChannelIDType; /* exceptions */ PyObject *ChannelError; @@ -288,12 +315,12 @@ static int traverse_module_state(module_state *state, visitproc visit, void *arg) { /* external types */ - Py_VISIT(state->xid_types.SendChannel); - Py_VISIT(state->xid_types.RecvChannel); + Py_VISIT(state->send_channel_type); + Py_VISIT(state->recv_channel_type); /* heap types */ Py_VISIT(state->ChannelInfoType); - Py_VISIT(state->xid_types.ChannelID); + Py_VISIT(state->ChannelIDType); /* exceptions */ Py_VISIT(state->ChannelError); @@ -308,10 +335,16 @@ traverse_module_state(module_state *state, visitproc visit, void *arg) static int clear_module_state(module_state *state) { - clear_xid_types(&state->xid_types); + /* external types */ + Py_CLEAR(state->send_channel_type); + Py_CLEAR(state->recv_channel_type); /* heap types */ Py_CLEAR(state->ChannelInfoType); + if (state->ChannelIDType != NULL) { + (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType); + } + Py_CLEAR(state->ChannelIDType); /* exceptions */ Py_CLEAR(state->ChannelError); @@ -2177,7 +2210,7 @@ channel_id_converter(PyObject *arg, void *ptr) struct channel_id_converter_data *data = ptr; module_state *state = get_module_state(data->module); assert(state != NULL); - if (PyObject_TypeCheck(arg, state->xid_types.ChannelID)) { + if (PyObject_TypeCheck(arg, state->ChannelIDType)) { cid = ((channelid *)arg)->cid; end = ((channelid *)arg)->end; } @@ -2373,14 +2406,14 @@ channelid_richcompare(PyObject *self, PyObject *other, int op) goto done; } - if (!PyObject_TypeCheck(self, state->xid_types.ChannelID)) { + if (!PyObject_TypeCheck(self, state->ChannelIDType)) { res = Py_NewRef(Py_NotImplemented); goto done; } channelid *cidobj = (channelid *)self; int equal; - if (PyObject_TypeCheck(other, state->xid_types.ChannelID)) { + if (PyObject_TypeCheck(other, state->ChannelIDType)) { channelid *othercidobj = (channelid *)other; equal = (cidobj->end == othercidobj->end) && (cidobj->cid == othercidobj->cid); } @@ -2461,7 +2494,7 @@ _channelid_from_xid(_PyCrossInterpreterData *data) // Note that we do not preserve the "resolve" flag. PyObject *cidobj = NULL; - int err = newchannelid(state->xid_types.ChannelID, xid->cid, xid->end, + int err = newchannelid(state->ChannelIDType, xid->cid, xid->end, _global_channels(), 0, 0, (channelid **)&cidobj); if (err != 0) { @@ -2581,26 +2614,6 @@ static PyType_Spec channelid_typespec = { .slots = channelid_typeslots, }; -static PyTypeObject * -add_channelid_type(PyObject *mod) -{ - PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec( - mod, &channelid_typespec, NULL); - if (cls == NULL) { - return NULL; - } - if (ensure_xid_class(cls, _channelid_shared)) { - Py_DECREF(cls); - return NULL; - } - if (PyModule_AddType(mod, cls) < 0) { - (void)_PyCrossInterpreterData_UnregisterClass(cls); - Py_DECREF(cls); - return NULL; - } - return cls; -} - /* SendChannel and RecvChannel classes */ @@ -2615,11 +2628,11 @@ _get_current_channelend_type(int end) } PyTypeObject *cls; if (end == CHANNEL_SEND) { - cls = state->xid_types.SendChannel; + cls = state->send_channel_type; } else { assert(end == CHANNEL_RECV); - cls = state->xid_types.RecvChannel; + cls = state->recv_channel_type; } if (cls == NULL) { // Force the module to be loaded, to register the type. @@ -2633,10 +2646,10 @@ _get_current_channelend_type(int end) } Py_DECREF(highlevel); if (end == CHANNEL_SEND) { - cls = state->xid_types.SendChannel; + cls = state->send_channel_type; } else { - cls = state->xid_types.RecvChannel; + cls = state->recv_channel_type; } assert(cls != NULL); } @@ -2684,30 +2697,23 @@ set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) if (state == NULL) { return -1; } - struct xid_types *types = &state->xid_types; + struct xid_class_registry *xid_classes = &state->xid_classes; - // Clear the old values if the .py module was reloaded. - if (types->SendChannel != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass( - types->SendChannel); - Py_CLEAR(types->SendChannel); - } - if (types->RecvChannel != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass( - types->RecvChannel); - Py_CLEAR(types->RecvChannel); + if (state->send_channel_type != NULL + || state->recv_channel_type != NULL) + { + PyErr_SetString(PyExc_TypeError, "already registered"); + return -1; } + state->send_channel_type = (PyTypeObject *)Py_NewRef(send); + state->recv_channel_type = (PyTypeObject *)Py_NewRef(recv); - // Add and register the new types. - if (ensure_xid_class(send, _channelend_shared) < 0) { + if (register_xid_class(send, _channelend_shared, xid_classes)) { return -1; } - if (ensure_xid_class(recv, _channelend_shared) < 0) { - (void)_PyCrossInterpreterData_UnregisterClass(send); + if (register_xid_class(recv, _channelend_shared, xid_classes)) { return -1; } - types->SendChannel = (PyTypeObject *)Py_NewRef(send); - types->RecvChannel = (PyTypeObject *)Py_NewRef(recv); return 0; } @@ -2786,7 +2792,7 @@ channelsmod_create(PyObject *self, PyObject *Py_UNUSED(ignored)) return NULL; } PyObject *cidobj = NULL; - int err = newchannelid(state->xid_types.ChannelID, cid, 0, + int err = newchannelid(state->ChannelIDType, cid, 0, &_globals.channels, 0, 0, (channelid **)&cidobj); if (handle_channel_error(err, self, cid)) { @@ -2858,7 +2864,7 @@ channelsmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) int64_t *cur = cids; for (int64_t i=0; i < count; cur++, i++) { PyObject *cidobj = NULL; - int err = newchannelid(state->xid_types.ChannelID, *cur, 0, + int err = newchannelid(state->ChannelIDType, *cur, 0, &_globals.channels, 0, 0, (channelid **)&cidobj); if (handle_channel_error(err, self, *cur)) { @@ -3208,7 +3214,7 @@ channelsmod__channel_id(PyObject *self, PyObject *args, PyObject *kwds) if (state == NULL) { return NULL; } - PyTypeObject *cls = state->xid_types.ChannelID; + PyTypeObject *cls = state->ChannelIDType; PyObject *mod = get_module_from_owned_type(cls); assert(mod == self); @@ -3288,13 +3294,13 @@ module_exec(PyObject *mod) if (_globals_init() != 0) { return -1; } - struct xid_types *xid_types = NULL; + struct xid_class_registry *xid_classes = NULL; module_state *state = get_module_state(mod); if (state == NULL) { goto error; } - xid_types = &state->xid_types; + xid_classes = &state->xid_classes; /* Add exception types */ if (exceptions_init(mod) != 0) { @@ -3313,8 +3319,9 @@ module_exec(PyObject *mod) } // ChannelID - xid_types->ChannelID = add_channelid_type(mod); - if (xid_types->ChannelID == NULL) { + state->ChannelIDType = add_new_type( + mod, &channelid_typespec, _channelid_shared, xid_classes); + if (state->ChannelIDType == NULL) { goto error; } @@ -3325,8 +3332,8 @@ module_exec(PyObject *mod) return 0; error: - if (xid_types != NULL) { - clear_xid_types(xid_types); + if (xid_classes != NULL) { + clear_xid_class_registry(xid_classes); } _globals_fini(); return -1; @@ -3353,6 +3360,9 @@ module_clear(PyObject *mod) module_state *state = get_module_state(mod); assert(state != NULL); + // Before clearing anything, we unregister the various XID types. */ + clear_xid_class_registry(&state->xid_classes); + // Now we clear the module state. clear_module_state(state); return 0; @@ -3364,6 +3374,9 @@ module_free(void *mod) module_state *state = get_module_state(mod); assert(state != NULL); + // Before clearing anything, we unregister the various XID types. */ + clear_xid_class_registry(&state->xid_classes); + // Now we clear the module state. clear_module_state(state); From 6f511d63a66b708a95adcc8b7670d18ca6943709 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 15 Feb 2024 11:18:20 -0700 Subject: [PATCH 4/8] Fix the channels module (more simply). --- Modules/_xxinterpchannelsmodule.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index a2974aced12ca0..8e3757bcf85893 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -2699,12 +2699,16 @@ set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) } struct xid_class_registry *xid_classes = &state->xid_classes; - if (state->send_channel_type != NULL - || state->recv_channel_type != NULL) - { - PyErr_SetString(PyExc_TypeError, "already registered"); - return -1; + // Clear the old values if the .py module was reloaded. + if (state->send_channel_type != NULL) { + (void)_PyCrossInterpreterData_UnregisterClass(state->send_channel_type); + Py_CLEAR(state->send_channel_type); } + if (state->recv_channel_type != NULL) { + (void)_PyCrossInterpreterData_UnregisterClass(state->recv_channel_type); + Py_CLEAR(state->recv_channel_type); + } + state->send_channel_type = (PyTypeObject *)Py_NewRef(send); state->recv_channel_type = (PyTypeObject *)Py_NewRef(recv); From 9c0fd58eec8f048d561b35add59698e1f85937e5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 15 Feb 2024 11:39:24 -0700 Subject: [PATCH 5/8] Properly clean up in set_channelend_types(). --- Modules/_xxinterpchannelsmodule.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 8e3757bcf85893..00eec2b152ee3f 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -2709,13 +2709,18 @@ set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) Py_CLEAR(state->recv_channel_type); } + // Add and register the types. state->send_channel_type = (PyTypeObject *)Py_NewRef(send); state->recv_channel_type = (PyTypeObject *)Py_NewRef(recv); - if (register_xid_class(send, _channelend_shared, xid_classes)) { + Py_CLEAR(state->send_channel_type); + Py_CLEAR(state->recv_channel_type); return -1; } if (register_xid_class(recv, _channelend_shared, xid_classes)) { + (void)_PyCrossInterpreterData_UnregisterClass(state->send_channel_type); + Py_CLEAR(state->send_channel_type); + Py_CLEAR(state->recv_channel_type); return -1; } From 46d1243d88c9131bd16b09190beacf5798b95761 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Mar 2024 10:41:08 -0700 Subject: [PATCH 6/8] Add tests. --- Lib/test/test__xxinterpchannels.py | 5 +++++ Lib/test/test_interpreters/test_channels.py | 19 +++++++++++++++++++ Lib/test/test_interpreters/test_queues.py | 14 ++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/Lib/test/test__xxinterpchannels.py b/Lib/test/test__xxinterpchannels.py index cc2ed7849b0c0f..c5d29bd2dd911f 100644 --- a/Lib/test/test__xxinterpchannels.py +++ b/Lib/test/test__xxinterpchannels.py @@ -18,6 +18,11 @@ channels = import_helper.import_module('_xxinterpchannels') +# Additional tests are found in Lib/test/test_interpreters/test_channels.py. +# New tests should be added there. +# XXX The tests here should be moved there. See the note under LowLevelTests. + + ################################## # helpers diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 07e503837bcf75..57204e2776468d 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -1,3 +1,4 @@ +import importlib import threading from textwrap import dedent import unittest @@ -11,6 +12,24 @@ from .utils import _run_output, TestBase +class LowLevelTests(TestBase): + + # The behaviors in the low-level module is important in as much + # as it is exercised by the high-level module. Therefore the + # most # important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + # Additional tests are found in Lib/test/test__xxinterpchannels.py. + # XXX Those should be either moved to LowLevelTests or eliminated + # in favor of high-level tests in this file. + + def test_highlevel_reloaded(self): + # See gh-115490 (https://github.com/python/cpython/issues/115490). + importlib.reload(channels) + + class TestChannels(TestBase): def test_create(self): diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 905ef035d43feb..5dd01708cd648b 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -1,3 +1,4 @@ +import importlib import threading from textwrap import dedent import unittest @@ -20,6 +21,19 @@ def tearDown(self): pass +class LowLevelTests(TestBase): + + # The behaviors in the low-level module is important in as much + # as it is exercised by the high-level module. Therefore the + # most # important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + def test_highlevel_reloaded(self): + importlib.reload(queues) + + class QueueTests(TestBase): def test_create(self): From eba6059827de79e5c22329407b6185aa37850d3d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Mar 2024 10:45:29 -0700 Subject: [PATCH 7/8] Add a comment. --- Lib/test/test_interpreters/test_queues.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 5dd01708cd648b..0a1fdb41f73166 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -31,6 +31,7 @@ class LowLevelTests(TestBase): # mostly shouldn't matter as much. def test_highlevel_reloaded(self): + # See gh-115490 (https://github.com/python/cpython/issues/115490). importlib.reload(queues) From a84bbfbe5f5ac2397a5d0dcade9434636fdec019 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Mar 2024 10:53:02 -0700 Subject: [PATCH 8/8] Revert "gh-115490: Work around test.support.interpreters.channels not handling unloading (#115515)" This reverts commit b0e5c35ded6d4a16d7a021c10c99bac94250edd0. --- Lib/test/libregrtest/main.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 6fbe3d00981ddd..126daca388fd7f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -384,15 +384,10 @@ def run_tests_sequentially(self, runtests) -> None: result = self.run_test(test_name, runtests, tracer) - # Unload the newly imported test modules (best effort - # finalization). To work around gh-115490, don't unload - # test.support.interpreters and its submodules even if they - # weren't loaded before. - keep = "test.support.interpreters" + # Unload the newly imported test modules (best effort finalization) new_modules = [module for module in sys.modules if module not in save_modules and - module.startswith(("test.", "test_")) - and not module.startswith(keep)] + module.startswith(("test.", "test_"))] for module in new_modules: sys.modules.pop(module, None) # Remove the attribute of the parent module.