diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 710135dca89eda..cbec8a699303bd 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -102,6 +102,10 @@ Module Objects Return a pointer to the :c:type:`PyModuleDef` struct from which the module was created, or ``NULL`` if the module wasn't created from a definition. + On error, return ``NULL`` with an exception set. + Use :c:func:`PyErr_Occurred` to tell this case apart from a mising + :c:type:`!PyModuleDef`. + .. c:function:: PyObject* PyModule_GetFilenameObject(PyObject *module) diff --git a/Include/exports.h b/Include/exports.h index 0c646d5beb6ad6..62feb09ed2b433 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -9,6 +9,7 @@ inside the Python core, they are private to the core. If in an extension module, it may be declared with external linkage depending on the platform. + PyMODEXPORT_FUNC: Like PyMODINIT_FUNC, but for a slots array As a number of platforms support/require "__declspec(dllimport/dllexport)", we support a HAVE_DECLSPEC_DLL macro to save duplication. @@ -62,9 +63,9 @@ /* module init functions inside the core need no external linkage */ /* except for Cygwin to handle embedding */ # if defined(__CYGWIN__) -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL # else /* __CYGWIN__ */ -# define PyMODINIT_FUNC PyObject* +# define _PyINIT_FUNC_DECLSPEC # endif /* __CYGWIN__ */ # else /* Py_BUILD_CORE */ /* Building an extension module, or an embedded situation */ @@ -78,9 +79,9 @@ # define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE /* module init functions outside the core must be exported */ # if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC extern "C" Py_EXPORTED_SYMBOL # else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL # endif /* __cplusplus */ # endif /* Py_BUILD_CORE */ # endif /* HAVE_DECLSPEC_DLL */ @@ -93,13 +94,15 @@ #ifndef PyAPI_DATA # define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE #endif -#ifndef PyMODINIT_FUNC +#ifndef _PyINIT_FUNC_DECLSPEC # if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC extern "C" Py_EXPORTED_SYMBOL # else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL # endif /* __cplusplus */ #endif +#define PyMODINIT_FUNC _PyINIT_FUNC_DECLSPEC PyObject* +#define PyMODEXPORT_FUNC _PyINIT_FUNC_DECLSPEC PyModuleDef_Slot* #endif /* Py_EXPORTS_H */ diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index 3ba9229cc21378..75a32b607dfbbc 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -28,6 +28,11 @@ typedef enum ext_module_origin { _Py_ext_module_origin_DYNAMIC = 3, } _Py_ext_module_origin; +struct hook_prefixes { + const char *const init_prefix; + const char *const export_prefix; +}; + /* Input for loading an extension module. */ struct _Py_ext_module_loader_info { PyObject *filename; @@ -40,7 +45,7 @@ struct _Py_ext_module_loader_info { * depending on if it's builtin or not. */ PyObject *path; _Py_ext_module_origin origin; - const char *hook_prefix; + const struct hook_prefixes *hook_prefixes; const char *newcontext; }; extern void _Py_ext_module_loader_info_clear( @@ -62,7 +67,9 @@ extern int _Py_ext_module_loader_info_init_from_spec( PyObject *spec); #endif -/* The result from running an extension module's init function. */ +/* The result from running an extension module's init function. + * Not used for modules defined via PyModExport (slots array). + */ struct _Py_ext_module_loader_result { PyModuleDef *def; PyObject *module; @@ -89,10 +96,12 @@ extern void _Py_ext_module_loader_result_apply_error( /* The module init function. */ typedef PyObject *(*PyModInitFunction)(void); +typedef PyModuleDef_Slot *(*PyModExportFunction)(PyObject *spec); #ifdef HAVE_DYNAMIC_LOADING -extern PyModInitFunction _PyImport_GetModInitFunc( +// function changed signature, the "2" suffix helps avoid ABI issues +extern int _PyImport_GetModInitFunc2( struct _Py_ext_module_loader_info *info, - FILE *fp); + FILE *fp, PyModInitFunction *modinit, PyModExportFunction *modexport); #endif extern int _PyImport_RunModInitFunc( PyModInitFunction p0, diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index b170d7bce702c6..7500ef6600b570 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -16,10 +16,14 @@ extern int _PyModule_IsPossiblyShadowing(PyObject *); extern int _PyModule_IsExtension(PyObject *obj); +typedef int (*_Py_modexecfunc)(PyObject *); + typedef struct { PyObject_HEAD PyObject *md_dict; - PyModuleDef *md_def; + // The PyModuleDef used to define the module, if any. + // (used to be `md_def`; renamed because all uses need inspection) + PyModuleDef *md_def_or_null; void *md_state; PyObject *md_weaklist; // for logging purposes after md_dict is cleared @@ -27,11 +31,17 @@ typedef struct { #ifdef Py_GIL_DISABLED void *md_gil; #endif + Py_ssize_t md_size; + traverseproc md_traverse; + inquiry md_clear; + freefunc md_free; + void *md_token; + _Py_modexecfunc md_exec; /* only set if md_def_or_null is NULL */ } PyModuleObject; -static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) { +static inline PyModuleDef* _PyModule_GetDefOrNull(PyObject *mod) { assert(PyModule_Check(mod)); - return ((PyModuleObject *)mod)->md_def; + return ((PyModuleObject *)mod)->md_def_or_null; } static inline void* _PyModule_GetState(PyObject* mod) { diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 2a17c891dda811..57d5b689b7151d 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -80,9 +80,22 @@ struct PyModuleDef_Slot { # define Py_mod_gil 4 #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +#define Py_mod_name 5 +#define Py_mod_doc 6 +#define Py_mod_state_size 7 +#define Py_mod_methods 8 +#define Py_mod_state_traverse 9 +#define Py_mod_state_clear 10 +#define Py_mod_state_free 11 +#define Py_mod_token 12 +#endif + #ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 4 +#define _Py_mod_LAST_SLOT 12 +PyAPI_FUNC(int) _PyModule_GetGCHooks( + PyObject *, traverseproc*, inquiry*, freefunc*); // For testing #endif #endif /* New in 3.5 */ @@ -104,6 +117,15 @@ struct PyModuleDef_Slot { PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(PyObject *) PyModule_FromSlotsAndSpec(PyModuleDef_Slot *, + PyObject *spec); +PyAPI_FUNC(int) PyModule_Exec(PyObject *mod); +PyAPI_FUNC(int) PyModule_GetSize(PyObject *mod, Py_ssize_t *size_p); +PyAPI_FUNC(int) PyModule_GetToken(PyObject *, void **token_p); +#endif + + struct PyModuleDef { PyModuleDef_Base m_base; const char* m_name; diff --git a/Lib/test/test_capi/test_module.py b/Lib/test/test_capi/test_module.py new file mode 100644 index 00000000000000..427a01e41897f7 --- /dev/null +++ b/Lib/test/test_capi/test_module.py @@ -0,0 +1,136 @@ +import unittest +import types +from test.support import import_helper + +# Skip this test if the _testcapi module isn't available. +_testcapi = import_helper.import_module('_testcapi') + + +class FakeSpec: + name = 'testmod' + +DEF_SLOTS = ( + 'Py_mod_name', 'Py_mod_doc', 'Py_mod_state_size', 'Py_mod_methods', + 'Py_mod_state_traverse', 'Py_mod_state_clear', 'Py_mod_state_free', + 'Py_mod_token', +) + +# The C functions used by this module are in: +# Modules/_testcapi/module.c + +class TestModFromSlotsAndSpec(unittest.TestCase): + def test_empty(self): + mod = _testcapi.module_from_slots_empty(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'testmod') + + def test_null_slots(self): + with self.assertRaises(SystemError): + _testcapi.module_from_slots_null(FakeSpec()) + + def test_none_spec(self): + # The spec currently must contain a name + with self.assertRaises(AttributeError): + _testcapi.module_from_slots_empty(None) + with self.assertRaises(AttributeError): + _testcapi.module_from_slots_name(None) + + def test_name(self): + # Py_mod_name (and PyModuleDef.m_name) are currently ignored when + # spec is given. + # We still test that it's accepted. + mod = _testcapi.module_from_slots_name(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + + def test_doc(self): + mod = _testcapi.module_from_slots_doc(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, 'the docstring') + + def test_size(self): + mod = _testcapi.module_from_slots_size(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + self.assertEqual(mod.size, 123) + + def test_methods(self): + mod = _testcapi.module_from_slots_methods(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + self.assertEqual(mod.a_method(456), (mod, 456)) + + def test_gc(self): + mod = _testcapi.module_from_slots_gc(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + + def test_token(self): + mod = _testcapi.module_from_slots_token(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + + def test_exec(self): + mod = _testcapi.module_from_slots_exec(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + self.assertEqual(mod.a_number, 456) + + def test_create(self): + spec = FakeSpec() + spec._gimme_this = "not a module object" + mod = _testcapi.module_from_slots_create(spec) + self.assertIsInstance(mod, str) + self.assertEqual(mod, "not a module object") + + def test_def_slot(self): + """Slots that replace PyModuleDef fields can't be used with PyModuleDef + """ + for name in DEF_SLOTS: + with self.subTest(name): + spec = FakeSpec() + spec._test_slot_id = getattr(_testcapi, name) + with self.assertRaises(SystemError) as cm: + _testcapi.module_from_def_slot(spec) + self.assertIn(name, str(cm.exception)) + self.assertIn("PyModuleDef", str(cm.exception)) + + def test_repeated_def_slot(self): + """Slots that replace PyModuleDef fields can't be repeated""" + for name in (*DEF_SLOTS, 'Py_mod_exec'): + with self.subTest(name): + spec = FakeSpec() + spec._test_slot_id = getattr(_testcapi, name) + with self.assertRaises(SystemError) as cm: + _testcapi.module_from_slots_repeat_slot(spec) + self.assertIn(name, str(cm.exception)) + self.assertIn("more than one", str(cm.exception)) + + def test_null_def_slot(self): + """Slots that replace PyModuleDef fields can't be NULL""" + for name in (*DEF_SLOTS, 'Py_mod_exec'): + with self.subTest(name): + spec = FakeSpec() + spec._test_slot_id = getattr(_testcapi, name) + with self.assertRaises(SystemError) as cm: + _testcapi.module_from_slots_null_slot(spec) + self.assertIn(name, str(cm.exception)) + self.assertIn("NULL", str(cm.exception)) + + def test_def_multiple_exec(self): + """PyModule_Exec runs all exec slots of PyModuleDef-defined module""" + mod = _testcapi.module_from_def_multiple_exec(FakeSpec()) + self.assertFalse(hasattr(mod, 'a_number')) + _testcapi.pymodule_exec(mod) + self.assertEqual(mod.a_number, 456) + self.assertEqual(mod.another_number, 789) + _testcapi.pymodule_exec(mod) + self.assertEqual(mod.a_number, 456) + self.assertEqual(mod.another_number, -789) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 6e34094c5aa422..bc3ab78e489056 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2466,6 +2466,66 @@ def test_multi_init_extension_per_interpreter_gil_compat(self): self.check_compatible_here( modname, filename, strict=False, isolated=False) + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_testmultiphase_exec_multiple(self): + modname = '_testmultiphase_exec_multiple' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + # All three exec's were called. + self.assertEqual(module.a, 1) + self.assertEqual(module.b, 2) + self.assertEqual(module.c, 3) + # They were called in order. + keys = list(module.__dict__) + self.assertLess(keys.index('a'), keys.index('b')) + self.assertLess(keys.index('b'), keys.index('c')) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_from_modexport(self): + modname = '_test_from_modexport' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + + self.assertEqual(module.__name__, modname) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_from_modexport_null(self): + modname = '_test_from_modexport_null' + filename = _testmultiphase.__file__ + with self.assertRaises(SystemError): + import_extension_from_file(modname, filename, + put_in_sys_modules=False) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_from_modexport_exception(self): + modname = '_test_from_modexport_exception' + filename = _testmultiphase.__file__ + with self.assertRaises(ValueError): + import_extension_from_file(modname, filename, + put_in_sys_modules=False) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_from_modexport_create_nonmodule(self): + modname = '_test_from_modexport_create_nonmodule' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + self.assertIsInstance(module, str) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_from_modexport_smoke(self): + # General positive test for sundry features + # (PyModule_FromSlotsAndSpec tests exercise test these more carefully) + modname = '_test_from_modexport_smoke' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + self.assertEqual(module.__doc__, "the expected docstring") + self.assertEqual(module.number, 147) + self.assertEqual(module.get_state_int(), 258) + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_python_compat(self): module = 'threading' diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index bf415894903e9b..d4e0ab63335e5e 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1707,9 +1707,10 @@ def get_gen(): yield 1 check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) # module if support.Py_GIL_DISABLED: - check(unittest, size('PPPPPP')) + md_gil = 'P' else: - check(unittest, size('PPPPP')) + md_gil = '' + check(unittest, size('PPPPP' + md_gil + 'NPPPPP')) # None check(None, size('')) # NotImplementedType diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 3a38a60a152e8c..ac2fc3abbe28bb 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -178,7 +178,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c @MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/module.c b/Modules/_testcapi/module.c new file mode 100644 index 00000000000000..e59b3f5af32b26 --- /dev/null +++ b/Modules/_testcapi/module.c @@ -0,0 +1,344 @@ +#include "parts.h" +#include "util.h" + +// Test PyModule_* API + +/* unittest Cases that use these functions are in: + * Lib/test/test_capi/test_module.py + */ + +static PyObject * +module_from_slots_empty(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_slots_null(PyObject *self, PyObject *spec) +{ + return PyModule_FromSlotsAndSpec(NULL, spec); +} + +static PyObject * +module_from_slots_name(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_name, "currently ignored..."}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_slots_doc(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_doc, "the docstring"}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_slots_size(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_state_size, (void*)123}, + {0}, + }; + PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); + if (!mod) { + return NULL; + } + Py_ssize_t size; + if (PyModule_GetSize(mod, &size) < 0) { + Py_DECREF(mod); + return NULL; + } + if (PyModule_AddIntConstant(mod, "size", size) < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} + +static PyObject * +a_method(PyObject *self, PyObject *arg) +{ + return PyTuple_Pack(2, self, arg); +} + +static PyMethodDef a_methoddef_array[] = { + {"a_method", a_method, METH_O}, + {0}, +}; + +static PyObject * +module_from_slots_methods(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_methods, a_methoddef_array}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static int trivial_traverse(PyObject *self, visitproc visit, void *arg) { + return 0; +} +static int trivial_clear(PyObject *self) { return 0; } +static void trivial_free(void *self) { } + +static PyObject * +module_from_slots_gc(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_state_traverse, trivial_traverse}, + {Py_mod_state_clear, trivial_clear}, + {Py_mod_state_free, trivial_free}, + {0}, + }; + PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); + if (!mod) { + return NULL; + } + traverseproc traverse; + inquiry clear; + freefunc free; + if (_PyModule_GetGCHooks(mod, &traverse, &clear, &free) < 0) { + Py_DECREF(mod); + return NULL; + } + assert(traverse == &trivial_traverse); + assert(clear == &trivial_clear); + assert(free == &trivial_free); + return mod; +} + +static char test_token; + +static PyObject * +module_from_slots_token(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_token, &test_token}, + {0}, + }; + PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); + if (!mod) { + return NULL; + } + void *got_token; + if (PyModule_GetToken(mod, &got_token) < 0) { + Py_DECREF(mod); + return NULL; + } + assert(got_token == &test_token); + return mod; +} + +static int +simple_exec(PyObject *module) +{ + return PyModule_AddIntConstant(module, "a_number", 456); +} + +static PyObject * +module_from_slots_exec(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_exec, simple_exec}, + {0}, + }; + PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); + if (!mod) { + return NULL; + } + int res = PyObject_HasAttrStringWithError(mod, "a_number"); + if (res < 0) { + Py_DECREF(mod); + return NULL; + } + assert(res == 0); + if (PyModule_Exec(mod) < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} + +static PyObject * +create_attr_from_spec(PyObject *spec, PyObject *def) +{ + assert(!def); + return PyObject_GetAttrString(spec, "_gimme_this"); +} + +static PyObject * +module_from_slots_create(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_create, create_attr_from_spec}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + + +static int +slot_from_object(PyObject *obj) +{ + PyObject *slot_id_obj = PyObject_GetAttrString(obj, "_test_slot_id"); + if (slot_id_obj == NULL) { + return -1; + } + int slot_id = PyLong_AsLong(slot_id_obj); + if (PyErr_Occurred()) { + return -1; + } + return slot_id; +} + +static PyObject * +module_from_slots_repeat_slot(PyObject *self, PyObject *spec) +{ + int slot_id = slot_from_object(spec); + if (slot_id < 0) { + return NULL; + } + PyModuleDef_Slot slots[] = { + {slot_id, "anything"}, + {slot_id, "anything else"}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_slots_null_slot(PyObject *self, PyObject *spec) +{ + int slot_id = slot_from_object(spec); + if (slot_id < 0) { + return NULL; + } + PyModuleDef_Slot slots[] = { + {slot_id, NULL}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_def_slot(PyObject *self, PyObject *spec) +{ + int slot_id = slot_from_object(spec); + if (slot_id < 0) { + return NULL; + } + PyModuleDef_Slot slots[] = { + {slot_id, "anything"}, + {0}, + }; + PyModuleDef def = { + .m_name = "currently ignored", + .m_slots = slots, + }; + // PyModuleDef is normally static; the real requirement is that it + // must outlive its module. + // Here, module creation fails, so it's fine on the stack. + PyObject *result = PyModule_FromDefAndSpec(&def, spec); + assert(result == NULL); + return result; +} + +static int +another_exec(PyObject *module) +{ + /* Make sure simple_exec was called */ + assert(PyObject_HasAttrString(module, "a_number")); + + /* Add or negate a global called 'another_number' */ + PyObject *another_number; + if (PyObject_GetOptionalAttrString(module, "another_number", + &another_number) < 0) { + return -1; + } + if (!another_number) { + return PyModule_AddIntConstant(module, "another_number", 789); + } + PyObject *neg_number = PyNumber_Negative(another_number); + Py_DECREF(another_number); + if (!neg_number) { + return -1; + } + int result = PyObject_SetAttrString(module, "another_number", + neg_number); + Py_DECREF(neg_number); + return result; +} + +static PyObject * +module_from_def_multiple_exec(PyObject *self, PyObject *spec) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_exec, simple_exec}, + {Py_mod_exec, another_exec}, + {0}, + }; + static PyModuleDef def = { + .m_name = "currently ignored", + .m_slots = slots, + }; + return PyModule_FromDefAndSpec(&def, spec); +} + +static PyObject * +pymodule_exec(PyObject *self, PyObject *module) +{ + if (PyModule_Exec(module) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"module_from_slots_empty", module_from_slots_empty, METH_O}, + {"module_from_slots_null", module_from_slots_null, METH_O}, + {"module_from_slots_name", module_from_slots_name, METH_O}, + {"module_from_slots_doc", module_from_slots_doc, METH_O}, + {"module_from_slots_size", module_from_slots_size, METH_O}, + {"module_from_slots_methods", module_from_slots_methods, METH_O}, + {"module_from_slots_gc", module_from_slots_gc, METH_O}, + {"module_from_slots_token", module_from_slots_token, METH_O}, + {"module_from_slots_exec", module_from_slots_exec, METH_O}, + {"module_from_slots_create", module_from_slots_create, METH_O}, + {"module_from_slots_repeat_slot", module_from_slots_repeat_slot, METH_O}, + {"module_from_slots_null_slot", module_from_slots_null_slot, METH_O}, + {"module_from_def_multiple_exec", module_from_def_multiple_exec, METH_O}, + {"module_from_def_slot", module_from_def_slot, METH_O}, + {"pymodule_exec", pymodule_exec, METH_O}, + {NULL}, +}; + +int +_PyTestCapi_Init_Module(PyObject *m) +{ +#define ADD_INT_MACRO(C) if (PyModule_AddIntConstant(m, #C, C) < 0) return -1; + ADD_INT_MACRO(Py_mod_create); + ADD_INT_MACRO(Py_mod_exec); + ADD_INT_MACRO(Py_mod_multiple_interpreters); + ADD_INT_MACRO(Py_mod_gil); + ADD_INT_MACRO(Py_mod_name); + ADD_INT_MACRO(Py_mod_doc); + ADD_INT_MACRO(Py_mod_state_size); + ADD_INT_MACRO(Py_mod_methods); + ADD_INT_MACRO(Py_mod_state_traverse); + ADD_INT_MACRO(Py_mod_state_clear); + ADD_INT_MACRO(Py_mod_state_free); + ADD_INT_MACRO(Py_mod_token); +#undef ADD_INT_MACRO + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index af6400162daf2b..af845957308c01 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -65,5 +65,6 @@ int _PyTestCapi_Init_Import(PyObject *mod); int _PyTestCapi_Init_Frame(PyObject *mod); int _PyTestCapi_Init_Type(PyObject *mod); int _PyTestCapi_Init_Function(PyObject *mod); +int _PyTestCapi_Init_Module(PyObject *mod); #endif // Py_TESTCAPI_PARTS_H diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 281c5b41137ac2..ccb4fbc8a7f53a 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3477,6 +3477,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Function(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Module(m) < 0) { + return NULL; + } PyState_AddModule(m, &_testcapimodule); return m; diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index bfec0678e2c669..17e1e308d20597 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -850,6 +850,26 @@ PyInit__testmultiphase_exec_unreported_exception(void) return PyModuleDef_Init(&def_exec_unreported_exception); } +static int execfunc_a1(PyObject*m) {return PyModule_AddIntConstant(m, "a", 1);} +static int execfunc_b2(PyObject*m) {return PyModule_AddIntConstant(m, "b", 2);} +static int execfunc_c3(PyObject*m) {return PyModule_AddIntConstant(m, "c", 3);} + +PyMODINIT_FUNC +PyInit__testmultiphase_exec_multiple(void) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_exec, execfunc_a1}, + {Py_mod_exec, execfunc_b2}, + {Py_mod_exec, execfunc_c3}, + {0} + }; + static PyModuleDef def = { + .m_name="_testmultiphase_exec_multiple", + .m_slots=slots, + }; + return PyModuleDef_Init(&def); +} + static int meth_state_access_exec(PyObject *m) { @@ -993,3 +1013,129 @@ PyInit__test_no_multiple_interpreter_slot(void) { return PyModuleDef_Init(&no_multiple_interpreter_slot_def); } + + +/* PyModExport_* hooks */ + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport(PyObject *spec) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_name, "_test_from_modexport"}, + {0}, + }; + return slots; +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_null(PyObject *spec) +{ + PyObject *exc; + if (PyObject_GetOptionalAttrString(spec, "_test_exception", &exc) < 0) { + return NULL; + } + if (exc) { + PyErr_SetObject((PyObject*)Py_TYPE(exc), exc); + } + Py_XDECREF(exc); + return NULL; +} + +PyMODINIT_FUNC +PyModInit__test_from_modexport_null(PyObject *spec) +{ + // This is not called as fallback for failed PyModExport_* + assert(0); + PyErr_SetString(PyExc_AssertionError, "PyInit_ fallback called"); + return NULL; +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_exception(PyObject *spec) +{ + PyErr_SetString(PyExc_ValueError, "failed as requested"); + return NULL; +} + +PyMODINIT_FUNC +PyModInit__test_from_modexport_exception(PyObject *spec) +{ + // This is not called as fallback for failed PyModExport_* + assert(0); + PyErr_SetString(PyExc_AssertionError, "PyInit_ fallback called"); + return NULL; +} + +static PyObject * +modexport_create_string(PyObject *spec, PyObject *def) +{ + assert(def == NULL); + return PyUnicode_FromString("is this \xf0\x9f\xa6\x8b... a module?"); +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_create_nonmodule(PyObject *spec) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_name, "_test_from_modexport_create_nonmodule"}, + {Py_mod_create, modexport_create_string}, + {0}, + }; + return slots; +} + +static int +modexport_smoke_exec(PyObject *mod) +{ + // "magic" values 147 & 258 are expected in the test + if (PyModule_AddIntConstant(mod, "number", 147) < 0) { + return 0; + } + int *state = PyModule_GetState(mod); + if (!state) { + return -1; + } + *state = 258; + Py_INCREF(mod); // do be cleared in modexport_smoke_free + return 0; +} + +static PyObject * +modexport_smoke_get_state_int(PyObject *mod, PyObject *arg) +{ + int *state = PyModule_GetState(mod); + if (!state) { + return NULL; + } + return PyLong_FromLong(*state); +} + +static void +modexport_smoke_free(PyObject *mod) +{ + int *state = PyModule_GetState(mod); + if (!state) { + PyErr_WriteUnraisable(mod); + } + assert(*state == 258); + Py_DECREF(mod); +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_smoke(PyObject *spec) +{ + static PyMethodDef methods[] = { + {"get_state_int", modexport_smoke_get_state_int, METH_NOARGS}, + {0}, + }; + static PyModuleDef_Slot slots[] = { + {Py_mod_name, "_test_from_modexport_smoke"}, + {Py_mod_doc, "the expected docstring"}, + {Py_mod_exec, modexport_smoke_exec}, + {Py_mod_state_size, (void*)sizeof(int)}, + {Py_mod_methods, methods}, + {Py_mod_state_free, modexport_smoke_free}, + {0}, + }; + return slots; +} diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 2c59085d15b5be..ee38d61b43a82a 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -244,6 +244,8 @@ static inline module_state * get_module_state(PyObject *module) { PyModuleDef *def = PyModule_GetDef(module); + assert(def); + if (def->m_size == -1) { return &global_state.module; } diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index ba86b41e945e9d..1820cdb6b18961 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -8,7 +8,7 @@ #include "pycore_interp.h" // PyInterpreterState.importlib #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyModule_CreateInitialized() -#include "pycore_moduleobject.h" // _PyModule_GetDef() +#include "pycore_moduleobject.h" // _PyModule_GetDefOrNull() #include "pycore_object.h" // _PyType_AllocNoTrack #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -26,6 +26,18 @@ static PyMemberDef module_members[] = { {0} }; +static void +assert_def_missing_or_redundant(PyModuleObject *m) { + if (m->md_def_or_null) { +#define DO_ASSERT(F) assert (m->md_def_or_null->m_ ## F == m->md_ ## F); + DO_ASSERT(size); + DO_ASSERT(traverse); + DO_ASSERT(clear); + DO_ASSERT(free); +#undef DO_ASSERT + } +} + PyTypeObject PyModuleDef_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -43,7 +55,10 @@ _PyModule_IsExtension(PyObject *obj) } PyModuleObject *module = (PyModuleObject*)obj; - PyModuleDef *def = module->md_def; + if (module->md_exec) { + return 1; + } + PyModuleDef *def = module->md_def_or_null; return (def != NULL && def->m_methods != NULL); } @@ -92,10 +107,16 @@ new_module_notrack(PyTypeObject *mt) m = (PyModuleObject *)_PyType_AllocNoTrack(mt, 0); if (m == NULL) return NULL; - m->md_def = NULL; + m->md_def_or_null = NULL; m->md_state = NULL; m->md_weaklist = NULL; m->md_name = NULL; + m->md_size = 0; + m->md_traverse = NULL; + m->md_clear = NULL; + m->md_free = NULL; + m->md_exec = NULL; + m->md_token = NULL; m->md_dict = PyDict_New(); if (m->md_dict == NULL) { Py_DECREF(m); @@ -210,6 +231,18 @@ PyModule_Create2(PyModuleDef* module, int module_api_version) return _PyModule_CreateInitialized(module, module_api_version); } +static void +module_copy_members_from_deflike( + PyModuleObject *md, + PyModuleDef *def_like /* not necessarily a valid Python object */) +{ + /* def may not be a valid PyObject*, see */ + md->md_size = def_like->m_size; + md->md_traverse = def_like->m_traverse; + md->md_clear = def_like->m_clear; + md->md_free = def_like->m_free; +} + PyObject * _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) { @@ -256,7 +289,8 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) return NULL; } } - m->md_def = module; + m->md_def_or_null = module; + module_copy_members_from_deflike(m, module); #ifdef Py_GIL_DISABLED m->md_gil = Py_MOD_GIL_USED; #endif @@ -264,7 +298,11 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) } PyObject * -PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_version) +module_from_def_and_spec( + PyModuleDef* def_like, /* not necessarily a valid Python object */ + PyObject *spec, + int module_api_version, + PyModuleDef* original_def /* NULL if not defined by a def */) { PyModuleDef_Slot* cur_slot; PyObject *(*create)(PyObject *, PyModuleDef*) = NULL; @@ -277,10 +315,10 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio int has_execution_slots = 0; const char *name; int ret; + void *token = NULL; + _Py_modexecfunc m_exec = NULL; PyInterpreterState *interp = _PyInterpreterState_GET(); - PyModuleDef_Init(def); - nameobj = PyObject_GetAttrString(spec, "name"); if (nameobj == NULL) { return NULL; @@ -294,7 +332,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio goto error; } - if (def->m_size < 0) { + if (def_like->m_size < 0) { PyErr_Format( PyExc_SystemError, "module %s: m_size may not be negative for multi-phase initialization", @@ -302,7 +340,33 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio goto error; } - for (cur_slot = def->m_slots; cur_slot && cur_slot->slot; cur_slot++) { + for (cur_slot = def_like->m_slots; cur_slot && cur_slot->slot; cur_slot++) { + // Macro to copy a non-NULL, non-repeatable slot that's unusable with + // PyModuleDef. The destination must be initially NULL. +#define COPY_COMMON_SLOT(SLOT, TYPE, DEST) \ + if (!(TYPE)(cur_slot->value)) { \ + PyErr_Format( \ + PyExc_SystemError, \ + "module %s: " #SLOT " must not be NULL", \ + name); \ + goto error; \ + } \ + if (original_def) { \ + PyErr_Format( \ + PyExc_SystemError, \ + "module %s: " #SLOT " used with PyModuleDef", \ + name); \ + goto error; \ + } \ + if (DEST) { \ + PyErr_Format( \ + PyExc_SystemError, \ + "module %s has more than one " #SLOT " slot", \ + name); \ + goto error; \ + } \ + DEST = (TYPE)(cur_slot->value); \ + ///////////////////////////////////////////////////////////// switch (cur_slot->slot) { case Py_mod_create: if (create) { @@ -316,6 +380,9 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio break; case Py_mod_exec: has_execution_slots = 1; + if (!original_def) { + COPY_COMMON_SLOT(Py_mod_exec, _Py_modexecfunc, m_exec); + } break; case Py_mod_multiple_interpreters: if (has_multiple_interpreters_slot) { @@ -339,6 +406,35 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio gil_slot = cur_slot->value; has_gil_slot = 1; break; + case Py_mod_name: + COPY_COMMON_SLOT(Py_mod_name, char*, def_like->m_name); + break; + case Py_mod_doc: + COPY_COMMON_SLOT(Py_mod_doc, char*, def_like->m_doc); + break; + case Py_mod_state_size: + COPY_COMMON_SLOT(Py_mod_state_size, Py_ssize_t, + def_like->m_size); + break; + case Py_mod_methods: + COPY_COMMON_SLOT(Py_mod_methods, PyMethodDef*, + def_like->m_methods); + break; + case Py_mod_state_traverse: + COPY_COMMON_SLOT(Py_mod_state_traverse, traverseproc, + def_like->m_traverse); + break; + case Py_mod_state_clear: + COPY_COMMON_SLOT(Py_mod_state_clear, inquiry, + def_like->m_clear); + break; + case Py_mod_state_free: + COPY_COMMON_SLOT(Py_mod_state_free, freefunc, + def_like->m_free); + break; + case Py_mod_token: + COPY_COMMON_SLOT(Py_mod_token, void*, token); + break; default: assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT); PyErr_Format( @@ -347,6 +443,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio name, cur_slot->slot); goto error; } +#undef COPY_COMMON_SLOT } /* By default, multi-phase init modules are expected @@ -370,7 +467,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio } if (create) { - m = create(spec, def); + m = create(spec, original_def); if (m == NULL) { if (!PyErr_Occurred()) { PyErr_Format( @@ -396,15 +493,29 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio } if (PyModule_Check(m)) { - ((PyModuleObject*)m)->md_state = NULL; - ((PyModuleObject*)m)->md_def = def; + PyModuleObject *mod = (PyModuleObject*)m; + mod->md_state = NULL; + module_copy_members_from_deflike(mod, def_like); + if (original_def) { + mod->md_def_or_null = original_def; + } #ifdef Py_GIL_DISABLED - ((PyModuleObject*)m)->md_gil = gil_slot; + mod->md_gil = gil_slot; #else (void)gil_slot; #endif + if (original_def) { + mod->md_token = original_def; + assert (!token); + } + else { + mod->md_token = token; + } + mod->md_exec = m_exec; } else { - if (def->m_size > 0 || def->m_traverse || def->m_clear || def->m_free) { + if (def_like->m_size > 0 || def_like->m_traverse || def_like->m_clear + || def_like->m_free) + { PyErr_Format( PyExc_SystemError, "module %s is not a module object, but requests module state", @@ -419,17 +530,25 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio name); goto error; } + if (token) { + PyErr_Format( + PyExc_SystemError, + "module %s specifies a token, but did not create " + "a ModuleType instance", + name); + goto error; + } } - if (def->m_methods != NULL) { - ret = _add_methods_to_object(m, nameobj, def->m_methods); + if (def_like->m_methods != NULL) { + ret = _add_methods_to_object(m, nameobj, def_like->m_methods); if (ret != 0) { goto error; } } - if (def->m_doc != NULL) { - ret = PyModule_SetDocString(m, def->m_doc); + if (def_like->m_doc != NULL) { + ret = PyModule_SetDocString(m, def_like->m_doc); if (ret != 0) { goto error; } @@ -444,6 +563,29 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio return NULL; } +PyObject * +PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_version) +{ + PyModuleDef_Init(def); + return module_from_def_and_spec(def, spec, module_api_version, def); +} + +PyObject * +PyModule_FromSlotsAndSpec(PyModuleDef_Slot *slots, PyObject *spec) +{ + if (!slots) { + PyErr_Format( + PyExc_SystemError, + "PyModule_FromSlotsAndSpec called with NULL slots"); + return NULL; + } + // Fill in enough of a PyModuleDef to pass to common machinery + PyModuleDef def_like = {.m_slots = slots}; + + return module_from_def_and_spec(&def_like, spec, PYTHON_API_VERSION, + NULL); +} + #ifdef Py_GIL_DISABLED int PyUnstable_Module_SetGIL(PyObject *module, void *gil) @@ -458,69 +600,92 @@ PyUnstable_Module_SetGIL(PyObject *module, void *gil) #endif int -PyModule_ExecDef(PyObject *module, PyModuleDef *def) +run_exec_func(PyObject *module, int (*exec)(PyObject *)) { - PyModuleDef_Slot *cur_slot; - const char *name; - int ret; + int ret = exec(module); + if (ret != 0) { + if (!PyErr_Occurred()) { + PyErr_Format( + PyExc_SystemError, + "execution of %S failed without setting an exception", + module); + } + return -1; + } + if (PyErr_Occurred()) { + _PyErr_FormatFromCause( + PyExc_SystemError, + "execution of module %S raised unreported exception", + module); + return -1; + } + return 0; +} - name = PyModule_GetName(module); - if (name == NULL) { +int +alloc_state(PyObject *module) +{ + if(!PyModule_Check(module)) { + PyErr_Format(PyExc_TypeError, "expected module, got %T", module); return -1; } + PyModuleObject *md = (PyModuleObject*)module; - if (def->m_size >= 0) { - PyModuleObject *md = (PyModuleObject*)module; + if (md->md_size >= 0) { if (md->md_state == NULL) { /* Always set a state pointer; this serves as a marker to skip * multiple initialization (importlib.reload() is no-op) */ - md->md_state = PyMem_Malloc(def->m_size); + md->md_state = PyMem_Malloc(md->md_size); if (!md->md_state) { PyErr_NoMemory(); return -1; } - memset(md->md_state, 0, def->m_size); + memset(md->md_state, 0, md->md_size); } } + return 0; +} + +int +PyModule_Exec(PyObject *module) +{ + if (alloc_state(module) < 0) { + return -1; + } + PyModuleObject *md = (PyModuleObject*)module; + if (md->md_exec) { + assert(!md->md_def_or_null); + return run_exec_func(module, md->md_exec); + } + + if (md->md_def_or_null) { + return PyModule_ExecDef(module, md->md_def_or_null); + } + return 0; +} + +int +PyModule_ExecDef(PyObject *module, PyModuleDef *def) +{ + PyModuleDef_Slot *cur_slot; + + if (alloc_state(module) < 0) { + return -1; + } + + assert(PyModule_Check(module)); if (def->m_slots == NULL) { return 0; } for (cur_slot = def->m_slots; cur_slot && cur_slot->slot; cur_slot++) { - switch (cur_slot->slot) { - case Py_mod_create: - /* handled in PyModule_FromDefAndSpec2 */ - break; - case Py_mod_exec: - ret = ((int (*)(PyObject *))cur_slot->value)(module); - if (ret != 0) { - if (!PyErr_Occurred()) { - PyErr_Format( - PyExc_SystemError, - "execution of module %s failed without setting an exception", - name); - } - return -1; - } - if (PyErr_Occurred()) { - _PyErr_FormatFromCause( - PyExc_SystemError, - "execution of module %s raised unreported exception", - name); - return -1; - } - break; - case Py_mod_multiple_interpreters: - case Py_mod_gil: - /* handled in PyModule_FromDefAndSpec2 */ - break; - default: - PyErr_Format( - PyExc_SystemError, - "module %s initialized with unknown slot %i", - name, cur_slot->slot); + if (cur_slot->slot == Py_mod_exec) { + int (*func)(PyObject *) = cur_slot->value; + if (run_exec_func(module, func) < 0) { return -1; + } + continue; } } return 0; @@ -564,6 +729,47 @@ PyModule_GetDict(PyObject *m) return _PyModule_GetDict(m); // borrowed reference } +int +PyModule_GetSize(PyObject *m, Py_ssize_t *size_p) +{ + *size_p = -1; + if (!PyModule_Check(m)) { + PyErr_BadInternalCall(); + return -1; + } + PyModuleObject *mod = (PyModuleObject *)m; + *size_p = mod->md_size; + return 0; +} + +int +PyModule_GetToken(PyObject *m, void **token_p) +{ + *token_p = NULL; + if (!PyModule_Check(m)) { + PyErr_BadInternalCall(); + return -1; + } + PyModuleObject *mod = (PyModuleObject *)m; + *token_p = mod->md_token; + return 0; +} + +int +_PyModule_GetGCHooks(PyObject *m, traverseproc *traverse, + inquiry *clear, freefunc *free) +{ + if (!PyModule_Check(m)) { + PyErr_BadInternalCall(); + return -1; + } + PyModuleObject *mod = (PyModuleObject *)m; + *traverse = mod->md_traverse; + *clear = mod->md_clear; + *free = mod->md_free; + return 0; +} + PyObject* PyModule_GetNameObject(PyObject *mod) { @@ -704,7 +910,7 @@ PyModule_GetDef(PyObject* m) PyErr_BadArgument(); return NULL; } - return _PyModule_GetDef(m); + return _PyModule_GetDefOrNull(m); } void* @@ -829,17 +1035,18 @@ module_dealloc(PyObject *self) if (m->md_weaklist != NULL) PyObject_ClearWeakRefs((PyObject *) m); + assert_def_missing_or_redundant(m); /* bpo-39824: Don't call m_free() if m_size > 0 and md_state=NULL */ - if (m->md_def && m->md_def->m_free - && (m->md_def->m_size <= 0 || m->md_state != NULL)) + if (m->md_free && (m->md_size <= 0 || m->md_state != NULL)) { - m->md_def->m_free(m); + m->md_free(m); } Py_XDECREF(m->md_dict); Py_XDECREF(m->md_name); - if (m->md_state != NULL) + if (m->md_state != NULL) { PyMem_Free(m->md_state); + } Py_TYPE(m)->tp_free((PyObject *)m); } @@ -1147,11 +1354,11 @@ module_traverse(PyObject *self, visitproc visit, void *arg) { PyModuleObject *m = _PyModule_CAST(self); + assert_def_missing_or_redundant(m); /* bpo-39824: Don't call m_traverse() if m_size > 0 and md_state=NULL */ - if (m->md_def && m->md_def->m_traverse - && (m->md_def->m_size <= 0 || m->md_state != NULL)) + if (m->md_traverse && (m->md_size <= 0 || m->md_state != NULL)) { - int res = m->md_def->m_traverse((PyObject*)m, visit, arg); + int res = m->md_traverse((PyObject*)m, visit, arg); if (res) return res; } @@ -1165,18 +1372,19 @@ module_clear(PyObject *self) { PyModuleObject *m = _PyModule_CAST(self); + assert_def_missing_or_redundant(m); /* bpo-39824: Don't call m_clear() if m_size > 0 and md_state=NULL */ - if (m->md_def && m->md_def->m_clear - && (m->md_def->m_size <= 0 || m->md_state != NULL)) + if (m->md_clear && (m->md_size <= 0 || m->md_state != NULL)) { - int res = m->md_def->m_clear((PyObject*)m); + int res = m->md_clear((PyObject*)m); if (PyErr_Occurred()) { PyErr_FormatUnraisable("Exception ignored in m_clear of module%s%V", m->md_name ? " " : "", m->md_name, ""); } - if (res) + if (res) { return res; + } } Py_CLEAR(m->md_dict); return 0; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index db923c164774b7..0e3a33d64307b4 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5719,7 +5719,7 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) else { PyHeapTypeObject *ht = (PyHeapTypeObject*)type; PyObject *module = ht->ht_module; - if (module && _PyModule_GetDef(module) == def) { + if (module && _PyModule_GetDefOrNull(module) == def) { return module; } } @@ -5747,7 +5747,7 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) PyHeapTypeObject *ht = (PyHeapTypeObject*)super; PyObject *module = ht->ht_module; - if (module && _PyModule_GetDef(module) == def) { + if (module && _PyModule_GetDefOrNull(module) == def) { res = module; break; } diff --git a/Python/import.c b/Python/import.c index 184dede335dfd6..4daf2dc2469927 100644 --- a/Python/import.c +++ b/Python/import.c @@ -743,7 +743,7 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) A. noop - ...for multi-phase init modules: + ...for multi-phase init modules from PyModInit_* (PyModuleDef): (6). every time: A. _imp_create_dynamic_impl() -> import_find_extension() (not found) @@ -753,6 +753,9 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) E. import_run_extension() -> _PyImport_RunModInitFunc() F. _PyImport_RunModInitFunc(): call G. import_run_extension() -> PyModule_FromDefAndSpec() + + PyModule_FromDefAndSpec(): + H. PyModule_FromDefAndSpec(): gather/check moduledef slots I. if there's a Py_mod_create slot: 1. PyModule_FromDefAndSpec(): call its function @@ -765,10 +768,29 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) (10). every time: A. _imp_exec_dynamic_impl() -> exec_builtin_or_dynamic() B. if mod->md_state == NULL (including if m_size == 0): - 1. exec_builtin_or_dynamic() -> PyModule_ExecDef() - 2. PyModule_ExecDef(): allocate mod->md_state + 1. exec_builtin_or_dynamic() -> PyModule_Exec() + 2. PyModule_Exec(): allocate mod->md_state 3. if there's a Py_mod_exec slot: - 1. PyModule_ExecDef(): call its function + 1. PyModule_Exec(): call its function + + + ...for multi-phase init modules from PyModExport_* (slots array): + + (6). every time: + + A. _imp_create_dynamic_impl() -> import_find_extension() (not found) + B. _imp_create_dynamic_impl() -> _PyImport_GetModInitFunc() + C. _PyImport_GetModInitFunc(): load + D. _imp_create_dynamic_impl() -> import_run_modexport() + E. import_run_modexport(): call + F. import_run_modexport() -> PyModule_FromSlotsAndSpec() + G. PyModule_FromSlotsAndSpec(): create temporary PyModuleDef-like + H. PyModule_FromSlotsAndSpec() -> PyModule_FromDefAndSpec() + + (PyModule_FromDefAndSpec behaves as for PyModInit_*, above) + + (10). every time: as for PyModInit_*, above + */ @@ -839,25 +861,19 @@ _PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val) /* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */ static int exec_builtin_or_dynamic(PyObject *mod) { - PyModuleDef *def; void *state; if (!PyModule_Check(mod)) { return 0; } - def = PyModule_GetDef(mod); - if (def == NULL) { - return 0; - } - state = PyModule_GetState(mod); if (state) { /* Already initialized; skip reload */ return 0; } - return PyModule_ExecDef(mod, def); + return PyModule_Exec(mod); } @@ -1801,7 +1817,7 @@ finish_singlephase_extension(PyThreadState *tstate, PyObject *mod, PyObject *name, PyObject *modules) { assert(mod != NULL && PyModule_Check(mod)); - assert(cached->def == _PyModule_GetDef(mod)); + assert(cached->def == _PyModule_GetDefOrNull(mod)); Py_ssize_t index = _get_cached_module_index(cached); if (_modules_by_index_set(tstate->interp, index, mod) < 0) { @@ -1879,8 +1895,8 @@ reload_singlephase_extension(PyThreadState *tstate, * due to violating interpreter isolation. * See the note in set_cached_m_dict(). * Until that is solved, we leave md_def set to NULL. */ - assert(_PyModule_GetDef(mod) == NULL - || _PyModule_GetDef(mod) == def); + assert(_PyModule_GetDefOrNull(mod) == NULL + || _PyModule_GetDefOrNull(mod) == def); } else { assert(cached->m_dict == NULL); @@ -1967,6 +1983,43 @@ import_find_extension(PyThreadState *tstate, return mod; } +static PyObject * +import_run_modexport(PyThreadState *tstate, PyModExportFunction ex0, + struct _Py_ext_module_loader_info *info, + PyObject *spec) +{ + /* This is like import_run_extension, but avoids interpreter switching + * and code for for single-phase modules. + */ + PyModuleDef_Slot *slots = ex0(spec); + if (!slots) { + if (!PyErr_Occurred()) { + PyErr_Format( + PyExc_SystemError, + "slot export function for module %s failed without setting an exception", + info->name); + } + return NULL; + } + if (PyErr_Occurred()) { + PyErr_Format( + PyExc_SystemError, + "slot export function for module %s raised unreported exception", + info->name); + } + PyObject *result = PyModule_FromSlotsAndSpec(slots, spec); + if (!result) { + return NULL; + } + if (PyModule_Check(result)) { + PyModuleObject *mod = (PyModuleObject *)result; + if (mod && !mod->md_token) { + mod->md_token = slots; + } + } + return result; +} + static PyObject * import_run_extension(PyThreadState *tstate, PyModInitFunction p0, struct _Py_ext_module_loader_info *info, @@ -2253,9 +2306,11 @@ _PyImport_FixupBuiltin(PyThreadState *tstate, PyObject *mod, const char *name, return -1; } - PyModuleDef *def = PyModule_GetDef(mod); + PyModuleDef *def = _PyModule_GetDefOrNull(mod); if (def == NULL) { - PyErr_BadInternalCall(); + if (!PyErr_Occurred()) { + PyErr_BadInternalCall(); + } goto finally; } @@ -2336,8 +2391,8 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) assert(!_PyErr_Occurred(tstate)); assert(cached != NULL); /* The module might not have md_def set in certain reload cases. */ - assert(_PyModule_GetDef(mod) == NULL - || cached->def == _PyModule_GetDef(mod)); + assert(_PyModule_GetDefOrNull(mod) == NULL + || cached->def == _PyModule_GetDefOrNull(mod)); assert_singlephase(cached); goto finally; } @@ -4689,8 +4744,8 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) assert(!_PyErr_Occurred(tstate)); assert(cached != NULL); /* The module might not have md_def set in certain reload cases. */ - assert(_PyModule_GetDef(mod) == NULL - || cached->def == _PyModule_GetDef(mod)); + assert(_PyModule_GetDefOrNull(mod) == NULL + || cached->def == _PyModule_GetDefOrNull(mod)); assert_singlephase(cached); goto finally; } @@ -4728,7 +4783,13 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) fp = NULL; } - PyModInitFunction p0 = _PyImport_GetModInitFunc(&info, fp); + PyModInitFunction p0 = NULL; + PyModExportFunction ex0 = NULL; + _PyImport_GetModInitFunc2(&info, fp, &p0, &ex0); + if (ex0) { + mod = import_run_modexport(tstate, ex0, &info, spec); + goto cleanup; + } if (p0 == NULL) { goto finally; } @@ -4750,6 +4811,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) } #endif +cleanup: // XXX Shouldn't this happen in the error cases too (i.e. in "finally")? if (fp) { fclose(fp); diff --git a/Python/importdl.c b/Python/importdl.c index 802843fe7b9dce..02f50c0c970708 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -5,7 +5,7 @@ #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_import.h" // _PyImport_SwapPackageContext() #include "pycore_importdl.h" -#include "pycore_moduleobject.h" // _PyModule_GetDef() +#include "pycore_moduleobject.h" // _PyModule_GetDefOrNull() #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_runtime.h" // _Py_ID() @@ -35,8 +35,10 @@ extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, /* module info to use when loading */ /***********************************/ -static const char * const ascii_only_prefix = "PyInit"; -static const char * const nonascii_prefix = "PyInitU"; +static const struct hook_prefixes ascii_only_prefixes = { + "PyInit", "PyModExport"}; +static const struct hook_prefixes nonascii_prefixes = { + "PyInitU", "PyModExportU"}; /* Get the variable part of a module's export symbol name. * Returns a bytes instance. For non-ASCII-named modules, the name is @@ -45,7 +47,7 @@ static const char * const nonascii_prefix = "PyInitU"; * nonascii_prefix, as appropriate. */ static PyObject * -get_encoded_name(PyObject *name, const char **hook_prefix) { +get_encoded_name(PyObject *name, const struct hook_prefixes **hook_prefixes) { PyObject *tmp; PyObject *encoded = NULL; PyObject *modname = NULL; @@ -72,7 +74,7 @@ get_encoded_name(PyObject *name, const char **hook_prefix) { /* Encode to ASCII or Punycode, as needed */ encoded = PyUnicode_AsEncodedString(name, "ascii", NULL); if (encoded != NULL) { - *hook_prefix = ascii_only_prefix; + *hook_prefixes = &ascii_only_prefixes; } else { if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) { PyErr_Clear(); @@ -80,7 +82,7 @@ get_encoded_name(PyObject *name, const char **hook_prefix) { if (encoded == NULL) { goto error; } - *hook_prefix = nonascii_prefix; + *hook_prefixes = &nonascii_prefixes; } else { goto error; } @@ -130,7 +132,7 @@ _Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info, assert(PyUnicode_GetLength(name) > 0); info.name = Py_NewRef(name); - info.name_encoded = get_encoded_name(info.name, &info.hook_prefix); + info.name_encoded = get_encoded_name(info.name, &info.hook_prefixes); if (info.name_encoded == NULL) { _Py_ext_module_loader_info_clear(&info); return -1; @@ -189,7 +191,7 @@ _Py_ext_module_loader_info_init_for_builtin( /* We won't need filename. */ .path=name, .origin=_Py_ext_module_origin_BUILTIN, - .hook_prefix=ascii_only_prefix, + .hook_prefixes=&ascii_only_prefixes, .newcontext=NULL, }; return 0; @@ -377,39 +379,62 @@ _Py_ext_module_loader_result_apply_error( /********************************************/ #ifdef HAVE_DYNAMIC_LOADING -PyModInitFunction -_PyImport_GetModInitFunc(struct _Py_ext_module_loader_info *info, - FILE *fp) +static dl_funcptr +findfuncptr(const char *prefix, const char *name_buf, + struct _Py_ext_module_loader_info *info, + FILE *fp) { - const char *name_buf = PyBytes_AS_STRING(info->name_encoded); - dl_funcptr exportfunc; #ifdef MS_WINDOWS - exportfunc = _PyImport_FindSharedFuncptrWindows( - info->hook_prefix, name_buf, info->filename, fp); + return _PyImport_FindSharedFuncptrWindows( + prefix, name_buf, info->filename, fp); #else - { - const char *path_buf = PyBytes_AS_STRING(info->filename_encoded); - exportfunc = _PyImport_FindSharedFuncptr( - info->hook_prefix, name_buf, path_buf, fp); - } + const char *path_buf = PyBytes_AS_STRING(info->filename_encoded); + return _PyImport_FindSharedFuncptr( + prefix, name_buf, path_buf, fp); #endif +} - if (exportfunc == NULL) { - if (!PyErr_Occurred()) { - PyObject *msg; - msg = PyUnicode_FromFormat( - "dynamic module does not define " - "module export function (%s_%s)", - info->hook_prefix, name_buf); - if (msg != NULL) { - PyErr_SetImportError(msg, info->name, info->filename); - Py_DECREF(msg); - } - } - return NULL; +int +_PyImport_GetModInitFunc2(struct _Py_ext_module_loader_info *info, + FILE *fp, + PyModInitFunction *modinit, + PyModExportFunction *modexport) +{ + *modinit = NULL; + *modexport = NULL; + + const char *name_buf = PyBytes_AS_STRING(info->name_encoded); + dl_funcptr exportfunc; + + exportfunc = findfuncptr( + info->hook_prefixes->export_prefix, + name_buf, info, fp); + if (exportfunc) { + *modexport = (PyModExportFunction)exportfunc; + return 2; + } + + exportfunc = findfuncptr( + info->hook_prefixes->init_prefix, + name_buf, info, fp); + if (exportfunc) { + *modinit = (PyModInitFunction)exportfunc; + return 1; } - return (PyModInitFunction)exportfunc; + if (!PyErr_Occurred()) { + PyObject *msg; + msg = PyUnicode_FromFormat( + "dynamic module does not define " + "module export function (%s_%s or %s_%s)", + info->hook_prefixes->export_prefix, name_buf, + info->hook_prefixes->init_prefix, name_buf); + if (msg != NULL) { + PyErr_SetImportError(msg, info->name, info->filename); + Py_DECREF(msg); + } + } + return -1; } #endif /* HAVE_DYNAMIC_LOADING */ @@ -477,7 +502,7 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, res.def = (PyModuleDef *)m; /* Run PyModule_FromDefAndSpec() to finish loading the module. */ } - else if (info->hook_prefix == nonascii_prefix) { + else if (info->hook_prefixes == &nonascii_prefixes) { /* Non-ASCII is only supported for multi-phase init. */ res.kind = _Py_ext_module_kind_MULTIPHASE; /* Don't allow legacy init for non-ASCII module names. */ @@ -496,7 +521,7 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, goto error; } - res.def = _PyModule_GetDef(m); + res.def = _PyModule_GetDefOrNull(m); if (res.def == NULL) { PyErr_Clear(); _Py_ext_module_loader_result_set_error(