From fc54fcb490684c4c3a9daab19db26641624fe18b Mon Sep 17 00:00:00 2001 From: Lawrence D'Anna Date: Tue, 30 Jun 2020 11:11:08 -0700 Subject: [PATCH] ctypes: use the correct ABI for variadic functions On arm64 the calling convention for variardic functions is different than the convention for fixed-arg functions of the same arg types. ctypes needs to use ffi_prep_cif_var to tell libffi which calling convention to use. This patch adds a new attribute "f.variadic" for ctypes function pointers, so users can specify which functions are variadic. It will also auto-detect varargs when a function is called with more arguments than f.argtypes specifies. Since this is a new option and it only matters on arm64 on apple platforms, lots of existing code will not set f.variadic. This will do the right thing in most situations. --- Doc/library/ctypes.rst | 7 +++ Lib/test/test_bytes.py | 2 + Lib/test/test_unicode.py | 3 ++ .../2020-06-30-18-26-26.bpo-41100.rcZuB5.rst | 1 + Modules/_ctypes/_ctypes.c | 32 ++++++++++++ Modules/_ctypes/callproc.c | 52 ++++++++++++++----- Modules/_ctypes/ctypes.h | 1 + 7 files changed, 86 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-06-30-18-26-26.bpo-41100.rcZuB5.rst diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 2d6c6d0a1c3c57..caed17a89d4aaf 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1579,6 +1579,13 @@ They are instances of a private class: value usable as argument (integer, string, ctypes instance). This allows defining adapters that can adapt custom objects as function parameters. + .. attribute:: variadic + + Assign a boolean to specify that the function takes a variable nubmer of + arguments. This does not matter on most platforms, but for Apple arm64 + platforms variadic functions have a different calling convention than + normal functions. + .. attribute:: errcheck Assign a Python function or another callable to this attribute. The diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 770e2c5592cc61..bc65b2d923a269 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1034,6 +1034,8 @@ def test_from_format(self): c_char_p) PyBytes_FromFormat = pythonapi.PyBytes_FromFormat + PyBytes_FromFormat.variadic = True + PyBytes_FromFormat.argtypes = (c_char_p,) PyBytes_FromFormat.restype = py_object # basic tests diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 59697935fe5cd8..5f3945152a25fb 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2509,11 +2509,14 @@ class CAPITest(unittest.TestCase): def test_from_format(self): support.import_module('ctypes') from ctypes import ( + c_char_p, pythonapi, py_object, sizeof, c_int, c_long, c_longlong, c_ssize_t, c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) name = "PyUnicode_FromFormat" _PyUnicode_FromFormat = getattr(pythonapi, name) + _PyUnicode_FromFormat.argtypes = (c_char_p,) + _PyUnicode_FromFormat.variadic = True _PyUnicode_FromFormat.restype = py_object def PyUnicode_FromFormat(format, *args): diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-06-30-18-26-26.bpo-41100.rcZuB5.rst b/Misc/NEWS.d/next/Core and Builtins/2020-06-30-18-26-26.bpo-41100.rcZuB5.rst new file mode 100644 index 00000000000000..646e5620e94cdc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-06-30-18-26-26.bpo-41100.rcZuB5.rst @@ -0,0 +1 @@ +ctypes: Mac OS 11: use correct ABI for variadic functions on arm64 \ No newline at end of file diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index ceae67ebb16127..2105631d9dae81 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3320,6 +3320,35 @@ PyCFuncPtr_get_restype(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) } } +static int +PyCFuncPtr_set_variadic(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored)) +{ + StgDictObject *dict = PyObject_stgdict((PyObject *)self); + assert(dict); + int r = PyObject_IsTrue(ob); + if (r == 1) { + dict->flags |= FUNCFLAG_VARIADIC; + return 0; + } else if (r == 0) { + dict->flags &= ~FUNCFLAG_VARIADIC; + return 0; + } else { + return -1; + } +} + +static PyObject * +PyCFuncPtr_get_variadic(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) +{ + StgDictObject *dict = PyObject_stgdict((PyObject *)self); + assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */ + if (dict->flags & FUNCFLAG_VARIADIC) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; +} + + static int PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored)) { @@ -3365,6 +3394,8 @@ static PyGetSetDef PyCFuncPtr_getsets[] = { { "argtypes", (getter)PyCFuncPtr_get_argtypes, (setter)PyCFuncPtr_set_argtypes, "specify the argument types", NULL }, + { "variadic", (getter)PyCFuncPtr_get_variadic, (setter)PyCFuncPtr_set_variadic, + "specify if function takes variable number of arguments", NULL }, { NULL, NULL } }; @@ -5839,6 +5870,7 @@ PyInit__ctypes(void) PyModule_AddObject(m, "FUNCFLAG_USE_ERRNO", PyLong_FromLong(FUNCFLAG_USE_ERRNO)); PyModule_AddObject(m, "FUNCFLAG_USE_LASTERROR", PyLong_FromLong(FUNCFLAG_USE_LASTERROR)); PyModule_AddObject(m, "FUNCFLAG_PYTHONAPI", PyLong_FromLong(FUNCFLAG_PYTHONAPI)); + PyModule_AddObject(m, "FUNCFLAG_VARIADIC", PyLong_FromLong(FUNCFLAG_VARIADIC)); PyModule_AddStringConstant(m, "__version__", "1.1.0"); PyModule_AddObject(m, "_memmove_addr", PyLong_FromVoidPtr(memmove)); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 6030cc3d43670d..f944fd91bdbf98 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -86,6 +86,10 @@ #define DONT_USE_SEH #endif +#if defined(__APPLE__) && __arm64__ +#define HAVE_FFI_PREP_CIF_VAR 1 +#endif + #define CTYPES_CAPSULE_NAME_PYMEM "_ctypes pymem" static void pymem_destructor(PyObject *ptr) @@ -812,7 +816,8 @@ static int _call_function_pointer(int flags, ffi_type **atypes, ffi_type *restype, void *resmem, - int argcount) + int argcount, + int argtypecount) { PyThreadState *_save = NULL; /* For Py_BLOCK_THREADS and Py_UNBLOCK_THREADS */ PyObject *error_object = NULL; @@ -835,15 +840,39 @@ static int _call_function_pointer(int flags, if ((flags & FUNCFLAG_CDECL) == 0) cc = FFI_STDCALL; #endif - if (FFI_OK != ffi_prep_cif(&cif, - cc, - argcount, - restype, - atypes)) { - PyErr_SetString(PyExc_RuntimeError, - "ffi_prep_cif failed"); - return -1; + +#if HAVE_FFI_PREP_CIF_VAR + /* Everyone SHOULD set f.variadic=True on variadic function pointers, but + * lots of existing code will not. If there's at least one arg and more + * args are passed than are defined in the prototype, then it must be a + * variadic function. */ + if ((flags & FUNCFLAG_VARIADIC) || + (argtypecount != 0 && argcount > argtypecount)) + { + if (FFI_OK != ffi_prep_cif_var(&cif, + cc, + argtypecount, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif_var failed"); + return -1; + } + } else { +#endif + if (FFI_OK != ffi_prep_cif(&cif, + cc, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif failed"); + return -1; + } +#if HAVE_FFI_PREP_CIF_VAR } +#endif if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { error_object = _ctypes_get_errobj(&space); @@ -1212,9 +1241,8 @@ PyObject *_ctypes_callproc(PPROC pProc, if (-1 == _call_function_pointer(flags, pProc, avalues, atypes, rtype, resbuf, - Py_SAFE_DOWNCAST(argcount, - Py_ssize_t, - int))) + Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int), + Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int))) goto cleanup; #ifdef WORDS_BIGENDIAN diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 1effccf9cc5ff9..c9015baeb287ed 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -285,6 +285,7 @@ PyObject *_ctypes_callproc(PPROC pProc, #define FUNCFLAG_PYTHONAPI 0x4 #define FUNCFLAG_USE_ERRNO 0x8 #define FUNCFLAG_USE_LASTERROR 0x10 +#define FUNCFLAG_VARIADIC 0x20 #define TYPEFLAG_ISPOINTER 0x100 #define TYPEFLAG_HASPOINTER 0x200