From 0858c0c9ff6d99aa215a952ca0a4c977e3ce358f Mon Sep 17 00:00:00 2001 From: Tobias Bergkvist Date: Tue, 20 Jul 2021 02:45:45 +0200 Subject: [PATCH 1/7] Add _ctypes.shared_library_is_loadable. Remove _ctypes._dyld_shared_cache_contains_path Fix tests --- Lib/ctypes/macholib/dyld.py | 26 +++++++--------- Modules/_ctypes/callproc.c | 60 ++++++++++++++----------------------- 2 files changed, 34 insertions(+), 52 deletions(-) diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py index 1c3f8fd38b0665..ab0aaf6e9323ab 100644 --- a/Lib/ctypes/macholib/dyld.py +++ b/Lib/ctypes/macholib/dyld.py @@ -6,11 +6,7 @@ from ctypes.macholib.framework import framework_info from ctypes.macholib.dylib import dylib_info from itertools import * -try: - from _ctypes import _dyld_shared_cache_contains_path -except ImportError: - def _dyld_shared_cache_contains_path(*args): - raise NotImplementedError +from _ctypes import shared_library_is_loadable __all__ = [ 'dyld_find', 'framework_find', @@ -122,19 +118,19 @@ def dyld_find(name, executable_path=None, env=None): """ Find a library or framework using dyld semantics """ - for path in dyld_image_suffix_search(chain( - dyld_override_search(name, env), - dyld_executable_path_search(name, executable_path), - dyld_default_search(name, env), - ), env): + paths = list(dyld_image_suffix_search(chain( + dyld_override_search(name, env), + dyld_executable_path_search(name, executable_path), + dyld_default_search(name, env), + ), env)) + for path in paths: if os.path.isfile(path): return path - try: - if _dyld_shared_cache_contains_path(path): - return path - except NotImplementedError: - pass + + for path in reversed(paths): + if shared_library_is_loadable(path): + return path raise ValueError("dylib %s could not be found" % (name,)) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 56ccc2f1e0b5da..eab1fff0ff5d04 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -66,10 +66,6 @@ #include "ctypes_dlfcn.h" #endif -#ifdef __APPLE__ -#include -#endif - #ifdef MS_WIN32 #include #endif @@ -1443,40 +1439,31 @@ copy_com_pointer(PyObject *self, PyObject *args) } #else -#ifdef HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH -static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *args) -{ - PyObject *name, *name2; - char *name_str; - - if (__builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *)) { - int r; - - if (!PyArg_ParseTuple(args, "O", &name)) - return NULL; - - if (name == Py_None) - Py_RETURN_FALSE; - - if (PyUnicode_FSConverter(name, &name2) == 0) - return NULL; - name_str = PyBytes_AS_STRING(name2); +#ifndef MS_WIN32 +static PyObject *py_shared_library_is_loadable(PyObject *self, PyObject *args) { + PyObject *name, *name2; + char *name_str; + void* handle; - r = _dyld_shared_cache_contains_path(name_str); - Py_DECREF(name2); + if (!PyArg_ParseTuple(args, "O", &name)) + return NULL; - if (r) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } + if (name == Py_None) + Py_RETURN_FALSE; - } else { - PyErr_SetString(PyExc_NotImplementedError, "_dyld_shared_cache_contains_path symbol is missing"); - return NULL; - } + if (PyUnicode_FSConverter(name, &name2) == 0) + return NULL; - } + name_str = PyBytes_AS_STRING(name2); + handle = dlopen(name_str, RTLD_LAZY); + Py_DECREF(name2); + if (handle == NULL) { + Py_RETURN_FALSE; + } else { + dlclose(handle); + Py_RETURN_TRUE; + } +} #endif static PyObject *py_dl_open(PyObject *self, PyObject *args) @@ -1991,9 +1978,8 @@ PyMethodDef _ctypes_module_methods[] = { "dlopen(name, flag={RTLD_GLOBAL|RTLD_LOCAL}) open a shared library"}, {"dlclose", py_dl_close, METH_VARARGS, "dlclose a library"}, {"dlsym", py_dl_sym, METH_VARARGS, "find symbol in shared library"}, -#endif -#ifdef HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH - {"_dyld_shared_cache_contains_path", py_dyld_shared_cache_contains_path, METH_VARARGS, "check if path is in the shared cache"}, + {"shared_library_is_loadable", py_shared_library_is_loadable, METH_VARARGS, + "check if shared library exists/is loadable with dlopen"}, #endif {"alignment", align_func, METH_O, alignment_doc}, {"sizeof", sizeof_func, METH_O, sizeof_doc}, From b7aa289e4e2b4ceb364bed9129f91276511719d1 Mon Sep 17 00:00:00 2001 From: Tobias Bergkvist Date: Tue, 20 Jul 2021 03:30:21 +0200 Subject: [PATCH 2/7] Fix test_ctypes for Windows. --- Lib/ctypes/macholib/dyld.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py index ab0aaf6e9323ab..f6d53dfdc6463e 100644 --- a/Lib/ctypes/macholib/dyld.py +++ b/Lib/ctypes/macholib/dyld.py @@ -6,7 +6,12 @@ from ctypes.macholib.framework import framework_info from ctypes.macholib.dylib import dylib_info from itertools import * -from _ctypes import shared_library_is_loadable + +try: + from _ctypes import shared_library_is_loadable +except ImportError: + def shared_library_is_loadable(path): + raise NotImplementedError __all__ = [ 'dyld_find', 'framework_find', From 00388a25ddf3c0391a6643958c1c746db1cf93b8 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 20 Jul 2021 22:27:02 +0000 Subject: [PATCH 3/7] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst diff --git a/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst b/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst new file mode 100644 index 00000000000000..5aeeab781696d9 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst @@ -0,0 +1 @@ +You no longer need to compile Python on the same version of MacOS that you will be running it on (Catalina vs Big Sur) in order for ctypes.util.find_library to work. \ No newline at end of file From 9d3c83474efb5a78c6452b1a812c8d72ec8771fa Mon Sep 17 00:00:00 2001 From: Tobias Bergkvist Date: Wed, 28 Jul 2021 01:11:23 +0200 Subject: [PATCH 4/7] bpo-44689: Remove dlopen in find_library. Load _dyld_shared_cache_contains_path at runtime instead of compile time. (GH-27251) --- Lib/ctypes/macholib/dyld.py | 20 +++++-------- Modules/_ctypes/callproc.c | 58 +++++++++++++++++++++++++++---------- 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py index f6d53dfdc6463e..5daf75fb88f67d 100644 --- a/Lib/ctypes/macholib/dyld.py +++ b/Lib/ctypes/macholib/dyld.py @@ -6,11 +6,10 @@ from ctypes.macholib.framework import framework_info from ctypes.macholib.dylib import dylib_info from itertools import * - try: - from _ctypes import shared_library_is_loadable + from _ctypes import _dyld_shared_cache_contains_path except ImportError: - def shared_library_is_loadable(path): + def _dyld_shared_cache_contains_path(*args): raise NotImplementedError __all__ = [ @@ -123,18 +122,15 @@ def dyld_find(name, executable_path=None, env=None): """ Find a library or framework using dyld semantics """ - paths = list(dyld_image_suffix_search(chain( - dyld_override_search(name, env), - dyld_executable_path_search(name, executable_path), - dyld_default_search(name, env), - ), env)) + for path in dyld_image_suffix_search(chain( + dyld_override_search(name, env), + dyld_executable_path_search(name, executable_path), + dyld_default_search(name, env), + ), env): - for path in paths: if os.path.isfile(path): return path - - for path in reversed(paths): - if shared_library_is_loadable(path): + if _dyld_shared_cache_contains_path(path): return path raise ValueError("dylib %s could not be found" % (name,)) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index eab1fff0ff5d04..97b2fe600c0c6e 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1439,30 +1439,57 @@ copy_com_pointer(PyObject *self, PyObject *args) } #else -#ifndef MS_WIN32 -static PyObject *py_shared_library_is_loadable(PyObject *self, PyObject *args) { +#ifdef __APPLE__ +static void* libsystem_b_handle; +typedef bool (*_dyld_shared_cache_contains_path_f)(const char* path); +static _dyld_shared_cache_contains_path_f _dyld_shared_cache_contains_path; +static bool _dyld_shared_cache_contains_path_fallback(const char* path) { + return false; +} + +__attribute__((constructor)) void load_libsystemb(void) { + libsystem_b_handle = dlopen("/usr/lib/libSystem.B.dylib", RTLD_LAZY); + if (libsystem_b_handle != NULL) { + _dyld_shared_cache_contains_path = dlsym(libsystem_b_handle, "_dyld_shared_cache_contains_path"); + } + + if (_dyld_shared_cache_contains_path == NULL) { + _dyld_shared_cache_contains_path = _dyld_shared_cache_contains_path_fallback; + } +} + +__attribute__((destructor)) void unload_libsystemb(void) { + if (libsystem_b_handle != NULL) { + dlclose(libsystem_b_handle); + } +} + +static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *args) { PyObject *name, *name2; - char *name_str; - void* handle; - if (!PyArg_ParseTuple(args, "O", &name)) + if (!PyArg_ParseTuple(args, "O", &name)) { return NULL; + } - if (name == Py_None) + if (name == Py_None) { Py_RETURN_FALSE; + } - if (PyUnicode_FSConverter(name, &name2) == 0) + if (PyUnicode_FSConverter(name, &name2) == 0) { return NULL; + } + + char *name_str = PyBytes_AS_STRING(name2); + int r = _dyld_shared_cache_contains_path(name_str); - name_str = PyBytes_AS_STRING(name2); - handle = dlopen(name_str, RTLD_LAZY); Py_DECREF(name2); - if (handle == NULL) { - Py_RETURN_FALSE; - } else { - dlclose(handle); + + if (r) { Py_RETURN_TRUE; } + else { + Py_RETURN_FALSE; + } } #endif @@ -1978,8 +2005,9 @@ PyMethodDef _ctypes_module_methods[] = { "dlopen(name, flag={RTLD_GLOBAL|RTLD_LOCAL}) open a shared library"}, {"dlclose", py_dl_close, METH_VARARGS, "dlclose a library"}, {"dlsym", py_dl_sym, METH_VARARGS, "find symbol in shared library"}, - {"shared_library_is_loadable", py_shared_library_is_loadable, METH_VARARGS, - "check if shared library exists/is loadable with dlopen"}, +#endif +#ifdef __APPLE__ + {"_dyld_shared_cache_contains_path", py_dyld_shared_cache_contains_path, METH_VARARGS, "check if path is in the shared cache"}, #endif {"alignment", align_func, METH_O, alignment_doc}, {"sizeof", sizeof_func, METH_O, sizeof_doc}, From 01bcf2bef5f4ffffb454da35cb66b186a7a12598 Mon Sep 17 00:00:00 2001 From: Tobias Bergkvist Date: Wed, 28 Jul 2021 01:13:53 +0200 Subject: [PATCH 5/7] Update Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst Co-authored-by: Erlend Egeberg Aasland --- .../next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst b/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst index 5aeeab781696d9..9c56f9a38ddddc 100644 --- a/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst +++ b/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst @@ -1 +1,3 @@ -You no longer need to compile Python on the same version of MacOS that you will be running it on (Catalina vs Big Sur) in order for ctypes.util.find_library to work. \ No newline at end of file +You no longer need to compile Python on the same version of macOS that you +will be running it on (Catalina vs Big Sur) in order for +:meth:`ctypes.util.find_library` to work. From 52473a484b2f0bfc6776dc7b483c0a253fb1f996 Mon Sep 17 00:00:00 2001 From: Tobias Bergkvist Date: Fri, 6 Aug 2021 12:55:55 +0200 Subject: [PATCH 6/7] bpo-44689: Revert to using weak linking implementation. Only use dynamic loading when compiling on MacOS < 11 --- Lib/ctypes/macholib/dyld.py | 7 +- .../2021-07-20-22-27-01.bpo-44689.mmT_xH.rst | 5 +- Modules/_ctypes/callproc.c | 77 +++++++++++-------- 3 files changed, 50 insertions(+), 39 deletions(-) diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py index 5daf75fb88f67d..1c3f8fd38b0665 100644 --- a/Lib/ctypes/macholib/dyld.py +++ b/Lib/ctypes/macholib/dyld.py @@ -130,8 +130,11 @@ def dyld_find(name, executable_path=None, env=None): if os.path.isfile(path): return path - if _dyld_shared_cache_contains_path(path): - return path + try: + if _dyld_shared_cache_contains_path(path): + return path + except NotImplementedError: + pass raise ValueError("dylib %s could not be found" % (name,)) diff --git a/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst b/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst index 9c56f9a38ddddc..7ae03d6bd1a715 100644 --- a/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst +++ b/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst @@ -1,3 +1,2 @@ -You no longer need to compile Python on the same version of macOS that you -will be running it on (Catalina vs Big Sur) in order for -:meth:`ctypes.util.find_library` to work. +You can now compile Python on MacOS < 11.0 and +:meth:`ctypes.util.find_library` will still work on MacOS >= 11.0. diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 97b2fe600c0c6e..89f82632ffd98d 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -66,6 +66,10 @@ #include "ctypes_dlfcn.h" #endif +#ifdef __APPLE__ +#include +#endif + #ifdef MS_WIN32 #include #endif @@ -1438,59 +1442,64 @@ copy_com_pointer(PyObject *self, PyObject *args) return r; } #else - #ifdef __APPLE__ -static void* libsystem_b_handle; -typedef bool (*_dyld_shared_cache_contains_path_f)(const char* path); -static _dyld_shared_cache_contains_path_f _dyld_shared_cache_contains_path; -static bool _dyld_shared_cache_contains_path_fallback(const char* path) { - return false; -} +#ifdef HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH +#define HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH_RUNTIME \ + __builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) +#else +// Support the deprecated case of compiling on an older MacOS version +static void *libsystem_b_handle; +static bool (*_dyld_shared_cache_contains_path)(const char *path); -__attribute__((constructor)) void load_libsystemb(void) { +__attribute__((constructor)) void load_dyld_shared_cache_contains_path(void) { libsystem_b_handle = dlopen("/usr/lib/libSystem.B.dylib", RTLD_LAZY); if (libsystem_b_handle != NULL) { _dyld_shared_cache_contains_path = dlsym(libsystem_b_handle, "_dyld_shared_cache_contains_path"); } - - if (_dyld_shared_cache_contains_path == NULL) { - _dyld_shared_cache_contains_path = _dyld_shared_cache_contains_path_fallback; - } } -__attribute__((destructor)) void unload_libsystemb(void) { +__attribute__((destructor)) void unload_dyld_shared_cache_contains_path(void) { if (libsystem_b_handle != NULL) { dlclose(libsystem_b_handle); } } +#define HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH_RUNTIME \ + _dyld_shared_cache_contains_path != NULL +#endif -static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *args) { - PyObject *name, *name2; +static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *args) +{ + PyObject *name, *name2; + char *name_str; - if (!PyArg_ParseTuple(args, "O", &name)) { - return NULL; - } + if (HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH_RUNTIME) { + int r; - if (name == Py_None) { - Py_RETURN_FALSE; - } + if (!PyArg_ParseTuple(args, "O", &name)) + return NULL; - if (PyUnicode_FSConverter(name, &name2) == 0) { - return NULL; - } + if (name == Py_None) + Py_RETURN_FALSE; - char *name_str = PyBytes_AS_STRING(name2); - int r = _dyld_shared_cache_contains_path(name_str); + if (PyUnicode_FSConverter(name, &name2) == 0) + return NULL; + name_str = PyBytes_AS_STRING(name2); - Py_DECREF(name2); + r = _dyld_shared_cache_contains_path(name_str); + Py_DECREF(name2); - if (r) { - Py_RETURN_TRUE; - } - else { - Py_RETURN_FALSE; - } -} + if (r) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + + } else { + PyErr_SetString(PyExc_NotImplementedError, "_dyld_shared_cache_contains_path symbol is missing"); + return NULL; + } + + } #endif static PyObject *py_dl_open(PyObject *self, PyObject *args) From dfcc35789e97e94a8ae04650ef39fc08f33ecc7d Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Mon, 30 Aug 2021 02:21:15 -0400 Subject: [PATCH 7/7] bpo-44689: edit NEWS blurb and comment --- .../next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst | 7 +++++-- Modules/_ctypes/callproc.c | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst b/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst index 7ae03d6bd1a715..b1e878d1ee44af 100644 --- a/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst +++ b/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst @@ -1,2 +1,5 @@ -You can now compile Python on MacOS < 11.0 and -:meth:`ctypes.util.find_library` will still work on MacOS >= 11.0. + :meth:`ctypes.util.find_library` now works correctly on macOS 11 Big Sur + even if Python is built on an older version of macOS. Previously, when + built on older macOS systems, ``find_library`` was not able to find + macOS system libraries when running on Big Sur due to changes in + how system libraries are stored. diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 89f82632ffd98d..f8f8efa4ee8793 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1447,7 +1447,7 @@ copy_com_pointer(PyObject *self, PyObject *args) #define HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH_RUNTIME \ __builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) #else -// Support the deprecated case of compiling on an older MacOS version +// Support the deprecated case of compiling on an older macOS version static void *libsystem_b_handle; static bool (*_dyld_shared_cache_contains_path)(const char *path);