From e907723be7b723fabc1324995ca8836f165ec20e Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Thu, 7 Mar 2024 16:06:09 +0800 Subject: [PATCH 01/10] Add iOS framework loading machinery. --- Lib/importlib/_bootstrap_external.py | 77 +++++++++++++++- Lib/importlib/machinery.py | 2 + Lib/test/test_import/__init__.py | 34 ++++++-- .../test_importlib/extension/test_finder.py | 24 ++++- .../test_importlib/extension/test_loader.py | 87 +++++++++++++++---- Lib/test/test_importlib/util.py | 16 +++- ...-03-07-16-12-39.gh-issue-114099.ujdjn2.rst | 2 + Python/dynload_shlib.c | 26 ++++-- 8 files changed, 228 insertions(+), 40 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 46ddceed07b0d4..a7cff3eef03f25 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -52,7 +52,7 @@ # Bootstrap-related code ###################################################### _CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win', -_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin' +_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin', 'ios', 'tvos', 'watchos' _CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY + _CASE_INSENSITIVE_PLATFORMS_STR_KEY) @@ -1711,6 +1711,77 @@ def __repr__(self): return f'FileFinder({self.path!r})' +class AppleFrameworkLoader(ExtensionFileLoader): + """A loader for modules that have been packaged as frameworks for + compatibility with Apple's App Store policies. + + For compatibility with the App Store, *all* binary modules must be .dylib + objects, contained in a framework, stored in the ``Frameworks`` folder of + the packaged app. There can be only a single binary per framework, and + there can be no executable binary material outside the Frameworks folder. + + If you're trying to run ``from foo.bar import _whiz``, and ``_whiz`` is + implemented with the binary module ``foo/bar/_whiz.abi3.dylib`` (or any + other ABI .dylib extension), this loader will look for + ``{sys.executable}/Frameworks/foo.bar._whiz.framework/_whiz.abi3.dylib`` + (forming the package name by taking the full import path of the library, + and replacing ``/`` with ``.``). + + However, the ``__file__`` attribute of the ``_whiz`` module will report as + the original location inside the ``foo/bar`` subdirectory. This so that + code that depends on walking directory trees will continue to work as + expected based on the *original* file location. + + The Xcode project building the app is responsible for converting any + ``.dylib`` files from wherever they exist in the ``PYTHONPATH`` into + frameworks in the ``Frameworks`` folder (including the addition of + framework metadata, and signing the resulting framework). This will usually + be done with a build step in the Xcode project; see the iOS documentation + for details on how to construct this build step. + """ + def __init__(self, fullname, dylib_file, path=None): + super().__init__(fullname, dylib_file) + self.parent_paths = path + + def create_module(self, spec): + mod = super().create_module(spec) + if self.parent_paths: + for parent_path in self.parent_paths: + if _path_isdir(parent_path): + mod.__file__ = _path_join( + parent_path, + _path_split(self.path)[-1], + ) + continue + return mod + + +class AppleFrameworkFinder: + """A finder for modules that have been packaged as Apple Frameworks + for compatibility with Apple's App Store policies. + + See AppleFrameworkLoader for details. + """ + def __init__(self, path): + self.frameworks_path = path + + def find_spec(self, fullname, path, target=None): + name = fullname.split(".")[-1] + + for extension in EXTENSION_SUFFIXES: + dylib_file = _path_join(self.frameworks_path, f"{fullname}.framework", f"{name}{extension}") + _bootstrap._verbose_message("Looking for Apple Framework dylib {}", dylib_file) + try: + dylib_exists = _path_isfile(dylib_file) + except ValueError: + pass + else: + if dylib_exists: + loader = AppleFrameworkLoader(fullname, dylib_file, path) + return _bootstrap.spec_from_loader(fullname, loader) + + return None + # Import setup ############################################################### def _fix_up_module(ns, name, pathname, cpathname=None): @@ -1760,3 +1831,7 @@ def _install(_bootstrap_module): supported_loaders = _get_supported_file_loaders() sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)]) sys.meta_path.append(PathFinder) + if sys.platform in {"ios", "tvos", "watchos"}: + frameworks_folder = _path_join(_path_split(sys.executable)[0], "Frameworks") + _bootstrap._verbose_message("Adding Apple Framework dylib finder at {}", frameworks_folder) + sys.meta_path.append(AppleFrameworkFinder(frameworks_folder)) diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index d9a19a13f7b275..928ed116a2c9a4 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -12,6 +12,8 @@ from ._bootstrap_external import SourceFileLoader from ._bootstrap_external import SourcelessFileLoader from ._bootstrap_external import ExtensionFileLoader +from ._bootstrap_external import AppleFrameworkLoader +from ._bootstrap_external import AppleFrameworkFinder from ._bootstrap_external import NamespaceLoader diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 7b0126226c4aba..38810f9c7db307 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -5,7 +5,7 @@ import importlib.util from importlib._bootstrap_external import _get_sourcefile from importlib.machinery import ( - BuiltinImporter, ExtensionFileLoader, FrozenImporter, SourceFileLoader, + AppleFrameworkLoader, BuiltinImporter, ExtensionFileLoader, FrozenImporter, SourceFileLoader, ) import marshal import os @@ -25,7 +25,7 @@ from test.support import os_helper from test.support import ( - STDLIB_DIR, swap_attr, swap_item, cpython_only, is_emscripten, + STDLIB_DIR, swap_attr, swap_item, cpython_only, is_apple_mobile, is_emscripten, is_wasi, run_in_subinterp, run_in_subinterp_with_config, Py_TRACE_REFS) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, ready_to_import, @@ -66,6 +66,7 @@ def _require_loader(module, loader, skip): MODULE_KINDS = { BuiltinImporter: 'built-in', ExtensionFileLoader: 'extension', + AppleFrameworkLoader: 'framework extension', FrozenImporter: 'frozen', SourceFileLoader: 'pure Python', } @@ -91,7 +92,10 @@ def require_builtin(module, *, skip=False): assert module.__spec__.origin == 'built-in', module.__spec__ def require_extension(module, *, skip=False): - _require_loader(module, ExtensionFileLoader, skip) + if is_apple_mobile: + _require_loader(module, AppleFrameworkLoader, skip) + else: + _require_loader(module, ExtensionFileLoader, skip) def require_frozen(module, *, skip=True): module = _require_loader(module, FrozenImporter, skip) @@ -360,7 +364,7 @@ def test_from_import_missing_attr_has_name_and_so_path(self): self.assertEqual(cm.exception.path, _testcapi.__file__) self.assertRegex( str(cm.exception), - r"cannot import name 'i_dont_exist' from '_testcapi' \(.*\.(so|pyd)\)" + r"cannot import name 'i_dont_exist' from '_testcapi' \(.*\.(so|dylib|pyd)\)" ) else: self.assertEqual( @@ -1689,6 +1693,12 @@ def pipe(self): os.set_blocking(r, False) return (r, w) + def create_extension_loader(self, modname, filename): + if is_apple_mobile: + return AppleFrameworkLoader(modname, filename, None) + else: + return ExtensionFileLoader(modname, filename) + def import_script(self, name, fd, filename=None, check_override=None): override_text = '' if check_override is not None: @@ -1883,7 +1893,7 @@ def test_multi_init_extension_compat(self): def test_multi_init_extension_non_isolated_compat(self): modname = '_test_non_isolated' filename = _testmultiphase.__file__ - loader = ExtensionFileLoader(modname, filename) + loader = self.create_extension_loader(modname, filename) spec = importlib.util.spec_from_loader(modname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -1901,7 +1911,7 @@ def test_multi_init_extension_non_isolated_compat(self): def test_multi_init_extension_per_interpreter_gil_compat(self): modname = '_test_shared_gil_only' filename = _testmultiphase.__file__ - loader = ExtensionFileLoader(modname, filename) + loader = self.create_extension_loader(modname, filename) spec = importlib.util.spec_from_loader(modname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -2034,10 +2044,13 @@ class SinglephaseInitTests(unittest.TestCase): @classmethod def setUpClass(cls): spec = importlib.util.find_spec(cls.NAME) - from importlib.machinery import ExtensionFileLoader + from importlib.machinery import AppleFrameworkLoader, ExtensionFileLoader cls.FILE = spec.origin cls.LOADER = type(spec.loader) - assert cls.LOADER is ExtensionFileLoader + if is_apple_mobile: + assert cls.LOADER is AppleFrameworkLoader + else: + assert cls.LOADER is ExtensionFileLoader # Start fresh. cls.clean_up() @@ -2077,7 +2090,10 @@ def _load_dynamic(self, name, path): """ # This is essentially copied from the old imp module. from importlib._bootstrap import _load - loader = self.LOADER(name, path) + if is_apple_mobile: + loader = self.LOADER(name, path, None) + else: + loader = self.LOADER(name, path) # Issue bpo-24748: Skip the sys.modules check in _load_module_shim; # always load new extension. diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index 1d5b6e7a5de94b..0bf4412c1c2ffb 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -1,8 +1,10 @@ +from test.support import is_apple_mobile from test.test_importlib import abc, util machinery = util.import_importlib('importlib.machinery') import unittest +import os import sys @@ -19,11 +21,25 @@ def setUp(self): ) def find_spec(self, fullname): - importer = self.machinery.FileFinder(util.EXTENSIONS.path, - (self.machinery.ExtensionFileLoader, - self.machinery.EXTENSION_SUFFIXES)) + if is_apple_mobile: + # Apple extensions must be distributed as frameworks. This requires + # a specialist finder. + frameworks_folder = os.path.join( + os.path.split(sys.executable)[0], "Frameworks" + ) + importer = self.machinery.AppleFrameworkFinder(frameworks_folder) + + return importer.find_spec(fullname, None) + else: + importer = self.machinery.FileFinder( + util.EXTENSIONS.path, + ( + self.machinery.ExtensionFileLoader, + self.machinery.EXTENSION_SUFFIXES + ) + ) - return importer.find_spec(fullname) + return importer.find_spec(fullname) def test_module(self): self.assertTrue(self.find_spec(util.EXTENSIONS.name)) diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 84a0680e4ec653..559dc4725e842e 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -1,3 +1,4 @@ +from test.support import is_apple_mobile from test.test_importlib import abc, util machinery = util.import_importlib('importlib.machinery') @@ -23,8 +24,15 @@ def setUp(self): raise unittest.SkipTest( f"{util.EXTENSIONS.name} is a builtin module" ) - self.loader = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + + self.loader = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path) def load_module(self, fullname): with warnings.catch_warnings(): @@ -32,13 +40,11 @@ def load_module(self, fullname): return self.loader.load_module(fullname) def test_equality(self): - other = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + other = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path) self.assertEqual(self.loader, other) def test_inequality(self): - other = self.machinery.ExtensionFileLoader('_' + util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + other = self.LoaderClass('_' + util.EXTENSIONS.name, util.EXTENSIONS.file_path) self.assertNotEqual(self.loader, other) def test_load_module_API(self): @@ -58,8 +64,7 @@ def test_module(self): ('__package__', '')]: self.assertEqual(getattr(module, attr), value) self.assertIn(util.EXTENSIONS.name, sys.modules) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) # No extension module as __init__ available for testing. test_package = None @@ -86,9 +91,44 @@ def test_is_package(self): self.assertFalse(self.loader.is_package(util.EXTENSIONS.name)) for suffix in self.machinery.EXTENSION_SUFFIXES: path = os.path.join('some', 'path', 'pkg', '__init__' + suffix) - loader = self.machinery.ExtensionFileLoader('pkg', path) + loader = self.LoaderClass('pkg', path) self.assertTrue(loader.is_package('pkg')) + @unittest.skipUnless(is_apple_mobile, "Only required on Apple mobile") + def test_file_origin(self): + # Apple Mobile requires that binary extension modules are moved from + # their "normal" location to the Frameworks folder. Many third-party + # packages assume that __file__ will return somewhere in the + # PYTHONPATH; this won't be true on Apple mobile, so the + # AppleFrameworkLoader rewrites __file__ to point at the original path. + # However, the standard library puts all it's modules in lib-dynload, + # so we have to fake the setup to validate the path-rewriting logic. + # + # Build a loader that has found the extension with a PYTHONPATH + # reflecting the location of the pure-python tests. + loader = self.machinery.AppleFrameworkLoader( + util.EXTENSIONS.name, + util.EXTENSIONS.file_path, + [ + "/non/existent/path", + os.path.dirname(__file__), + ] + ) + + # Make sure we have a clean import cache + try: + del sys.modules[util.EXTENSIONS.name] + except KeyError: + pass + + # Load the module, and check the filename reports as the + # "fake" original name, not the extension's actual file path. + module = loader.load_module(util.EXTENSIONS.name) + assert module.__file__ == os.path.join( + os.path.dirname(__file__), + os.path.split(util.EXTENSIONS.file_path)[-1] + ) + (Frozen_LoaderTests, Source_LoaderTests @@ -101,6 +141,12 @@ class SinglePhaseExtensionModuleTests(abc.LoaderTests): def setUp(self): if not self.machinery.EXTENSION_SUFFIXES: raise unittest.SkipTest("Requires dynamic loading support.") + + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + self.name = '_testsinglephase' if self.name in sys.builtin_module_names: raise unittest.SkipTest( @@ -109,8 +155,8 @@ def setUp(self): finder = self.machinery.FileFinder(None) self.spec = importlib.util.find_spec(self.name) assert self.spec - self.loader = self.machinery.ExtensionFileLoader( - self.name, self.spec.origin) + + self.loader = self.LoaderClass(self.name, self.spec.origin) def load_module(self): with warnings.catch_warnings(): @@ -120,7 +166,7 @@ def load_module(self): def load_module_by_name(self, fullname): # Load a module from the test extension by name. origin = self.spec.origin - loader = self.machinery.ExtensionFileLoader(fullname, origin) + loader = self.LoaderClass(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -137,8 +183,7 @@ def test_module(self): with self.assertRaises(AttributeError): module.__path__ self.assertIs(module, sys.modules[self.name]) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) # No extension module as __init__ available for testing. test_package = None @@ -182,6 +227,12 @@ class MultiPhaseExtensionModuleTests(abc.LoaderTests): def setUp(self): if not self.machinery.EXTENSION_SUFFIXES: raise unittest.SkipTest("Requires dynamic loading support.") + + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + self.name = '_testmultiphase' if self.name in sys.builtin_module_names: raise unittest.SkipTest( @@ -190,8 +241,7 @@ def setUp(self): finder = self.machinery.FileFinder(None) self.spec = importlib.util.find_spec(self.name) assert self.spec - self.loader = self.machinery.ExtensionFileLoader( - self.name, self.spec.origin) + self.loader = self.LoaderClass(self.name, self.spec.origin) def load_module(self): # Load the module from the test extension. @@ -202,7 +252,7 @@ def load_module(self): def load_module_by_name(self, fullname): # Load a module from the test extension by name. origin = self.spec.origin - loader = self.machinery.ExtensionFileLoader(fullname, origin) + loader = self.LoaderClass(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -228,8 +278,7 @@ def test_module(self): with self.assertRaises(AttributeError): module.__path__ self.assertIs(module, sys.modules[self.name]) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) def test_functionality(self): # Test basic functionality of stuff defined in an extension module. diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index c25be096e52874..011e48fda44f43 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -7,6 +7,7 @@ import os import os.path from test.support import import_helper +from test.support import is_apple_mobile from test.support import os_helper import unittest import sys @@ -31,7 +32,20 @@ def _extension_details(): global EXTENSIONS - for path in sys.path: + # On Apple mobile, extension modules can only exist in the Frameworks + # folder, so don't bother checking the rest of the system path. + if is_apple_mobile: + paths = [ + os.path.join( + os.path.split(sys.executable)[0], + "Frameworks", + EXTENSIONS.name + ".framework" + ) + ] + else: + paths = sys.paths + + for path in paths: for ext in machinery.EXTENSION_SUFFIXES: filename = EXTENSIONS.name + ext file_path = os.path.join(path, filename) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst new file mode 100644 index 00000000000000..f7e5f2c0747d2d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst @@ -0,0 +1,2 @@ +Added a Finder and Loader to discover extension modules in an iOS-style +Frameworks folder. diff --git a/Python/dynload_shlib.c b/Python/dynload_shlib.c index 5a37a83805ba78..d130a2cf28ac24 100644 --- a/Python/dynload_shlib.c +++ b/Python/dynload_shlib.c @@ -28,6 +28,10 @@ #define LEAD_UNDERSCORE "" #endif +#ifdef __APPLE__ +# include "TargetConditionals.h" +#endif /* __APPLE__ */ + /* The .so extension module ABI tag, supplied by the Makefile via Makefile.pre.in and configure. This is used to discriminate between incompatible .so files so that extensions for different Python builds can @@ -38,12 +42,22 @@ const char *_PyImport_DynLoadFiletab[] = { #ifdef __CYGWIN__ ".dll", #else /* !__CYGWIN__ */ - "." SOABI ".so", -#ifdef ALT_SOABI - "." ALT_SOABI ".so", -#endif - ".abi" PYTHON_ABI_STRING ".so", - ".so", +# ifdef __APPLE__ +// TARGET_OS_IPHONE covers any non-macOS Apple platform. +# if TARGET_OS_IPHONE +# define SHLIB_SUFFIX ".dylib" +# else +# define SHLIB_SUFFIX ".so" +# endif +# else +# define SHLIB_SUFFIX ".so" +# endif + "." SOABI SHLIB_SUFFIX, +# ifdef ALT_SOABI + "." ALT_SOABI SHLIB_SUFFIX, +# endif + ".abi" PYTHON_ABI_STRING SHLIB_SUFFIX, + SHLIB_SUFFIX, #endif /* __CYGWIN__ */ NULL, }; From 1359bc80e49138e0dfc0cd8df675ec579a51e28c Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Thu, 7 Mar 2024 17:29:19 +0800 Subject: [PATCH 02/10] Removed a stray character. --- Lib/test/test_importlib/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index 011e48fda44f43..9a077b17ff07d8 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -43,7 +43,7 @@ def _extension_details(): ) ] else: - paths = sys.paths + paths = sys.path for path in paths: for ext in machinery.EXTENSION_SUFFIXES: From 5e0659efa061c2c794900d978568081e5a6e3937 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 8 Mar 2024 07:16:34 +0800 Subject: [PATCH 03/10] Simplify logic for dylib extensions. --- Python/dynload_shlib.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Python/dynload_shlib.c b/Python/dynload_shlib.c index d130a2cf28ac24..04126c48b57d76 100644 --- a/Python/dynload_shlib.c +++ b/Python/dynload_shlib.c @@ -42,13 +42,9 @@ const char *_PyImport_DynLoadFiletab[] = { #ifdef __CYGWIN__ ".dll", #else /* !__CYGWIN__ */ -# ifdef __APPLE__ -// TARGET_OS_IPHONE covers any non-macOS Apple platform. -# if TARGET_OS_IPHONE -# define SHLIB_SUFFIX ".dylib" -# else -# define SHLIB_SUFFIX ".so" -# endif +// TARGET_OS_IPHONE covers any non-macOS Apple platform. +# if defined(__APPLE__) && TARGET_OS_IPHONE +# define SHLIB_SUFFIX ".dylib" # else # define SHLIB_SUFFIX ".so" # endif From d5fda7e1eb178d1464c0e53f6250a3f7471c8951 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 8 Mar 2024 09:41:57 +0800 Subject: [PATCH 04/10] Clarified docstrings and implementation on iOS Loader/Finder, and added public docs. --- Doc/library/importlib.rst | 68 ++++++++++++++++++++++++++++ Lib/importlib/_bootstrap_external.py | 41 +++++++++-------- 2 files changed, 89 insertions(+), 20 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index d92bb2f8e5cf83..2c9a7fa753ae81 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1241,6 +1241,74 @@ find and load modules. and how the module's :attr:`__file__` is populated. +.. class:: AppleFrameworkLoader(fullname, dylib_path, parent_paths=None) + + A specialization of :class:`importlib.machinery.ExtensionFileLoader` that + is able to load extension modules in Framework format. + + For compatibility with the iOS App Store, *all* binary modules in an iOS app + must be ``.dylib objects``, contained in a framework with appropriate + metadata, stored in the ``Frameworks`` folder of the packaged app. There can + be only a single binary per framework, and there can be no executable binary + material outside the Frameworks folder. + + If you're trying to run ``from foo.bar import _whiz``, and ``_whiz`` is + implemented with the binary module ``foo/bar/_whiz.abi3.dylib`` (or any + other ABI .dylib extension), this loader will look for + ``{sys.executable}/Frameworks/foo.bar._whiz.framework/_whiz.abi3.dylib`` + (forming the package name by taking the full import path of the library, + and replacing ``/`` with ``.``). + + However, this loader will re-write the ``__file__`` attribute of the + ``_whiz`` module will report as the original location inside the ``foo/bar`` + subdirectory. This so that code that depends on walking directory trees will + continue to work as expected based on the *original* file location. + + The *fullname* argument specifies the name of the module the loader is to + support. The *dylib_path* argument is the path to the framework's ``.dylib`` + file. The ``parent_paths`` is the path or paths that was searched to find + the extension module. + + .. versionadded:: 3.13 + + .. availability:: iOS. + + .. attribute:: fullname + + Name of the module the loader supports. + + .. attribute:: dylib_path + + Path to the ``.dylib`` file in the framework. + + .. attribute:: parent_paths + + The parent paths that were originally searched to find the module. + +.. class:: AppleFrameworkFinder(framework_path) + + An extension module finder which is able to load extension modules packaged + as frameworks in an iOS app. + + See the documentation for :class:`AppleFrameworkLoader` for details on the + requirements of binary extension modules on iOS. + + The *framework_path* argument is the Frameworks directory for the app. + + .. versionadded:: 3.13 + + .. availability:: iOS. + + .. attribute:: framework_path + + The path the finder will search for frameworks. + + .. method:: find_spec(fullname, paths, target=None) + + Attempt to find the spec to handle ``fullname``, imported from one + of the filesystem locations described by ``paths``. + + :mod:`importlib.util` -- Utility code for importers --------------------------------------------------- diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index a7cff3eef03f25..59a05f7ba17fac 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -1713,12 +1713,13 @@ def __repr__(self): class AppleFrameworkLoader(ExtensionFileLoader): """A loader for modules that have been packaged as frameworks for - compatibility with Apple's App Store policies. + compatibility with Apple's iOS App Store policies. - For compatibility with the App Store, *all* binary modules must be .dylib - objects, contained in a framework, stored in the ``Frameworks`` folder of - the packaged app. There can be only a single binary per framework, and - there can be no executable binary material outside the Frameworks folder. + For compatibility with the iOS App Store, *all* binary modules in an iOS + app must be .dylib objects, contained in a framework with appropriate + metadata, stored in the ``Frameworks`` folder of the packaged app. There + can be only a single binary per framework, and there can be no executable + binary material outside the Frameworks folder. If you're trying to run ``from foo.bar import _whiz``, and ``_whiz`` is implemented with the binary module ``foo/bar/_whiz.abi3.dylib`` (or any @@ -1739,9 +1740,9 @@ class AppleFrameworkLoader(ExtensionFileLoader): be done with a build step in the Xcode project; see the iOS documentation for details on how to construct this build step. """ - def __init__(self, fullname, dylib_file, path=None): + def __init__(self, fullname, dylib_file, parent_paths=None): super().__init__(fullname, dylib_file) - self.parent_paths = path + self.parent_paths = parent_paths def create_module(self, spec): mod = super().create_module(spec) @@ -1752,7 +1753,7 @@ def create_module(self, spec): parent_path, _path_split(self.path)[-1], ) - continue + break return mod @@ -1762,23 +1763,23 @@ class AppleFrameworkFinder: See AppleFrameworkLoader for details. """ - def __init__(self, path): - self.frameworks_path = path + def __init__(self, frameworks_path): + self.frameworks_path = frameworks_path - def find_spec(self, fullname, path, target=None): + def find_spec(self, fullname, paths, target=None): name = fullname.split(".")[-1] for extension in EXTENSION_SUFFIXES: - dylib_file = _path_join(self.frameworks_path, f"{fullname}.framework", f"{name}{extension}") + dylib_file = _path_join( + self.frameworks_path, + f"{fullname}.framework", f"{name}{extension}" + ) _bootstrap._verbose_message("Looking for Apple Framework dylib {}", dylib_file) - try: - dylib_exists = _path_isfile(dylib_file) - except ValueError: - pass - else: - if dylib_exists: - loader = AppleFrameworkLoader(fullname, dylib_file, path) - return _bootstrap.spec_from_loader(fullname, loader) + + dylib_exists = _path_isfile(dylib_file) + if dylib_exists: + loader = AppleFrameworkLoader(fullname, dylib_file, paths) + return _bootstrap.spec_from_loader(fullname, loader) return None From 284e225118fdbeaf3b0874ff94f269cae548f31a Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 8 Mar 2024 10:04:04 +0800 Subject: [PATCH 05/10] Add iOS to the list of platforms known by documentation. --- Doc/tools/extensions/pyspecific.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index cd441836f62bde..9709c4f4dc54aa 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -133,7 +133,7 @@ class Availability(SphinxDirective): known_platforms = frozenset({ "AIX", "Android", "BSD", "DragonFlyBSD", "Emscripten", "FreeBSD", "GNU/kFreeBSD", "Linux", "NetBSD", "OpenBSD", "POSIX", "Solaris", - "Unix", "VxWorks", "WASI", "Windows", "macOS", + "Unix", "VxWorks", "WASI", "Windows", "macOS", "iOS", # libc "BSD libc", "glibc", "musl", # POSIX platforms with pthreads From 9f42faab97f6c1afd28359942ae5631324343fde Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Mon, 11 Mar 2024 13:09:56 +0800 Subject: [PATCH 06/10] Improvements to documentation. Co-authored-by: Malcolm Smith Co-authored-by: Eric Snow --- Doc/library/importlib.rst | 5 ++--- Lib/importlib/_bootstrap_external.py | 25 ------------------------- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 2c9a7fa753ae81..87c00e428ccd55 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1255,9 +1255,8 @@ find and load modules. If you're trying to run ``from foo.bar import _whiz``, and ``_whiz`` is implemented with the binary module ``foo/bar/_whiz.abi3.dylib`` (or any other ABI .dylib extension), this loader will look for - ``{sys.executable}/Frameworks/foo.bar._whiz.framework/_whiz.abi3.dylib`` - (forming the package name by taking the full import path of the library, - and replacing ``/`` with ``.``). + ``{dirname(sys.executable)}/Frameworks/foo.bar._whiz.framework/_whiz.abi3.dylib`` + (forming the framework name from the full name of the module). However, this loader will re-write the ``__file__`` attribute of the ``_whiz`` module will report as the original location inside the ``foo/bar`` diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 59a05f7ba17fac..03755765d549ac 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -1714,31 +1714,6 @@ def __repr__(self): class AppleFrameworkLoader(ExtensionFileLoader): """A loader for modules that have been packaged as frameworks for compatibility with Apple's iOS App Store policies. - - For compatibility with the iOS App Store, *all* binary modules in an iOS - app must be .dylib objects, contained in a framework with appropriate - metadata, stored in the ``Frameworks`` folder of the packaged app. There - can be only a single binary per framework, and there can be no executable - binary material outside the Frameworks folder. - - If you're trying to run ``from foo.bar import _whiz``, and ``_whiz`` is - implemented with the binary module ``foo/bar/_whiz.abi3.dylib`` (or any - other ABI .dylib extension), this loader will look for - ``{sys.executable}/Frameworks/foo.bar._whiz.framework/_whiz.abi3.dylib`` - (forming the package name by taking the full import path of the library, - and replacing ``/`` with ``.``). - - However, the ``__file__`` attribute of the ``_whiz`` module will report as - the original location inside the ``foo/bar`` subdirectory. This so that - code that depends on walking directory trees will continue to work as - expected based on the *original* file location. - - The Xcode project building the app is responsible for converting any - ``.dylib`` files from wherever they exist in the ``PYTHONPATH`` into - frameworks in the ``Frameworks`` folder (including the addition of - framework metadata, and signing the resulting framework). This will usually - be done with a build step in the Xcode project; see the iOS documentation - for details on how to construct this build step. """ def __init__(self, fullname, dylib_file, parent_paths=None): super().__init__(fullname, dylib_file) From 8308611c46eb79a96d15ad19b43f07daf0ea388f Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Mon, 11 Mar 2024 14:09:24 +0800 Subject: [PATCH 07/10] Modify loader to do more processing in the finder. Co-authored-by: Eric Snow --- Lib/importlib/_bootstrap_external.py | 36 +++++++++++++++------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 03755765d549ac..5c29423638d80b 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -1715,20 +1715,10 @@ class AppleFrameworkLoader(ExtensionFileLoader): """A loader for modules that have been packaged as frameworks for compatibility with Apple's iOS App Store policies. """ - def __init__(self, fullname, dylib_file, parent_paths=None): - super().__init__(fullname, dylib_file) - self.parent_paths = parent_paths - def create_module(self, spec): mod = super().create_module(spec) - if self.parent_paths: - for parent_path in self.parent_paths: - if _path_isdir(parent_path): - mod.__file__ = _path_join( - parent_path, - _path_split(self.path)[-1], - ) - break + if spec.loader_state.origfile is not None: + mod.__file__ = spec.loader_state.origfile return mod @@ -1742,19 +1732,33 @@ def __init__(self, frameworks_path): self.frameworks_path = frameworks_path def find_spec(self, fullname, paths, target=None): - name = fullname.split(".")[-1] + name = fullname.rpartition(".")[-1] for extension in EXTENSION_SUFFIXES: dylib_file = _path_join( self.frameworks_path, - f"{fullname}.framework", f"{name}{extension}" + f"{fullname}.framework", + f"{name}{extension}", ) _bootstrap._verbose_message("Looking for Apple Framework dylib {}", dylib_file) dylib_exists = _path_isfile(dylib_file) if dylib_exists: - loader = AppleFrameworkLoader(fullname, dylib_file, paths) - return _bootstrap.spec_from_loader(fullname, loader) + origfile = None + if paths: + for parent_path in paths: + if _path_isdir(parent_path): + origfile = _path_join( + parent_path, + _path_split(self.path)[-1], + ) + break + loader = AppleFrameworkLoader(fullname, dylib_file) + spec = _bootstrap.spec_from_loader(fullname, loader) + spec.loader_state = type(sys.implementation)( + origfile=origfile, + ) + return spec return None From fa3ffbbfce0eda360042266548b14703a9f41524 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Thu, 14 Mar 2024 14:26:06 +0800 Subject: [PATCH 08/10] Switch to a FileLoader-based approach for Apple framework loading. --- .gitignore | 2 +- Doc/library/importlib.rst | 77 +++++++--------- Lib/ctypes/__init__.py | 11 +++ Lib/ctypes/util.py | 2 +- Lib/importlib/_bootstrap_external.py | 89 +++++++++---------- Lib/importlib/abc.py | 6 +- Lib/importlib/machinery.py | 1 - Lib/modulefinder.py | 7 +- Lib/test/test_capi/test_misc.py | 16 +++- Lib/test/test_import/__init__.py | 31 +++++-- .../test_importlib/extension/test_finder.py | 29 +++--- .../test_importlib/extension/test_loader.py | 39 +------- Lib/test/test_importlib/test_util.py | 11 ++- Lib/test/test_importlib/util.py | 21 ++--- ...-03-07-16-12-39.gh-issue-114099.ujdjn2.rst | 4 +- Python/dynload_shlib.c | 22 ++--- configure | 1 - configure.ac | 1 - .../iOSTestbed.xcodeproj/project.pbxproj | 2 +- iOS/testbed/iOSTestbedTests/iOSTestbedTests.m | 2 +- 20 files changed, 178 insertions(+), 196 deletions(-) diff --git a/.gitignore b/.gitignore index 3e1213ef925305..c2c465fdedc6d2 100644 --- a/.gitignore +++ b/.gitignore @@ -69,7 +69,7 @@ Lib/test/data/* /_bootstrap_python /Makefile /Makefile.pre -iOSTestbed.* +/iOSTestbed.* iOS/Frameworks/ iOS/Resources/Info.plist iOS/testbed/build diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 87c00e428ccd55..c6327cfadff29c 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1241,71 +1241,56 @@ find and load modules. and how the module's :attr:`__file__` is populated. -.. class:: AppleFrameworkLoader(fullname, dylib_path, parent_paths=None) +.. class:: AppleFrameworkLoader(name, path) A specialization of :class:`importlib.machinery.ExtensionFileLoader` that is able to load extension modules in Framework format. For compatibility with the iOS App Store, *all* binary modules in an iOS app - must be ``.dylib objects``, contained in a framework with appropriate + must be dynamic libraries, contained in a framework with appropriate metadata, stored in the ``Frameworks`` folder of the packaged app. There can be only a single binary per framework, and there can be no executable binary material outside the Frameworks folder. If you're trying to run ``from foo.bar import _whiz``, and ``_whiz`` is - implemented with the binary module ``foo/bar/_whiz.abi3.dylib`` (or any - other ABI .dylib extension), this loader will look for - ``{dirname(sys.executable)}/Frameworks/foo.bar._whiz.framework/_whiz.abi3.dylib`` - (forming the framework name from the full name of the module). - - However, this loader will re-write the ``__file__`` attribute of the - ``_whiz`` module will report as the original location inside the ``foo/bar`` - subdirectory. This so that code that depends on walking directory trees will - continue to work as expected based on the *original* file location. - - The *fullname* argument specifies the name of the module the loader is to - support. The *dylib_path* argument is the path to the framework's ``.dylib`` - file. The ``parent_paths`` is the path or paths that was searched to find - the extension module. + implemented with the binary module ``foo/bar/_whiz.abi3.so`` (or any other + ABI extension), this module *must* be distributed as + ``{dirname(sys.executable)}/Frameworks/foo.bar._whiz.framework/foo.bar._whiz`` + (creating the framework name from the full import path of the module), with + an ``Info.plist`` file in the ``.framework`` file identifying the binary. + + To accomodate this requirement, when running on iOS, this loader will be + registered against the ``.fwork`` file extension. A ``.fwork`` file is an + placeholder that flags that an extension module with the corresponding name + and path can be loaded from the ``Frameworks`` folder. The ``.fwork`` file + contains the path of the actual binary, relative to the app bundle. The + ``foo.bar._whiz`` module in the previous example would generate a + ``foo/bar/_whiz.abi3.fwork`` marker file, containing the path + ``Frameworks/foo.bar._whiz/foo.bar._whiz``. The spec created from loading + this module will have an origin referencing the ``.fwork`` file; when the + module is created, it will load the location referenced by the content of + the ``.fwork`` file. + + The Xcode project building the app is responsible for converting any ``.so`` + files from wherever they exist in the ``PYTHONPATH`` into frameworks in the + ``Frameworks`` folder (including stripping extensions from the module file, + the addition of framework metadata, and signing the resulting framework), + and creating the ``.fwork`` file in place of the ``.so`` file to flag that + framework loading is required. This will usually be done with a build step + in the Xcode project; see the iOS documentation for details on how to + construct this build step. .. versionadded:: 3.13 .. availability:: iOS. - .. attribute:: fullname + .. attribute:: name Name of the module the loader supports. - .. attribute:: dylib_path - - Path to the ``.dylib`` file in the framework. - - .. attribute:: parent_paths - - The parent paths that were originally searched to find the module. - -.. class:: AppleFrameworkFinder(framework_path) - - An extension module finder which is able to load extension modules packaged - as frameworks in an iOS app. - - See the documentation for :class:`AppleFrameworkLoader` for details on the - requirements of binary extension modules on iOS. - - The *framework_path* argument is the Frameworks directory for the app. - - .. versionadded:: 3.13 - - .. availability:: iOS. - - .. attribute:: framework_path - - The path the finder will search for frameworks. - - .. method:: find_spec(fullname, paths, target=None) + .. attribute:: path - Attempt to find the spec to handle ``fullname``, imported from one - of the filesystem locations described by ``paths``. + Path to the ``.fwork`` file for the extension module. :mod:`importlib.util` -- Utility code for importers diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index d54ee05b15f5bd..93ec01c4ba343d 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -348,6 +348,17 @@ def __init__(self, name, mode=DEFAULT_MODE, handle=None, winmode=None): if name: name = _os.fspath(name) + + # If the filename that has been provided is an iOS/tvOS/watchOS + # .fwork file, dereference the location to the true origin of the + # binary. + if name.endswith(".fwork"): + with open(name) as f: + name = _os.path.join( + _os.path.dirname(_sys.executable), + f.read().strip() + ) + self._name = name flags = self._func_flags_ if use_errno: diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index c550883e7c7d4b..12d7428fe9a776 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -67,7 +67,7 @@ def find_library(name): return fname return None -elif os.name == "posix" and sys.platform == "darwin": +elif os.name == "posix" and sys.platform in {"darwin", "ios", "tvos", "watchos"}: from ctypes.macholib.dyld import dyld_find as _dyld_find def find_library(name): possible = ['lib%s.dylib' % name, diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 5c29423638d80b..7b46b52b41a88e 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -1715,52 +1715,40 @@ class AppleFrameworkLoader(ExtensionFileLoader): """A loader for modules that have been packaged as frameworks for compatibility with Apple's iOS App Store policies. """ - def create_module(self, spec): - mod = super().create_module(spec) - if spec.loader_state.origfile is not None: - mod.__file__ = spec.loader_state.origfile - return mod + def create_module(self, spec): + # If the ModuleSpec has been created by the FileFinder, it will have + # been created with an origin pointing to the .fwork file. We need to + # dereference this to the "true origin" - the actual binary file + # location in the Frameworks folder + if spec.origin.endswith(".fwork"): + with _io.FileIO(spec.origin, 'r') as file: + framework_binary = file.read().decode().strip() + bundle_path = _path_split(sys.executable)[0] + true_origin = _path_join(bundle_path, framework_binary) + else: + true_origin = spec.origin + + # The implementation of the importer needs to operate on the actual + # binary. Temporarily switch the origin of the spec to the true origin; + # once the module has been loaded, switch the origin back so that + # the .fwork location isn't lost. + origin = spec.origin + spec.origin = true_origin + module = _bootstrap._call_with_frames_removed(_imp.create_dynamic, spec) + spec.origin = origin + + _bootstrap._verbose_message( + "Apple framework extension module {!r} loaded from {!r} (true origin {!r})", + spec.name, + self.path, + true_origin, + ) -class AppleFrameworkFinder: - """A finder for modules that have been packaged as Apple Frameworks - for compatibility with Apple's App Store policies. + # Ensure that the __file__ points at the .fwork location + module.__file__ = origin - See AppleFrameworkLoader for details. - """ - def __init__(self, frameworks_path): - self.frameworks_path = frameworks_path - - def find_spec(self, fullname, paths, target=None): - name = fullname.rpartition(".")[-1] - - for extension in EXTENSION_SUFFIXES: - dylib_file = _path_join( - self.frameworks_path, - f"{fullname}.framework", - f"{name}{extension}", - ) - _bootstrap._verbose_message("Looking for Apple Framework dylib {}", dylib_file) - - dylib_exists = _path_isfile(dylib_file) - if dylib_exists: - origfile = None - if paths: - for parent_path in paths: - if _path_isdir(parent_path): - origfile = _path_join( - parent_path, - _path_split(self.path)[-1], - ) - break - loader = AppleFrameworkLoader(fullname, dylib_file) - spec = _bootstrap.spec_from_loader(fullname, loader) - spec.loader_state = type(sys.implementation)( - origfile=origfile, - ) - return spec - - return None + return module # Import setup ############################################################### @@ -1794,10 +1782,17 @@ def _get_supported_file_loaders(): Each item is a tuple (loader, suffixes). """ - extensions = ExtensionFileLoader, _imp.extension_suffixes() + if sys.platform in {"ios", "tvos", "watchos"}: + extension_loaders = [(AppleFrameworkLoader, [ + suffix.replace(".so", ".fwork") + for suffix in _imp.extension_suffixes() + ])] + else: + extension_loaders = [] + extension_loaders.append((ExtensionFileLoader, _imp.extension_suffixes())) source = SourceFileLoader, SOURCE_SUFFIXES bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES - return [extensions, source, bytecode] + return extension_loaders + [source, bytecode] def _set_bootstrap_module(_bootstrap_module): @@ -1811,7 +1806,3 @@ def _install(_bootstrap_module): supported_loaders = _get_supported_file_loaders() sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)]) sys.meta_path.append(PathFinder) - if sys.platform in {"ios", "tvos", "watchos"}: - frameworks_folder = _path_join(_path_split(sys.executable)[0], "Frameworks") - _bootstrap._verbose_message("Adding Apple Framework dylib finder at {}", frameworks_folder) - sys.meta_path.append(AppleFrameworkFinder(frameworks_folder)) diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index b56fa94eb9c135..37fef357fe2c0c 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -180,7 +180,11 @@ def get_code(self, fullname): else: return self.source_to_code(source, path) -_register(ExecutionLoader, machinery.ExtensionFileLoader) +_register( + ExecutionLoader, + machinery.ExtensionFileLoader, + machinery.AppleFrameworkLoader, +) class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader): diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index 928ed116a2c9a4..fbd30b159fb752 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -13,7 +13,6 @@ from ._bootstrap_external import SourcelessFileLoader from ._bootstrap_external import ExtensionFileLoader from ._bootstrap_external import AppleFrameworkLoader -from ._bootstrap_external import AppleFrameworkFinder from ._bootstrap_external import NamespaceLoader diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index a0a020f9eeb9b4..ac478ee7f51722 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -72,7 +72,12 @@ def _find_module(name, path=None): if isinstance(spec.loader, importlib.machinery.SourceFileLoader): kind = _PY_SOURCE - elif isinstance(spec.loader, importlib.machinery.ExtensionFileLoader): + elif isinstance( + spec.loader, ( + importlib.machinery.ExtensionFileLoader, + importlib.machinery.AppleFrameworkLoader, + ) + ): kind = _C_EXTENSION elif isinstance(spec.loader, importlib.machinery.SourcelessFileLoader): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 67fbef4f269814..0f202350878134 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1999,6 +1999,13 @@ def test_module_state_shared_in_global(self): self.addCleanup(os.close, r) self.addCleanup(os.close, w) + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + script = textwrap.dedent(f""" import importlib.machinery import importlib.util @@ -2006,7 +2013,7 @@ def test_module_state_shared_in_global(self): fullname = '_test_module_state_shared' origin = importlib.util.find_spec('_testmultiphase').origin - loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + loader = importlib.machinery.{loader}(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) attr_id = str(id(module.Error)).encode() @@ -2303,7 +2310,12 @@ class Test_ModuleStateAccess(unittest.TestCase): def setUp(self): fullname = '_testmultiphase_meth_state_access' # XXX origin = importlib.util.find_spec('_testmultiphase').origin - loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = importlib.machinery.AppleFrameworkLoader(fullname, origin) + else: + loader = importlib.machinery.ExtensionFileLoader(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 38810f9c7db307..979e8b1e104a90 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -5,7 +5,11 @@ import importlib.util from importlib._bootstrap_external import _get_sourcefile from importlib.machinery import ( - AppleFrameworkLoader, BuiltinImporter, ExtensionFileLoader, FrozenImporter, SourceFileLoader, + AppleFrameworkLoader, + BuiltinImporter, + ExtensionFileLoader, + FrozenImporter, + SourceFileLoader, ) import marshal import os @@ -92,6 +96,8 @@ def require_builtin(module, *, skip=False): assert module.__spec__.origin == 'built-in', module.__spec__ def require_extension(module, *, skip=False): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. if is_apple_mobile: _require_loader(module, AppleFrameworkLoader, skip) else: @@ -364,7 +370,7 @@ def test_from_import_missing_attr_has_name_and_so_path(self): self.assertEqual(cm.exception.path, _testcapi.__file__) self.assertRegex( str(cm.exception), - r"cannot import name 'i_dont_exist' from '_testcapi' \(.*\.(so|dylib|pyd)\)" + r"cannot import name 'i_dont_exist' from '_testcapi' \(.*\.(so|fwork|pyd)\)" ) else: self.assertEqual( @@ -1694,8 +1700,10 @@ def pipe(self): return (r, w) def create_extension_loader(self, modname, filename): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. if is_apple_mobile: - return AppleFrameworkLoader(modname, filename, None) + return AppleFrameworkLoader(modname, filename) else: return ExtensionFileLoader(modname, filename) @@ -1707,12 +1715,19 @@ def import_script(self, name, fd, filename=None, check_override=None): _imp._override_multi_interp_extensions_check({check_override}) ''' if filename: + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + return textwrap.dedent(f''' from importlib.util import spec_from_loader, module_from_spec - from importlib.machinery import ExtensionFileLoader + from importlib.machinery import {loader} import os, sys {override_text} - loader = ExtensionFileLoader({name!r}, {filename!r}) + loader = {loader}({name!r}, {filename!r}) spec = spec_from_loader({name!r}, loader) try: module = module_from_spec(spec) @@ -2044,9 +2059,11 @@ class SinglephaseInitTests(unittest.TestCase): @classmethod def setUpClass(cls): spec = importlib.util.find_spec(cls.NAME) - from importlib.machinery import AppleFrameworkLoader, ExtensionFileLoader cls.FILE = spec.origin cls.LOADER = type(spec.loader) + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. if is_apple_mobile: assert cls.LOADER is AppleFrameworkLoader else: @@ -2091,7 +2108,7 @@ def _load_dynamic(self, name, path): # This is essentially copied from the old imp module. from importlib._bootstrap import _load if is_apple_mobile: - loader = self.LOADER(name, path, None) + loader = self.LOADER(name, path) else: loader = self.LOADER(name, path) diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index 0bf4412c1c2ffb..55905cd61311ab 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -4,7 +4,6 @@ machinery = util.import_importlib('importlib.machinery') import unittest -import os import sys @@ -22,24 +21,28 @@ def setUp(self): def find_spec(self, fullname): if is_apple_mobile: - # Apple extensions must be distributed as frameworks. This requires - # a specialist finder. - frameworks_folder = os.path.join( - os.path.split(sys.executable)[0], "Frameworks" - ) - importer = self.machinery.AppleFrameworkFinder(frameworks_folder) - - return importer.find_spec(fullname, None) + # Apple mobile platforms require a specialist loader that uses + # .fwork files as placeholders for the true `.so` files. + loaders = [ + ( + self.machinery.AppleFrameworkLoader, + [ + ext.replace(".so", ".fwork") + for ext in self.machinery.EXTENSION_SUFFIXES + ] + ) + ] else: - importer = self.machinery.FileFinder( - util.EXTENSIONS.path, + loaders = [ ( self.machinery.ExtensionFileLoader, self.machinery.EXTENSION_SUFFIXES ) - ) + ] + + importer = self.machinery.FileFinder(util.EXTENSIONS.path, *loaders) - return importer.find_spec(fullname) + return importer.find_spec(fullname) def test_module(self): self.assertTrue(self.find_spec(util.EXTENSIONS.name)) diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 559dc4725e842e..a540cc4bf2935b 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -94,41 +94,6 @@ def test_is_package(self): loader = self.LoaderClass('pkg', path) self.assertTrue(loader.is_package('pkg')) - @unittest.skipUnless(is_apple_mobile, "Only required on Apple mobile") - def test_file_origin(self): - # Apple Mobile requires that binary extension modules are moved from - # their "normal" location to the Frameworks folder. Many third-party - # packages assume that __file__ will return somewhere in the - # PYTHONPATH; this won't be true on Apple mobile, so the - # AppleFrameworkLoader rewrites __file__ to point at the original path. - # However, the standard library puts all it's modules in lib-dynload, - # so we have to fake the setup to validate the path-rewriting logic. - # - # Build a loader that has found the extension with a PYTHONPATH - # reflecting the location of the pure-python tests. - loader = self.machinery.AppleFrameworkLoader( - util.EXTENSIONS.name, - util.EXTENSIONS.file_path, - [ - "/non/existent/path", - os.path.dirname(__file__), - ] - ) - - # Make sure we have a clean import cache - try: - del sys.modules[util.EXTENSIONS.name] - except KeyError: - pass - - # Load the module, and check the filename reports as the - # "fake" original name, not the extension's actual file path. - module = loader.load_module(util.EXTENSIONS.name) - assert module.__file__ == os.path.join( - os.path.dirname(__file__), - os.path.split(util.EXTENSIONS.file_path)[-1] - ) - (Frozen_LoaderTests, Source_LoaderTests @@ -142,6 +107,8 @@ def setUp(self): if not self.machinery.EXTENSION_SUFFIXES: raise unittest.SkipTest("Requires dynamic loading support.") + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. if is_apple_mobile: self.LoaderClass = self.machinery.AppleFrameworkLoader else: @@ -228,6 +195,8 @@ def setUp(self): if not self.machinery.EXTENSION_SUFFIXES: raise unittest.SkipTest("Requires dynamic loading support.") + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. if is_apple_mobile: self.LoaderClass = self.machinery.AppleFrameworkLoader else: diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index fe5e7b31d9c32b..44df206ee69c34 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -707,13 +707,20 @@ def test_single_phase_init_module(self): @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") def test_incomplete_multi_phase_init_module(self): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + prescript = textwrap.dedent(f''' from importlib.util import spec_from_loader, module_from_spec - from importlib.machinery import ExtensionFileLoader + from importlib.machinery import {loader} name = '_test_shared_gil_only' filename = {_testmultiphase.__file__!r} - loader = ExtensionFileLoader(name, filename) + loader = {loader}(name, filename) spec = spec_from_loader(name, loader) ''') diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index 9a077b17ff07d8..ffa22a28899fb3 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -31,22 +31,13 @@ EXTENSIONS.name = '_testsinglephase' def _extension_details(): - global EXTENSIONS - # On Apple mobile, extension modules can only exist in the Frameworks - # folder, so don't bother checking the rest of the system path. - if is_apple_mobile: - paths = [ - os.path.join( - os.path.split(sys.executable)[0], - "Frameworks", - EXTENSIONS.name + ".framework" - ) - ] - else: - paths = sys.path - - for path in paths: + for path in sys.path: for ext in machinery.EXTENSION_SUFFIXES: + # Apple mobile platforms mechanically load .so files, + # but the findable files are labelled .fwork + if is_apple_mobile: + ext = ext.replace(".so", ".fwork") + filename = EXTENSIONS.name + ext file_path = os.path.join(path, filename) if os.path.exists(file_path): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst index f7e5f2c0747d2d..5405a3bdc36f9e 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst @@ -1,2 +1,2 @@ -Added a Finder and Loader to discover extension modules in an iOS-style -Frameworks folder. +Added a Loader that can discover extension modules in an iOS-style Frameworks +folder. diff --git a/Python/dynload_shlib.c b/Python/dynload_shlib.c index 04126c48b57d76..5a37a83805ba78 100644 --- a/Python/dynload_shlib.c +++ b/Python/dynload_shlib.c @@ -28,10 +28,6 @@ #define LEAD_UNDERSCORE "" #endif -#ifdef __APPLE__ -# include "TargetConditionals.h" -#endif /* __APPLE__ */ - /* The .so extension module ABI tag, supplied by the Makefile via Makefile.pre.in and configure. This is used to discriminate between incompatible .so files so that extensions for different Python builds can @@ -42,18 +38,12 @@ const char *_PyImport_DynLoadFiletab[] = { #ifdef __CYGWIN__ ".dll", #else /* !__CYGWIN__ */ -// TARGET_OS_IPHONE covers any non-macOS Apple platform. -# if defined(__APPLE__) && TARGET_OS_IPHONE -# define SHLIB_SUFFIX ".dylib" -# else -# define SHLIB_SUFFIX ".so" -# endif - "." SOABI SHLIB_SUFFIX, -# ifdef ALT_SOABI - "." ALT_SOABI SHLIB_SUFFIX, -# endif - ".abi" PYTHON_ABI_STRING SHLIB_SUFFIX, - SHLIB_SUFFIX, + "." SOABI ".so", +#ifdef ALT_SOABI + "." ALT_SOABI ".so", +#endif + ".abi" PYTHON_ABI_STRING ".so", + ".so", #endif /* __CYGWIN__ */ NULL, }; diff --git a/configure b/configure index e62cb2baa29a78..9e68bcb19669e6 100755 --- a/configure +++ b/configure @@ -12735,7 +12735,6 @@ if test -z "$SHLIB_SUFFIX"; then esac ;; CYGWIN*) SHLIB_SUFFIX=.dll;; - iOS) SHLIB_SUFFIX=.dylib;; *) SHLIB_SUFFIX=.so;; esac fi diff --git a/configure.ac b/configure.ac index 990215b613aef4..077c7215b3f2a4 100644 --- a/configure.ac +++ b/configure.ac @@ -3284,7 +3284,6 @@ if test -z "$SHLIB_SUFFIX"; then esac ;; CYGWIN*) SHLIB_SUFFIX=.dll;; - iOS) SHLIB_SUFFIX=.dylib;; *) SHLIB_SUFFIX=.so;; esac fi diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj index 4f138a4e7ccefd..c46435c6709714 100644 --- a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj +++ b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -273,7 +273,7 @@ ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_DYLIB=$2\n\n # The name of the .dylib file\n DYLIB=$(basename \"$FULL_DYLIB\")\n # The name of the .dylib file, relative to the install base\n RELATIVE_DYLIB=${FULL_DYLIB#$CODESIGNING_FOLDER_PATH/$INSTALL_BASE/}\n # The full dotted name of the binary module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $RELATIVE_DYLIB | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_DYLIB\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$DYLIB\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\" \n fi\n \n echo \"Installing binary for $RELATIVE_DYLIB\" \n mv \"$FULL_DYLIB\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library dylibs...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.dylib\" | while read FULL_DYLIB; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload \"$FULL_DYLIB\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; + shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The name of the .dylib file, relative to the install base\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/$INSTALL_BASE/}\n # The full dotted name of the binary module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $RELATIVE_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\" \n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" >> ${FULL_EXT%.so}.fwork\n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; }; /* End PBXShellScriptBuildPhase section */ diff --git a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m index 53ea107db4a2de..e6a919c304ec8d 100644 --- a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m +++ b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m @@ -15,7 +15,7 @@ - (void)testPython { const char *argv[] = { "iOSTestbed", // argv[0] is the process that is running. "-uall", // Enable all resources - "-v", // run in verbose mode so we get test failure information + "-W", // Display test output on failure // To run a subset of tests, add the test names below; e.g., // "test_os", // "test_sys", From e66aee1152984e16e9388779fba4ae2139820a23 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 15 Mar 2024 12:51:49 +0800 Subject: [PATCH 09/10] Add the ability to reverse a framework back to it's origin location. --- Doc/library/importlib.rst | 55 +++++++++++-------- Lib/importlib/_bootstrap_external.py | 35 ++++++------ Lib/inspect.py | 7 ++- Lib/test/test_import/__init__.py | 43 +++++++++------ .../iOSTestbed.xcodeproj/project.pbxproj | 2 +- 5 files changed, 84 insertions(+), 58 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index c6327cfadff29c..b58ef359378e4f 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1252,33 +1252,44 @@ find and load modules. be only a single binary per framework, and there can be no executable binary material outside the Frameworks folder. - If you're trying to run ``from foo.bar import _whiz``, and ``_whiz`` is - implemented with the binary module ``foo/bar/_whiz.abi3.so`` (or any other - ABI extension), this module *must* be distributed as - ``{dirname(sys.executable)}/Frameworks/foo.bar._whiz.framework/foo.bar._whiz`` - (creating the framework name from the full import path of the module), with - an ``Info.plist`` file in the ``.framework`` file identifying the binary. - - To accomodate this requirement, when running on iOS, this loader will be - registered against the ``.fwork`` file extension. A ``.fwork`` file is an - placeholder that flags that an extension module with the corresponding name - and path can be loaded from the ``Frameworks`` folder. The ``.fwork`` file - contains the path of the actual binary, relative to the app bundle. The - ``foo.bar._whiz`` module in the previous example would generate a - ``foo/bar/_whiz.abi3.fwork`` marker file, containing the path - ``Frameworks/foo.bar._whiz/foo.bar._whiz``. The spec created from loading - this module will have an origin referencing the ``.fwork`` file; when the - module is created, it will load the location referenced by the content of - the ``.fwork`` file. + To accomodate this requirement, when running on iOS, extension module + binaries are *not* packaged as ``.so`` files on ``sys.path``, but as + individual standalone frameworks. To discover those frameworks, this loader + is be registered against the ``.fwork`` file extension, with a ``.fwork`` + file acting as a placeholder in the original location of the binary on + ``sys.path``. The ``.fwork`` file contains the path of the actual binary in + the ``Frameworks`` folder, relative to the app bundle. To allow for + resolving a framework-packaged binary back to the original location, the + framework is expected to contain a ``.origin`` file that contains the + location of the ``.fwork`` file, relative to the app bundle. + + For example, consider the case of an import ``from foo.bar import _whiz``, + where ``_whiz`` is implemented with the binary module + ``sources/foo/bar/_whiz.abi3.so``, with ``sources`` being the location + registered on ``sys.path``, relative to the application bundle. This module + *must* be distributed as + ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz`` (creating the framework + name from the full import path of the module), with an ``Info.plist`` file + in the ``.framework`` directory identifying the binary as a framework. The + ``foo.bar._whiz`` module would be represented in the original location with + a ``sources/foo/bar/_whiz.abi3.fwork`` marker file, containing the path + ``Frameworks/foo.bar._whiz/foo.bar._whiz``. The framework would also contain + ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz.origin``, containing the + path to the ``.fwork`` file. + + When a module is loaded with this loader, the ``__file__`` for the module + will report as the location of the ``.fwork`` file. This allows code to use + the ``__file__`` of a module as an anchor for file system traveral. + However, the spec origin will reference the location of the *actual* binary + in the ``.framework`` folder. The Xcode project building the app is responsible for converting any ``.so`` files from wherever they exist in the ``PYTHONPATH`` into frameworks in the ``Frameworks`` folder (including stripping extensions from the module file, the addition of framework metadata, and signing the resulting framework), - and creating the ``.fwork`` file in place of the ``.so`` file to flag that - framework loading is required. This will usually be done with a build step - in the Xcode project; see the iOS documentation for details on how to - construct this build step. + and creating the ``.fwork`` and ``.origin`` files. This will usually be done + with a build step in the Xcode project; see the iOS documentation for + details on how to construct this build step. .. versionadded:: 3.13 diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 7b46b52b41a88e..c831c4fafc193b 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -1715,38 +1715,39 @@ class AppleFrameworkLoader(ExtensionFileLoader): """A loader for modules that have been packaged as frameworks for compatibility with Apple's iOS App Store policies. """ - def create_module(self, spec): # If the ModuleSpec has been created by the FileFinder, it will have # been created with an origin pointing to the .fwork file. We need to - # dereference this to the "true origin" - the actual binary file - # location in the Frameworks folder + # redirect this to the location in the Frameworks folder, using the + # content of the .fwork file. if spec.origin.endswith(".fwork"): with _io.FileIO(spec.origin, 'r') as file: framework_binary = file.read().decode().strip() bundle_path = _path_split(sys.executable)[0] - true_origin = _path_join(bundle_path, framework_binary) + spec.origin = _path_join(bundle_path, framework_binary) + + # If the loader is created based on the spec for a loaded module, the + # path will be pointing at the Framework location. If this occurs, + # get the original .fwork location to use as the module's __file__. + if self.path.endswith(".fwork"): + path = self.path else: - true_origin = spec.origin - - # The implementation of the importer needs to operate on the actual - # binary. Temporarily switch the origin of the spec to the true origin; - # once the module has been loaded, switch the origin back so that - # the .fwork location isn't lost. - origin = spec.origin - spec.origin = true_origin + with _io.FileIO(self.path + ".origin", 'r') as file: + origin = file.read().decode().strip() + bundle_path = _path_split(sys.executable)[0] + path = _path_join(bundle_path, origin) + module = _bootstrap._call_with_frames_removed(_imp.create_dynamic, spec) - spec.origin = origin _bootstrap._verbose_message( - "Apple framework extension module {!r} loaded from {!r} (true origin {!r})", + "Apple framework extension module {!r} loaded from {!r} (path {!r})", spec.name, - self.path, - true_origin, + spec.origin, + path, ) # Ensure that the __file__ points at the .fwork location - module.__file__ = origin + module.__file__ = path return module diff --git a/Lib/inspect.py b/Lib/inspect.py index 8a2b2c96e993b5..7336cea0dc3fdc 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -954,6 +954,10 @@ def getsourcefile(object): elif any(filename.endswith(s) for s in importlib.machinery.EXTENSION_SUFFIXES): return None + elif filename.endswith(".fwork"): + # Apple mobile framework markers are another type of non-source file + return None + # return a filename found in the linecache even if it doesn't exist on disk if filename in linecache.cache: return filename @@ -984,6 +988,7 @@ def getmodule(object, _filename=None): return object if hasattr(object, '__module__'): return sys.modules.get(object.__module__) + # Try the filename to modulename cache if _filename is not None and _filename in modulesbyfile: return sys.modules.get(modulesbyfile[_filename]) @@ -1119,7 +1124,7 @@ def findsource(object): # Allow filenames in form of "" to pass through. # `doctest` monkeypatches `linecache` module to enable # inspection, so let `linecache.getlines` to be called. - if not (file.startswith('<') and file.endswith('>')): + if (not (file.startswith('<') and file.endswith('>'))) or file.endswith('.fwork'): raise OSError('source code not available') module = getmodule(object, file) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 979e8b1e104a90..4deed7f3ba2522 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -144,7 +144,8 @@ def restore__testsinglephase(*, _orig=_testsinglephase): # it to its nominal state. sys.modules.pop('_testsinglephase', None) _orig._clear_globals() - _testinternalcapi.clear_extension('_testsinglephase', _orig.__file__) + origin = _orig.__spec__.origin + _testinternalcapi.clear_extension('_testsinglephase', origin) import _testsinglephase @@ -2059,16 +2060,26 @@ class SinglephaseInitTests(unittest.TestCase): @classmethod def setUpClass(cls): spec = importlib.util.find_spec(cls.NAME) - cls.FILE = spec.origin cls.LOADER = type(spec.loader) # Apple extensions must be distributed as frameworks. This requires - # a specialist loader. + # a specialist loader, and we need to differentiate between the + # spec.origin and the original file location. if is_apple_mobile: assert cls.LOADER is AppleFrameworkLoader + + cls.ORIGIN = spec.origin + with open(spec.origin + ".origin", "r") as f: + cls.FILE = os.path.join( + os.path.dirname(sys.executable), + f.read().strip() + ) else: assert cls.LOADER is ExtensionFileLoader + cls.ORIGIN = spec.origin + cls.FILE = spec.origin + # Start fresh. cls.clean_up() @@ -2083,14 +2094,15 @@ def tearDown(self): @classmethod def clean_up(cls): name = cls.NAME - filename = cls.FILE if name in sys.modules: if hasattr(sys.modules[name], '_clear_globals'): - assert sys.modules[name].__file__ == filename + assert sys.modules[name].__file__ == cls.FILE, \ + f"{sys.modules[name].__file__} != {cls.FILE}" + sys.modules[name]._clear_globals() del sys.modules[name] # Clear all internally cached data for the extension. - _testinternalcapi.clear_extension(name, filename) + _testinternalcapi.clear_extension(name, cls.ORIGIN) ######################### # helpers @@ -2098,7 +2110,7 @@ def clean_up(cls): def add_module_cleanup(self, name): def clean_up(): # Clear all internally cached data for the extension. - _testinternalcapi.clear_extension(name, self.FILE) + _testinternalcapi.clear_extension(name, self.ORIGIN) self.addCleanup(clean_up) def _load_dynamic(self, name, path): @@ -2107,10 +2119,7 @@ def _load_dynamic(self, name, path): """ # This is essentially copied from the old imp module. from importlib._bootstrap import _load - if is_apple_mobile: - loader = self.LOADER(name, path) - else: - loader = self.LOADER(name, path) + loader = self.LOADER(name, path) # Issue bpo-24748: Skip the sys.modules check in _load_module_shim; # always load new extension. @@ -2124,7 +2133,7 @@ def load(self, name): except AttributeError: already_loaded = self.already_loaded = {} assert name not in already_loaded - mod = self._load_dynamic(name, self.FILE) + mod = self._load_dynamic(name, self.ORIGIN) self.assertNotIn(mod, already_loaded.values()) already_loaded[name] = mod return types.SimpleNamespace( @@ -2136,7 +2145,7 @@ def load(self, name): def re_load(self, name, mod): assert sys.modules[name] is mod assert mod.__dict__ == mod.__dict__ - reloaded = self._load_dynamic(name, self.FILE) + reloaded = self._load_dynamic(name, self.ORIGIN) return types.SimpleNamespace( name=name, module=reloaded, @@ -2162,7 +2171,7 @@ def clean_up(): name = {self.NAME!r} if name in sys.modules: sys.modules.pop(name)._clear_globals() - _testinternalcapi.clear_extension(name, {self.FILE!r}) + _testinternalcapi.clear_extension(name, {self.ORIGIN!r}) ''')) _interpreters.destroy(interpid) self.addCleanup(clean_up) @@ -2179,7 +2188,7 @@ def import_in_subinterp(self, interpid=None, *, postcleanup = f''' {import_} mod._clear_globals() - _testinternalcapi.clear_extension(name, {self.FILE!r}) + _testinternalcapi.clear_extension(name, {self.ORIGIN!r}) ''' try: @@ -2217,7 +2226,7 @@ def check_common(self, loaded): # mod.__name__ might not match, but the spec will. self.assertEqual(mod.__spec__.name, loaded.name) self.assertEqual(mod.__file__, self.FILE) - self.assertEqual(mod.__spec__.origin, self.FILE) + self.assertEqual(mod.__spec__.origin, self.ORIGIN) if not isolated: self.assertTrue(issubclass(mod.error, Exception)) self.assertEqual(mod.int_const, 1969) @@ -2611,7 +2620,7 @@ def test_basic_multiple_interpreters_deleted_no_reset(self): # First, load in the main interpreter but then completely clear it. loaded_main = self.load(self.NAME) loaded_main.module._clear_globals() - _testinternalcapi.clear_extension(self.NAME, self.FILE) + _testinternalcapi.clear_extension(self.NAME, self.ORIGIN) # At this point: # * alive in 0 interpreters diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj index c46435c6709714..4389c08ac1960d 100644 --- a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj +++ b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -273,7 +273,7 @@ ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The name of the .dylib file, relative to the install base\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/$INSTALL_BASE/}\n # The full dotted name of the binary module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $RELATIVE_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\" \n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" >> ${FULL_EXT%.so}.fwork\n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; + shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The location of the extension file, relative to the bundle\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n # The path to the extension file, relative to the install base\n PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n # The full dotted name of the extension module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n # Create a back reference to the .so file location in the framework\n echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\" \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; }; /* End PBXShellScriptBuildPhase section */ From 04d1c79b4ebc81413c4d9ebb901a3cc9dc521246 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 19 Mar 2024 08:19:15 +0800 Subject: [PATCH 10/10] Ensure CFBundleShortVersionString meets App Store guidelines. --- iOS/Resources/Info.plist.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iOS/Resources/Info.plist.in b/iOS/Resources/Info.plist.in index 3ecdc894f0a285..52c0a6e7fd7a55 100644 --- a/iOS/Resources/Info.plist.in +++ b/iOS/Resources/Info.plist.in @@ -17,7 +17,7 @@ CFBundlePackageType FMWK CFBundleShortVersionString - %VERSION% + @VERSION@ CFBundleLongVersionString %VERSION%, (c) 2001-2024 Python Software Foundation. CFBundleSignature