From 36a99c44a44b3214d5ce42ccfcd04991e19b1a0c Mon Sep 17 00:00:00 2001 From: Pravin Date: Fri, 8 Mar 2019 19:17:43 +1100 Subject: [PATCH 1/8] Fix libffi linker errors when compiling on WSL Force the use of lld (LLVM's linker), and include a python37 bugfix. https://clang.llvm.org/docs/Toolchain.html https://lld.llvm.org/ https://bz-attachments.freebsd.org/attachment.cgi?id=200526 --- Dockerfile.py3 | 2 +- pythonforandroid/python.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile.py3 b/Dockerfile.py3 index 307f8ddfda..9f54afe59e 100644 --- a/Dockerfile.py3 +++ b/Dockerfile.py3 @@ -106,7 +106,7 @@ RUN dpkg --add-architecture i386 \ build-essential ccache git python3 python3-dev \ libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \ libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 \ - zip zlib1g-dev zlib1g:i386 \ + zip zlib1g-dev zlib1g:i386 lld \ && apt -y autoremove # specific recipes dependencies (e.g. libffi requires autoreconf binary) diff --git a/pythonforandroid/python.py b/pythonforandroid/python.py index afddfe372b..0ef4c2bc56 100755 --- a/pythonforandroid/python.py +++ b/pythonforandroid/python.py @@ -197,6 +197,7 @@ def add_flags(include_flags, link_dirs, link_libs): add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)), ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'), ' -lffi') + env["LDFLAGS"] += ' -L. -fuse-ld=lld' if 'openssl' in self.ctx.recipe_build_order: info('Activating flags for openssl') From f9d7bbb0cc5adb6e7cf667815c7d627277c4b5fd Mon Sep 17 00:00:00 2001 From: Pravin Date: Fri, 8 Mar 2019 21:41:19 +1100 Subject: [PATCH 2/8] Remove lld unsupported fix-cortex-a8 from LDFLAGS https://github.com/android-ndk/ndk/issues/313 https://github.com/android-ndk/ndk/issues/766 https://developer.android.com/ndk/guides/standalone_toolchain --- pythonforandroid/recipes/python3/__init__.py | 2 +- .../python3/patches/remove-fix-cortex-a8.patch | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 pythonforandroid/recipes/python3/patches/remove-fix-cortex-a8.patch diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index e7a9e5403f..34302cb1b3 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -21,7 +21,7 @@ class Python3Recipe(GuestPythonRecipe): url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz' name = 'python3' - patches = ["patches/fix-ctypes-util-find-library.patch"] + patches = ["patches/fix-ctypes-util-find-library.patch", "patches/remove-fix-cortex-a8.patch"] depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi'] conflicts = ['python3crystax', 'python2', 'python2legacy'] diff --git a/pythonforandroid/recipes/python3/patches/remove-fix-cortex-a8.patch b/pythonforandroid/recipes/python3/patches/remove-fix-cortex-a8.patch new file mode 100644 index 0000000000..78137b8ae3 --- /dev/null +++ b/pythonforandroid/recipes/python3/patches/remove-fix-cortex-a8.patch @@ -0,0 +1,12 @@ +diff --git a/configure b/configure +--- a/configure ++++ b/configure +@@ -5671,7 +5671,7 @@ $as_echo_n "checking for the Android arm ABI... " >&6; } + $as_echo "$_arm_arch" >&6; } + if test "$_arm_arch" = 7; then + BASECFLAGS="${BASECFLAGS} -mfloat-abi=softfp -mfpu=vfpv3-d16" +- LDFLAGS="${LDFLAGS} -march=armv7-a -Wl,--fix-cortex-a8" ++ LDFLAGS="${LDFLAGS} -march=armv7-a" + fi + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: not Android" >&5 \ No newline at end of file From 9cfcb03e3b1ef99fadba8f80d251dd995ca43cbc Mon Sep 17 00:00:00 2001 From: Pravin Date: Tue, 12 Mar 2019 19:20:49 +1100 Subject: [PATCH 3/8] Fix libffi recipe, runtime libffi linker errors. Libtool didnt like the space after --sysroot, replaced with = --- pythonforandroid/archs.py | 2 +- pythonforandroid/recipes/libffi/__init__.py | 38 +-------------------- 2 files changed, 2 insertions(+), 38 deletions(-) diff --git a/pythonforandroid/archs.py b/pythonforandroid/archs.py index ec5a6fb3d5..09ebba4fc1 100644 --- a/pythonforandroid/archs.py +++ b/pythonforandroid/archs.py @@ -86,7 +86,7 @@ def get_env(self, with_flags_in_cc=True, clang=False): self.ctx.python_recipe.version[0:3]) ) - env['LDFLAGS'] += '--sysroot {} '.format(self.ctx.ndk_platform) + env['LDFLAGS'] += '--sysroot={} '.format(self.ctx.ndk_platform) env["CXXFLAGS"] = env["CFLAGS"] diff --git a/pythonforandroid/recipes/libffi/__init__.py b/pythonforandroid/recipes/libffi/__init__.py index c1fd4a40c4..ac1cb0f7c1 100644 --- a/pythonforandroid/recipes/libffi/__init__.py +++ b/pythonforandroid/recipes/libffi/__init__.py @@ -37,46 +37,10 @@ def build_arch(self, arch): '--prefix=' + self.get_build_dir(arch.arch), '--disable-builddir', '--enable-shared', _env=env) - # '--with-sysroot={}'.format(self.ctx.ndk_platform), - # '--target={}'.format(arch.toolchain_prefix), - # ndk 15 introduces unified headers required --sysroot and - # -isysroot for libraries and headers. libtool's head explodes - # trying to weave them into it's own magic. The result is a link - # failure trying to link libc. We call make to compile the bits - # and manually link... + shprint(sh.make, '-j5', 'libffi.la', _env=env) - try: - shprint(sh.make, '-j5', 'libffi.la', _env=env) - except sh.ErrorReturnCode_2: - info("make libffi.la failed as expected") - cc = sh.Command(env['CC'].split()[0]) - cflags = env['CC'].split()[1:] host_build = self.get_build_dir(arch.arch) - - arch_flags = '' - if '-march=' in env['CFLAGS']: - arch_flags = '-march={}'.format(env['CFLAGS'].split('-march=')[1]) - - src_arch = arch.command_prefix.split('-')[0] - if src_arch == 'x86_64': - # libffi has not specific arch files for x86_64...so...using - # the ones from x86 which seems to build fine... - src_arch = 'x86' - - cflags.extend(arch_flags.split()) - cflags.extend(['-shared', '-fPIC', '-DPIC']) - cflags.extend(glob(join(host_build, 'src/.libs/*.o'))) - cflags.extend(glob(join(host_build, 'src', src_arch, '.libs/*.o'))) - - ldflags = env['LDFLAGS'].split() - cflags.extend(ldflags) - cflags.extend(['-Wl,-soname', '-Wl,libffi.so', '-o', - '.libs/libffi.so']) - - with current_directory(host_build): - shprint(cc, *cflags, _env=env) - ensure_dir(self.ctx.get_libs_dir(arch.arch)) shprint(sh.cp, join(host_build, '.libs', 'libffi.so'), From f1ca66d2d81e82311cd1cefe125881478636544d Mon Sep 17 00:00:00 2001 From: Pravin Date: Tue, 12 Mar 2019 19:34:20 +1100 Subject: [PATCH 4/8] Appease flake8 --- pythonforandroid/recipes/libffi/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pythonforandroid/recipes/libffi/__init__.py b/pythonforandroid/recipes/libffi/__init__.py index ac1cb0f7c1..30042cbb38 100644 --- a/pythonforandroid/recipes/libffi/__init__.py +++ b/pythonforandroid/recipes/libffi/__init__.py @@ -1,8 +1,7 @@ from os.path import exists, join from pythonforandroid.recipe import Recipe -from pythonforandroid.logger import info, shprint +from pythonforandroid.logger import shprint from pythonforandroid.util import current_directory, ensure_dir -from glob import glob import sh From 696e88fdb5af24d6ae3dc555bd53da1bc4b97815 Mon Sep 17 00:00:00 2001 From: Pravin Date: Thu, 21 Mar 2019 00:56:09 +1100 Subject: [PATCH 5/8] CRAs - Check for lld before using it. Changed hardcoded cpu count to cpu_count(). Added comment to python patch. Made remove-fix-cortex-a8.patch conditional on lld. Moved LDFLAGS change to get_recipe_env() so its not libffi specific within python.py. --- pythonforandroid/python.py | 10 +++++++++- pythonforandroid/recipes/libffi/__init__.py | 3 ++- pythonforandroid/recipes/python3/__init__.py | 7 ++++++- .../recipes/python3/patches/remove-fix-cortex-a8.patch | 2 ++ 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/pythonforandroid/python.py b/pythonforandroid/python.py index 0ef4c2bc56..9e3b4b715a 100755 --- a/pythonforandroid/python.py +++ b/pythonforandroid/python.py @@ -170,6 +170,15 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): env['SYSROOT'] = sysroot + exitcode, _ = subprocess.getstatusoutput('which lld') + if exitcode == 0: + # Note: The -L. is to fix a bug in python 3.7. + # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409 + env["LDFLAGS"] += ' -L. -fuse-ld=lld' + else: + logger.warning('lld not found, linking without it. ' + + 'Consider installing lld if linker errors occur.') + return env def set_libs_flags(self, env, arch): @@ -197,7 +206,6 @@ def add_flags(include_flags, link_dirs, link_libs): add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)), ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'), ' -lffi') - env["LDFLAGS"] += ' -L. -fuse-ld=lld' if 'openssl' in self.ctx.recipe_build_order: info('Activating flags for openssl') diff --git a/pythonforandroid/recipes/libffi/__init__.py b/pythonforandroid/recipes/libffi/__init__.py index 30042cbb38..31ed9c69b5 100644 --- a/pythonforandroid/recipes/libffi/__init__.py +++ b/pythonforandroid/recipes/libffi/__init__.py @@ -1,4 +1,5 @@ from os.path import exists, join +from multiprocessing import cpu_count from pythonforandroid.recipe import Recipe from pythonforandroid.logger import shprint from pythonforandroid.util import current_directory, ensure_dir @@ -37,7 +38,7 @@ def build_arch(self, arch): '--disable-builddir', '--enable-shared', _env=env) - shprint(sh.make, '-j5', 'libffi.la', _env=env) + shprint(sh.make, '-j', str(cpu_count()), 'libffi.la', _env=env) host_build = self.get_build_dir(arch.arch) ensure_dir(self.ctx.get_libs_dir(arch.arch)) diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index 34302cb1b3..ab2064a68d 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -1,3 +1,4 @@ +import subprocess from pythonforandroid.python import GuestPythonRecipe from pythonforandroid.recipe import Recipe @@ -21,7 +22,11 @@ class Python3Recipe(GuestPythonRecipe): url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz' name = 'python3' - patches = ["patches/fix-ctypes-util-find-library.patch", "patches/remove-fix-cortex-a8.patch"] + patches = ["patches/fix-ctypes-util-find-library.patch"] + + exitcode, _ = subprocess.getstatusoutput('which lld') + if exitcode == 0: + patches = patches + ["patches/remove-fix-cortex-a8.patch"] depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi'] conflicts = ['python3crystax', 'python2', 'python2legacy'] diff --git a/pythonforandroid/recipes/python3/patches/remove-fix-cortex-a8.patch b/pythonforandroid/recipes/python3/patches/remove-fix-cortex-a8.patch index 78137b8ae3..5ddc3c432f 100644 --- a/pythonforandroid/recipes/python3/patches/remove-fix-cortex-a8.patch +++ b/pythonforandroid/recipes/python3/patches/remove-fix-cortex-a8.patch @@ -1,3 +1,5 @@ +This patch removes --fix-cortex-a8 from the linker flags in order to support linking +with lld, as lld does not support this flag (https://github.com/android-ndk/ndk/issues/766). diff --git a/configure b/configure --- a/configure +++ b/configure From 93d2e12063e40523a7342763b0e8c3dedd3ae080 Mon Sep 17 00:00:00 2001 From: Pravin Date: Thu, 21 Mar 2019 01:31:46 +1100 Subject: [PATCH 6/8] Replaced subprocess.getstatusoutput with sh.which for python2 compatibility. --- pythonforandroid/python.py | 3 +-- pythonforandroid/recipes/python3/__init__.py | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pythonforandroid/python.py b/pythonforandroid/python.py index 9e3b4b715a..23db0bcd3e 100755 --- a/pythonforandroid/python.py +++ b/pythonforandroid/python.py @@ -170,8 +170,7 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): env['SYSROOT'] = sysroot - exitcode, _ = subprocess.getstatusoutput('which lld') - if exitcode == 0: + if sh.which('lld') is not None: # Note: The -L. is to fix a bug in python 3.7. # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409 env["LDFLAGS"] += ' -L. -fuse-ld=lld' diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index ab2064a68d..c6d5ba5e50 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -1,4 +1,4 @@ -import subprocess +import sh from pythonforandroid.python import GuestPythonRecipe from pythonforandroid.recipe import Recipe @@ -24,8 +24,7 @@ class Python3Recipe(GuestPythonRecipe): patches = ["patches/fix-ctypes-util-find-library.patch"] - exitcode, _ = subprocess.getstatusoutput('which lld') - if exitcode == 0: + if sh.which('lld') is not None: patches = patches + ["patches/remove-fix-cortex-a8.patch"] depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi'] From 0c2a2c52575a4569db99e7246f6cd7e6ec3534f8 Mon Sep 17 00:00:00 2001 From: Pravin Date: Thu, 21 Mar 2019 01:38:58 +1100 Subject: [PATCH 7/8] Seriously tox? --- pythonforandroid/python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonforandroid/python.py b/pythonforandroid/python.py index 23db0bcd3e..3a214ee714 100755 --- a/pythonforandroid/python.py +++ b/pythonforandroid/python.py @@ -171,7 +171,7 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): env['SYSROOT'] = sysroot if sh.which('lld') is not None: - # Note: The -L. is to fix a bug in python 3.7. + # Note: The -L. is to fix a bug in python 3.7. # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409 env["LDFLAGS"] += ' -L. -fuse-ld=lld' else: From 8202e4d98816b0d009e42c92e17a829936afe257 Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Wed, 20 Mar 2019 23:52:09 +0100 Subject: [PATCH 8/8] Update Dockerfile revert lld dep --- Dockerfile.py3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.py3 b/Dockerfile.py3 index 9f54afe59e..307f8ddfda 100644 --- a/Dockerfile.py3 +++ b/Dockerfile.py3 @@ -106,7 +106,7 @@ RUN dpkg --add-architecture i386 \ build-essential ccache git python3 python3-dev \ libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \ libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 \ - zip zlib1g-dev zlib1g:i386 lld \ + zip zlib1g-dev zlib1g:i386 \ && apt -y autoremove # specific recipes dependencies (e.g. libffi requires autoreconf binary)