Skip to content

Commit 7ec64aa

Browse files
opacaminclement
opacam
authored and
inclement
committed
[LIBS - PART II] Part II of NDK r19 migration - Initial STL lib migration (#1947)
* [recipe-stl] Add android's STL lib support to `Recipe` To allow us to refactor some common operations that we use in our recipes that depends on android's STL library. Note: This commit will allow us to begin the migration to `c++_shared`. This is a must when we move to android's NDK r19+, because as for android NDK >= 18 is the only one supported STL library. * [recipe-stl] Make CppCompiledComponentsPythonRecipe use Recipe's STL support * [recipe-stl] Make icu a library recipe with STL support (rework) Also done here:   - Remove hardcoded version in url   - Disable versioned shared libraries   - Make it to be build as a shared libraries (instead of static)   - Disable the build of static libraries (because we build them as shared ones, so we avoid to link with them without our consents)   - Shorten long lines to be pep8's friendly   - Remove icu from ci/constants   - Remove `RuntimeError` about the need to use NDK api <= 19 (because that is not the case anymore)   - consider host's number of cpu to perform the build * [recipe-stl] Rework pyicu recipe to match latest icu changes Also done here:   - Remove icu.patch because now we don't have the version in our icu libraries   - Shorten long lines to be pep8's friendly * [tests] Add tests for recipe with STL support * [tests] Add tests for icu recipe * [tests] Add tests for pyicu recipe
1 parent f6df2bd commit 7ec64aa

File tree

9 files changed

+563
-168
lines changed

9 files changed

+563
-168
lines changed

‎ci/constants.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,6 @@ class TargetPython(Enum):
6565
# IndexError: list index out of range
6666
'secp256k1',
6767
'ffpyplayer',
68-
'icu',
6968
# requires `libpq-dev` system dependency e.g. for `pg_config` binary
7069
'psycopg2',
7170
'protobuf_cpp',

‎pythonforandroid/recipe.py

Lines changed: 60 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,46 @@ class Recipe(with_metaclass(RecipeMeta)):
125125
path: `'.', None or ''`
126126
"""
127127

128+
need_stl_shared = False
129+
'''Some libraries or python packages may need to be linked with android's
130+
stl. We can automatically do this for any recipe if we set this property to
131+
`True`'''
132+
133+
stl_lib_name = 'c++_shared'
134+
'''
135+
The default STL shared lib to use: `c++_shared`.
136+
137+
.. note:: Android NDK version > 17 only supports 'c++_shared', because
138+
starting from NDK r18 the `gnustl_shared` lib has been deprecated.
139+
'''
140+
141+
stl_lib_source = '{ctx.ndk_dir}/sources/cxx-stl/llvm-libc++'
142+
'''
143+
The source directory of the selected stl lib, defined in property
144+
`stl_lib_name`
145+
'''
146+
147+
@property
148+
def stl_include_dir(self):
149+
return join(self.stl_lib_source.format(ctx=self.ctx), 'include')
150+
151+
def get_stl_lib_dir(self, arch):
152+
return join(
153+
self.stl_lib_source.format(ctx=self.ctx), 'libs', arch.arch
154+
)
155+
156+
def get_stl_library(self, arch):
157+
return join(
158+
self.get_stl_lib_dir(arch),
159+
'lib{name}.so'.format(name=self.stl_lib_name),
160+
)
161+
162+
def install_stl_lib(self, arch):
163+
if not self.ctx.has_lib(
164+
arch.arch, 'lib{name}.so'.format(name=self.stl_lib_name)
165+
):
166+
self.install_libs(arch, self.get_stl_library(arch))
167+
128168
@property
129169
def version(self):
130170
key = 'VERSION_' + self.name
@@ -454,7 +494,22 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True):
454494
"""
455495
if arch is None:
456496
arch = self.filtered_archs[0]
457-
return arch.get_env(with_flags_in_cc=with_flags_in_cc)
497+
env = arch.get_env(with_flags_in_cc=with_flags_in_cc)
498+
499+
if self.need_stl_shared:
500+
env['CPPFLAGS'] = env.get('CPPFLAGS', '')
501+
env['CPPFLAGS'] += ' -I{}'.format(self.stl_include_dir)
502+
503+
env['CXXFLAGS'] = env['CFLAGS'] + ' -frtti -fexceptions'
504+
505+
if with_flags_in_cc:
506+
env['CXX'] += ' -frtti -fexceptions'
507+
508+
env['LDFLAGS'] += ' -L{}'.format(self.get_stl_lib_dir(arch))
509+
env['LIBS'] = env.get('LIBS', '') + " -l{}".format(
510+
self.stl_lib_name
511+
)
512+
return env
458513

459514
def prebuild_arch(self, arch):
460515
'''Run any pre-build tasks for the Recipe. By default, this checks if
@@ -538,6 +593,9 @@ def postbuild_arch(self, arch):
538593
if hasattr(self, postbuild):
539594
getattr(self, postbuild)()
540595

596+
if self.need_stl_shared:
597+
self.install_stl_lib(arch)
598+
541599
def prepare_build_dir(self, arch):
542600
'''Copies the recipe data into a build dir for the given arch. By
543601
default, this unpacks a downloaded recipe. You should override
@@ -982,35 +1040,7 @@ def rebuild_compiled_components(self, arch, env):
9821040
class CppCompiledComponentsPythonRecipe(CompiledComponentsPythonRecipe):
9831041
""" Extensions that require the cxx-stl """
9841042
call_hostpython_via_targetpython = False
985-
986-
def get_recipe_env(self, arch):
987-
env = super(CppCompiledComponentsPythonRecipe, self).get_recipe_env(arch)
988-
keys = dict(
989-
ctx=self.ctx,
990-
arch=arch,
991-
arch_noeabi=arch.arch.replace('eabi', '')
992-
)
993-
env['LDSHARED'] = env['CC'] + ' -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions'
994-
env['CFLAGS'] += (
995-
" -I{ctx.ndk_dir}/platforms/android-{ctx.android_api}/arch-{arch_noeabi}/usr/include" +
996-
" -I{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/include" +
997-
" -I{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}/include").format(**keys)
998-
env['CXXFLAGS'] = env['CFLAGS'] + ' -frtti -fexceptions'
999-
env['LDFLAGS'] += (
1000-
" -L{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}" +
1001-
" -lgnustl_shared").format(**keys)
1002-
1003-
return env
1004-
1005-
def build_compiled_components(self, arch):
1006-
super(CppCompiledComponentsPythonRecipe, self).build_compiled_components(arch)
1007-
1008-
# Copy libgnustl_shared.so
1009-
with current_directory(self.get_build_dir(arch.arch)):
1010-
sh.cp(
1011-
"{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}/libgnustl_shared.so".format(ctx=self.ctx, arch=arch),
1012-
self.ctx.get_libs_dir(arch.arch)
1013-
)
1043+
need_stl_shared = True
10141044

10151045

10161046
class CythonRecipe(PythonRecipe):

‎pythonforandroid/recipes/icu/__init__.py

Lines changed: 47 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,54 @@
11
import sh
22
import os
3-
from os.path import join, isdir
4-
from pythonforandroid.recipe import NDKRecipe
3+
from os.path import join, isdir, exists
4+
from multiprocessing import cpu_count
5+
from pythonforandroid.recipe import Recipe
56
from pythonforandroid.toolchain import shprint
67
from pythonforandroid.util import current_directory, ensure_dir
78

89

9-
class ICURecipe(NDKRecipe):
10+
class ICURecipe(Recipe):
1011
name = 'icu4c'
1112
version = '57.1'
12-
url = 'http://download.icu-project.org/files/icu4c/57.1/icu4c-57_1-src.tgz'
13+
major_version = version.split('.')[0]
14+
url = ('http://download.icu-project.org/files/icu4c/'
15+
'{version}/icu4c-{version_underscore}-src.tgz')
1316

1417
depends = [('hostpython2', 'hostpython3')] # installs in python
15-
generated_libraries = [
16-
'libicui18n.so', 'libicuuc.so', 'libicudata.so', 'libicule.so']
17-
18-
def get_lib_dir(self, arch):
19-
lib_dir = join(self.ctx.get_python_install_dir(), "lib")
20-
ensure_dir(lib_dir)
21-
return lib_dir
22-
23-
def prepare_build_dir(self, arch):
24-
if self.ctx.android_api > 19:
25-
# greater versions do not have /usr/include/sys/exec_elf.h
26-
raise RuntimeError("icu needs an android api <= 19")
27-
28-
super(ICURecipe, self).prepare_build_dir(arch)
29-
30-
def build_arch(self, arch, *extra_args):
18+
patches = ['disable-libs-version.patch']
19+
20+
built_libraries = {
21+
'libicui18n{}.so'.format(major_version): 'build_icu_android/lib',
22+
'libicuuc{}.so'.format(major_version): 'build_icu_android/lib',
23+
'libicudata{}.so'.format(major_version): 'build_icu_android/lib',
24+
'libicule{}.so'.format(major_version): 'build_icu_android/lib',
25+
'libicuio{}.so'.format(major_version): 'build_icu_android/lib',
26+
'libicutu{}.so'.format(major_version): 'build_icu_android/lib',
27+
'libiculx{}.so'.format(major_version): 'build_icu_android/lib',
28+
}
29+
need_stl_shared = True
30+
31+
@property
32+
def versioned_url(self):
33+
if self.url is None:
34+
return None
35+
return self.url.format(
36+
version=self.version,
37+
version_underscore=self.version.replace('.', '_'))
38+
39+
def get_recipe_dir(self):
40+
"""
41+
.. note:: We need to overwrite `Recipe.get_recipe_dir` due to the
42+
mismatch name between the recipe's folder (icu) and the value
43+
of `ICURecipe.name` (icu4c).
44+
"""
45+
if self.ctx.local_recipes is not None:
46+
local_recipe_dir = join(self.ctx.local_recipes, 'icu')
47+
if exists(local_recipe_dir):
48+
return local_recipe_dir
49+
return join(self.ctx.root_dir, 'recipes', 'icu')
50+
51+
def build_arch(self, arch):
3152
env = self.get_recipe_env(arch).copy()
3253
build_root = self.get_build_dir(arch.arch)
3354

@@ -60,74 +81,35 @@ def make_build_dest(dest):
6081
"--prefix="+icu_build,
6182
"--enable-extras=no",
6283
"--enable-strict=no",
63-
"--enable-static",
84+
"--enable-static=no",
6485
"--enable-tests=no",
6586
"--enable-samples=no",
6687
_env=host_env)
67-
shprint(sh.make, "-j5", _env=host_env)
88+
shprint(sh.make, "-j", str(cpu_count()), _env=host_env)
6889
shprint(sh.make, "install", _env=host_env)
6990

7091
build_android, exists = make_build_dest("build_icu_android")
7192
if not exists:
7293

7394
configure = sh.Command(join(build_root, "source", "configure"))
7495

75-
include = (
76-
" -I{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/include/"
77-
" -I{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/"
78-
"{arch}/include")
79-
include = include.format(ndk=self.ctx.ndk_dir,
80-
version=env["TOOLCHAIN_VERSION"],
81-
arch=arch.arch)
82-
env["CPPFLAGS"] = env["CXXFLAGS"] + " "
83-
env["CPPFLAGS"] += host_env["CPPFLAGS"]
84-
env["CPPFLAGS"] += include
85-
86-
lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}"
87-
lib = lib.format(ndk=self.ctx.ndk_dir,
88-
version=env["TOOLCHAIN_VERSION"],
89-
arch=arch.arch)
90-
env["LDFLAGS"] += " -lgnustl_shared -L"+lib
91-
92-
env.pop("CFLAGS", None)
93-
env.pop("CXXFLAGS", None)
94-
9596
with current_directory(build_android):
9697
shprint(
9798
configure,
9899
"--with-cross-build="+build_linux,
99100
"--enable-extras=no",
100101
"--enable-strict=no",
101-
"--enable-static",
102+
"--enable-static=no",
102103
"--enable-tests=no",
103104
"--enable-samples=no",
104105
"--host="+env["TOOLCHAIN_PREFIX"],
105106
"--prefix="+icu_build,
106107
_env=env)
107-
shprint(sh.make, "-j5", _env=env)
108+
shprint(sh.make, "-j", str(cpu_count()), _env=env)
108109
shprint(sh.make, "install", _env=env)
109110

110-
self.copy_files(arch)
111-
112-
def copy_files(self, arch):
113-
env = self.get_recipe_env(arch)
114-
115-
lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}"
116-
lib = lib.format(ndk=self.ctx.ndk_dir,
117-
version=env["TOOLCHAIN_VERSION"],
118-
arch=arch.arch)
119-
stl_lib = join(lib, "libgnustl_shared.so")
120-
dst_dir = join(self.ctx.get_site_packages_dir(), "..", "lib-dynload")
121-
shprint(sh.cp, stl_lib, dst_dir)
122-
123-
src_lib = join(self.get_build_dir(arch.arch), "icu_build", "lib")
124-
dst_lib = self.get_lib_dir(arch)
125-
126-
src_suffix = "." + self.version
127-
dst_suffix = "." + self.version.split(".")[0] # main version
128-
for lib in self.generated_libraries:
129-
shprint(sh.cp, join(src_lib, lib+src_suffix),
130-
join(dst_lib, lib+dst_suffix))
111+
def install_libraries(self, arch):
112+
super(ICURecipe, self).install_libraries(arch)
131113

132114
src_include = join(
133115
self.get_build_dir(arch.arch), "icu_build", "include")
@@ -137,16 +119,5 @@ def copy_files(self, arch):
137119
shprint(sh.cp, "-r", join(src_include, "layout"), dst_include)
138120
shprint(sh.cp, "-r", join(src_include, "unicode"), dst_include)
139121

140-
# copy stl library
141-
lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}"
142-
lib = lib.format(ndk=self.ctx.ndk_dir,
143-
version=env["TOOLCHAIN_VERSION"],
144-
arch=arch.arch)
145-
stl_lib = join(lib, "libgnustl_shared.so")
146-
147-
dst_dir = join(self.ctx.get_python_install_dir(), "lib")
148-
ensure_dir(dst_dir)
149-
shprint(sh.cp, stl_lib, dst_dir)
150-
151122

152123
recipe = ICURecipe()

‎pythonforandroid/recipes/icu/disable-libs-version.patch

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
diff -aur icu4c-org/source/config/Makefile.inc.in icu4c/source/config/Makefile.inc.in
2+
--- icu/source/config/Makefile.inc.in.orig 2016-03-23 21:50:50.000000000 +0100
3+
+++ icu/source/config/Makefile.inc.in 2019-02-15 17:59:28.331749766 +0100
4+
@@ -142,8 +142,8 @@
5+
LDLIBRARYPATH_ENVVAR = LD_LIBRARY_PATH
6+
7+
# Versioned target for a shared library
8+
-FINAL_SO_TARGET = $(SO_TARGET).$(SO_TARGET_VERSION)
9+
-MIDDLE_SO_TARGET = $(SO_TARGET).$(SO_TARGET_VERSION_MAJOR)
10+
+FINAL_SO_TARGET = $(SO_TARGET).$(SO_TARGET_VERSION)
11+
+MIDDLE_SO_TARGET = $(SO_TARGET)
12+
13+
# Access to important ICU tools.
14+
# Use as follows: $(INVOKE) $(GENRB) arguments ..
15+
diff -aur icu4c-org/source/config/mh-linux icu4c/source/config/mh-linux
16+
--- icu4c-org/source/config/mh-linux 2017-07-05 13:23:06.000000000 +0200
17+
+++ icu4c/source/config/mh-linux 2017-07-06 14:02:52.275016858 +0200
18+
@@ -24,9 +24,17 @@
19+
20+
## Compiler switch to embed a library name
21+
# The initial tab in the next line is to prevent icu-config from reading it.
22+
- LD_SONAME = -Wl,-soname -Wl,$(notdir $(MIDDLE_SO_TARGET))
23+
+ LD_SONAME = -Wl,-soname -Wl,$(notdir $(SO_TARGET))
24+
+ DATA_STUBNAME = data$(SO_TARGET_VERSION_MAJOR)
25+
+ COMMON_STUBNAME = uc$(SO_TARGET_VERSION_MAJOR)
26+
+ I18N_STUBNAME = i18n$(SO_TARGET_VERSION_MAJOR)
27+
+ LAYOUT_STUBNAME = le$(SO_TARGET_VERSION_MAJOR)
28+
+ LAYOUTEX_STUBNAME = lx$(SO_TARGET_VERSION_MAJOR)
29+
+ IO_STUBNAME = io$(SO_TARGET_VERSION_MAJOR)
30+
+ TOOLUTIL_STUBNAME = tu$(SO_TARGET_VERSION_MAJOR)
31+
+ CTESTFW_STUBNAME = test$(SO_TARGET_VERSION_MAJOR)
32+
#SH# # We can't depend on MIDDLE_SO_TARGET being set.
33+
-#SH# LD_SONAME=
34+
+#SH# LD_SONAME=$(SO_TARGET)
35+
36+
## Shared library options
37+
LD_SOOPTIONS= -Wl,-Bsymbolic
38+
@@ -64,10 +64,10 @@
39+
40+
## Versioned libraries rules
41+
42+
-%.$(SO).$(SO_TARGET_VERSION_MAJOR): %.$(SO).$(SO_TARGET_VERSION)
43+
- $(RM) $@ && ln -s ${<F} $@
44+
-%.$(SO): %.$(SO).$(SO_TARGET_VERSION_MAJOR)
45+
- $(RM) $@ && ln -s ${*F}.$(SO).$(SO_TARGET_VERSION) $@
46+
+%.$(SO).$(SO_TARGET_VERSION_MAJOR): %.$(SO).$(SO_TARGET_VERSION)
47+
+ $(RM) $@ && ln -s ${<F} $@
48+
+%.$(SO): %.$(SO).$(SO_TARGET_VERSION_MAJOR)
49+
+ $(RM) $@ && ln -s ${*F}.$(SO).$(SO_TARGET_VERSION) $@
50+
51+
## Bind internal references
52+
53+
diff -aur icu4c-org/source/data/Makefile.in icu4c/source/data/Makefile.in
54+
--- icu4c-org/source/data/Makefile.in 2017-07-05 13:23:06.000000000 +0200
55+
+++ icu4c/source/data/Makefile.in 2017-07-06 14:05:31.607995855 +0200
56+
@@ -24,9 +24,9 @@
57+
ifeq ($(PKGDATA_OPTS),)
58+
PKGDATA_OPTS = -O $(top_builddir)/data/icupkg.inc
59+
endif
60+
-ifeq ($(PKGDATA_VERSIONING),)
61+
-PKGDATA_VERSIONING = -r $(SO_TARGET_VERSION)
62+
-endif
63+
+#ifeq ($(PKGDATA_VERSIONING),)
64+
+#PKGDATA_VERSIONING = -r $(SO_TARGET_VERSION)
65+
+#endif
66+

0 commit comments

Comments
 (0)