Skip to content

Commit 15b82a7

Browse files
opacamopacam
committed
Improve the libraries loading mechanism when initializing the apk
The old libraries loading mechanism, has become inefficient because, over the years, more library recipes has been added to p4a project as well as more python versions. So...here we introduce a new mechanism which creates a file with the needed libraries, which will be read at runtime and load the libraries defined in there, so we avoid checking each library against a pattern as the old method. This new mechanism is still incomplete, because still lacks most of the libraries, but has the same functionality, because all the basic libraries used in the old loading mechanism are inside the new mechanism, plus some others which where not handled before like the shared stl libs. With this new mechanism we will have the advantage that will be a lot easier to add the necessary libraries for all the bootstraps at once.
1 parent 9bda7a8 commit 15b82a7

File tree

13 files changed

+190
-160
lines changed

13 files changed

+190
-160
lines changed

‎pythonforandroid/bootstrap.py

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ def copy_files(src_root, dest_root, override=True):
3636
class Bootstrap(object):
3737
'''An Android project template, containing recipe stuff for
3838
compilation and templated fields for APK info.
39+
40+
.. versionchanged:: 0.6.0
41+
Adds attribute :attr:`libraries_to_load` and method
42+
:meth:`collect_libraries_to_load`.
3943
'''
4044
name = ''
4145
jni_subdir = '/jni'
@@ -56,6 +60,11 @@ class Bootstrap(object):
5660
from Bootstrap.get_bootstrap_from_recipes.
5761
'''
5862

63+
libraries_to_load = []
64+
'''The list of libraries that should be loaded by android on python's app
65+
initialization. This list will be dynamically created at
66+
:meth:`collect_libraries_to_load` depending on the recipes build order.'''
67+
5968
# Other things a Bootstrap might need to track (maybe separately):
6069
# ndk_main.c
6170
# whitelist.txt
@@ -257,6 +266,64 @@ def _unpack_aar(self, aar, arch):
257266
for f in so_files:
258267
shprint(sh.cp, '-a', f, so_tgt_dir)
259268

269+
def collect_libraries_to_load(self, arch):
270+
'''
271+
Collect the shared libraries needed to load when apk initialises and
272+
returns a list. This list will be used to create a file in our
273+
distribution's assets folder which will be used by our java loading
274+
mechanism to load those shared libraries at runtime.
275+
276+
.. warning:: the loading order matters, this cannot be arbitrary,
277+
for example, if some lib has been build against our python, then
278+
needs to be loaded after the python library has been loaded.
279+
'''
280+
info('Collect compiled shared libraries to load at runtime')
281+
282+
libs_to_load = []
283+
# Add libs that should be loaded before bootstrap libs (here goes some
284+
# special shared libs, like stl libs used at compile time or libs that
285+
# the bootstrap may be dependant).
286+
# Todo: All the recipes using gnustl_shared should be migrated to use
287+
# c++_shared because in ndk r18 it will be the only stl available:
288+
# https://developer.android.com/ndk/guides/cpp-support
289+
stl_gnu_shared_recipes = (
290+
'boost', 'icu', 'leveldb', 'libzmq', 'protobuf_cpp',
291+
# CppCompiledPythonRecipes depends on gnustl_shared so...
292+
'atom', 'kiwisolver',)
293+
stl_cxx_shared_recipes = ()
294+
for recipe_name in self.ctx.recipe_build_order:
295+
if 'c++_shared' not in libs_to_load and recipe_name in stl_cxx_shared_recipes:
296+
libs_to_load.append('c++_shared')
297+
if 'gnustl_shared' not in libs_to_load and recipe_name in stl_gnu_shared_recipes:
298+
libs_to_load.append('gnustl_shared')
299+
if all(lib in ('c++_shared', 'gnustl_shared') for lib in libs_to_load):
300+
break
301+
302+
# Add crystax if necessary
303+
if self.ctx.python_recipe.from_crystax:
304+
libs_to_load.append('crystax')
305+
306+
# Add libraries that python depends on
307+
if 'sqlite3' in self.ctx.recipe_build_order:
308+
libs_to_load.append('sqlite3')
309+
if 'ffi' in self.ctx.recipe_build_order:
310+
libs_to_load.append('ffi')
311+
if 'openssl' in self.ctx.recipe_build_order:
312+
recipe = Recipe.get_recipe('openssl', self.ctx)
313+
libs_to_load.append('ssl' + recipe.version)
314+
libs_to_load.append('crypto' + recipe.version)
315+
316+
# Add the bootstrap libs
317+
[libs_to_load.append(i) for i in self.libraries_to_load]
318+
319+
# Add the corresponding python lib
320+
python_version = self.ctx.python_recipe.major_minor_version_string
321+
if python_version[0] == '3':
322+
python_version += 'm'
323+
libs_to_load.append('python' + python_version)
324+
325+
return libs_to_load
326+
260327
def strip_libraries(self, arch):
261328
info('Stripping libraries')
262329
if self.ctx.python_recipe.from_crystax:

‎pythonforandroid/bootstraps/sdl2/__init__.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ class SDL2GradleBootstrap(Bootstrap):
1010

1111
recipe_depends = ['sdl2', ('python2', 'python3', 'python3crystax')]
1212

13+
libraries_to_load = ['SDL2', 'SDL2_image', 'SDL2_mixer', 'SDL2_ttf']
14+
1315
def run_distribute(self):
1416
info_main("# Creating Android project ({})".format(self.name))
1517

@@ -47,6 +49,13 @@ def run_distribute(self):
4749
with open('blacklist.txt', 'a') as fileh:
4850
fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n')
4951

52+
# collect libs to load by android os at runtime
53+
libs = self.collect_libraries_to_load(arch)
54+
target_file = join('src', 'main', 'assets', 'libraries_to_load.txt')
55+
with open(target_file, 'a') as libs_file:
56+
for lib in libs:
57+
libs_file.write(lib + '\n')
58+
5059
self.strip_libraries(arch)
5160
self.fry_eggs(site_packages_dir)
5261
super(SDL2GradleBootstrap, self).run_distribute()

‎pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,9 +74,7 @@ protected void onCreate(Bundle savedInstanceState) {
7474
}
7575

7676
public void loadLibraries() {
77-
String app_root = new String(getAppRoot());
78-
File app_root_file = new File(app_root);
79-
PythonUtil.loadLibraries(app_root_file);
77+
PythonUtil.loadLibraries(this);
8078
}
8179

8280
public void recursiveDelete(File f) {

‎pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonService.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import android.os.Process;
1515
import java.io.File;
1616

17+
import org.kivy.android.PythonActivity;
1718
import org.kivy.android.PythonUtil;
1819

1920
import org.renpy.android.Hardware;
@@ -142,9 +143,7 @@ public void onTaskRemoved(Intent rootIntent) {
142143

143144
@Override
144145
public void run(){
145-
String app_root = getFilesDir().getAbsolutePath() + "/app";
146-
File app_root_file = new File(app_root);
147-
PythonUtil.loadLibraries(app_root_file);
146+
PythonUtil.loadLibraries(PythonActivity.mActivity);
148147
this.mService = this;
149148
nativeStart(
150149
androidPrivate, androidArgument,

‎pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonUtil.java

Lines changed: 30 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -3,61 +3,51 @@
33
import java.io.File;
44

55
import android.util.Log;
6+
import android.app.Activity;
7+
import android.content.res.AssetManager;
8+
69
import java.util.ArrayList;
7-
import java.io.FilenameFilter;
8-
import java.util.regex.Pattern;
10+
import java.io.BufferedReader;
11+
import java.io.InputStreamReader;
12+
import java.io.FileNotFoundException;
13+
import java.io.IOException;
914

1015
public class PythonUtil {
1116
private static final String TAG = "pythonutil";
1217

13-
protected static void addLibraryIfExists(ArrayList<String> libsList, String pattern, File libsDir) {
14-
// pattern should be the name of the lib file, without the
15-
// preceding "lib" or suffix ".so", for instance "ssl.*" will
16-
// match files of the form "libssl.*.so".
17-
File [] files = libsDir.listFiles();
18+
protected static ArrayList<String> getLibraries(Activity activity) {
19+
AssetManager assets = activity.getAssets();
1820

19-
pattern = "lib" + pattern + "\\.so";
20-
Pattern p = Pattern.compile(pattern);
21-
for (int i = 0; i < files.length; ++i) {
22-
File file = files[i];
23-
String name = file.getName();
24-
Log.v(TAG, "Checking pattern " + pattern + " against " + name);
25-
if (p.matcher(name).matches()) {
26-
Log.v(TAG, "Pattern " + pattern + " matched file " + name);
27-
libsList.add(name.substring(3, name.length() - 3));
21+
StringBuilder sb = new StringBuilder();
22+
ArrayList<String> libsList = new ArrayList<String>();
23+
BufferedReader reader = null;
24+
try {
25+
reader = new BufferedReader(
26+
new InputStreamReader(assets.open("libraries_to_load.txt")));
27+
String mLine;
28+
while ((mLine = reader.readLine()) != null) {
29+
libsList.add(mLine);
30+
}
31+
} catch (IOException e) {
32+
Log.v(TAG, "Error reading Libraries file...no libs will be added");
33+
} finally {
34+
if (reader != null) {
35+
try {
36+
reader.close();
37+
} catch (IOException e) {
38+
Log.v(TAG, "Error on closing libraries file...going on");
39+
}
2840
}
2941
}
30-
}
31-
32-
protected static ArrayList<String> getLibraries(File filesDir) {
33-
34-
String libsDirPath = filesDir.getParentFile().getParentFile().getAbsolutePath() + "/lib/";
35-
File libsDir = new File(libsDirPath);
36-
37-
ArrayList<String> libsList = new ArrayList<String>();
38-
addLibraryIfExists(libsList, "crystax", libsDir);
39-
addLibraryIfExists(libsList, "sqlite3", libsDir);
40-
addLibraryIfExists(libsList, "ffi", libsDir);
41-
libsList.add("SDL2");
42-
libsList.add("SDL2_image");
43-
libsList.add("SDL2_mixer");
44-
libsList.add("SDL2_ttf");
45-
addLibraryIfExists(libsList, "ssl.*", libsDir);
46-
addLibraryIfExists(libsList, "crypto.*", libsDir);
47-
libsList.add("python2.7");
48-
libsList.add("python3.5m");
49-
libsList.add("python3.6m");
50-
libsList.add("python3.7m");
5142
libsList.add("main");
5243
return libsList;
5344
}
5445

55-
public static void loadLibraries(File filesDir) {
46+
public static void loadLibraries(Activity activity) {
5647

57-
String filesDirPath = filesDir.getAbsolutePath();
5848
boolean foundPython = false;
5949

60-
for (String lib : getLibraries(filesDir)) {
50+
for (String lib : getLibraries(activity)) {
6151
Log.v(TAG, "Loading library: " + lib);
6252
try {
6353
System.loadLibrary(lib);
@@ -80,20 +70,6 @@ public static void loadLibraries(File filesDir) {
8070
}
8171
}
8272

83-
try {
84-
System.load(filesDirPath + "/lib/python2.7/lib-dynload/_io.so");
85-
System.load(filesDirPath + "/lib/python2.7/lib-dynload/unicodedata.so");
86-
} catch(UnsatisfiedLinkError e) {
87-
Log.v(TAG, "Failed to load _io.so or unicodedata.so...but that's okay.");
88-
}
89-
90-
try {
91-
// System.loadLibrary("ctypes");
92-
System.load(filesDirPath + "/lib/python2.7/lib-dynload/_ctypes.so");
93-
} catch(UnsatisfiedLinkError e) {
94-
Log.v(TAG, "Unsatisfied linker when loading ctypes");
95-
}
96-
9773
Log.v(TAG, "Loaded everything!");
9874
}
9975
}

‎pythonforandroid/bootstraps/service_only/__init__.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,13 @@ def run_distribute(self):
4343
with open('blacklist.txt', 'a') as fileh:
4444
fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n')
4545

46+
# collect libs to load by android os at runtime
47+
libs = self.collect_libraries_to_load(arch)
48+
target_file = join('src', 'main', 'assets', 'libraries_to_load.txt')
49+
with open(target_file, 'a') as libs_file:
50+
for lib in libs:
51+
libs_file.write(lib + '\n')
52+
4653
self.strip_libraries(arch)
4754
self.fry_eggs(site_packages_dir)
4855
super(ServiceOnlyBootstrap, self).run_distribute()

‎pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -173,9 +173,7 @@ public void onClick(DialogInterface dialog,int id) {
173173
}
174174

175175
public void loadLibraries() {
176-
String app_root = new String(getAppRoot());
177-
File app_root_file = new File(app_root);
178-
PythonUtil.loadLibraries(app_root_file);
176+
PythonUtil.loadLibraries(this);
179177
}
180178

181179
public void recursiveDelete(File f) {

‎pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonService.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,12 @@
1414
import android.util.Log;
1515
import java.io.File;
1616

17+
import org.kivy.android.PythonActivity;
1718
import org.kivy.android.PythonUtil;
1819

1920
import org.renpy.android.Hardware;
2021

22+
2123
public abstract class PythonService extends Service implements Runnable {
2224
private static String TAG = PythonService.class.getSimpleName();
2325

@@ -144,9 +146,7 @@ public void onDestroy() {
144146
*/
145147
@Override
146148
public void run() {
147-
String app_root = getFilesDir().getAbsolutePath() + "/app";
148-
File app_root_file = new File(app_root);
149-
PythonUtil.loadLibraries(app_root_file);
149+
PythonUtil.loadLibraries(PythonActivity.mActivity);
150150
nativeStart(
151151
androidPrivate, androidArgument,
152152
serviceEntrypoint, pythonName,

‎pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonUtil.java

Lines changed: 30 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -3,58 +3,51 @@
33
import java.io.File;
44

55
import android.util.Log;
6-
import java.util.ArrayList;
7-
import java.io.FilenameFilter;
8-
import java.util.regex.Pattern;
6+
import android.app.Activity;
7+
import android.content.res.AssetManager;
98

9+
import java.util.ArrayList;
10+
import java.io.BufferedReader;
11+
import java.io.InputStreamReader;
12+
import java.io.FileNotFoundException;
13+
import java.io.IOException;
1014

1115
public class PythonUtil {
1216
private static final String TAG = "PythonUtil";
1317

14-
protected static void addLibraryIfExists(ArrayList<String> libsList, String pattern, File libsDir) {
15-
// pattern should be the name of the lib file, without the
16-
// preceding "lib" or suffix ".so", for instance "ssl.*" will
17-
// match files of the form "libssl.*.so".
18-
File [] files = libsDir.listFiles();
18+
protected static ArrayList<String> getLibraries(Activity activity) {
19+
AssetManager assets = activity.getAssets();
1920

20-
pattern = "lib" + pattern + "\\.so";
21-
Pattern p = Pattern.compile(pattern);
22-
for (int i = 0; i < files.length; ++i) {
23-
File file = files[i];
24-
String name = file.getName();
25-
Log.v(TAG, "Checking pattern " + pattern + " against " + name);
26-
if (p.matcher(name).matches()) {
27-
Log.v(TAG, "Pattern " + pattern + " matched file " + name);
28-
libsList.add(name.substring(3, name.length() - 3));
21+
StringBuilder sb = new StringBuilder();
22+
ArrayList<String> libsList = new ArrayList<String>();
23+
BufferedReader reader = null;
24+
try {
25+
reader = new BufferedReader(
26+
new InputStreamReader(assets.open("libraries_to_load.txt")));
27+
String mLine;
28+
while ((mLine = reader.readLine()) != null) {
29+
libsList.add(mLine);
30+
}
31+
} catch (IOException e) {
32+
Log.v(TAG, "Error reading Libraries file...no libs will be added");
33+
} finally {
34+
if (reader != null) {
35+
try {
36+
reader.close();
37+
} catch (IOException e) {
38+
Log.v(TAG, "Error on closing libraries file...going on");
39+
}
2940
}
3041
}
31-
}
32-
33-
protected static ArrayList<String> getLibraries(File filesDir) {
34-
35-
String libsDirPath = filesDir.getParentFile().getParentFile().getAbsolutePath() + "/lib/";
36-
File libsDir = new File(libsDirPath);
37-
38-
ArrayList<String> libsList = new ArrayList<String>();
39-
addLibraryIfExists(libsList, "crystax", libsDir);
40-
addLibraryIfExists(libsList, "sqlite3", libsDir);
41-
addLibraryIfExists(libsList, "ffi", libsDir);
42-
addLibraryIfExists(libsList, "ssl.*", libsDir);
43-
addLibraryIfExists(libsList, "crypto.*", libsDir);
44-
libsList.add("python2.7");
45-
libsList.add("python3.5m");
46-
libsList.add("python3.6m");
47-
libsList.add("python3.7m");
4842
libsList.add("main");
4943
return libsList;
5044
}
5145

52-
public static void loadLibraries(File filesDir) {
46+
public static void loadLibraries(Activity activity) {
5347

54-
String filesDirPath = filesDir.getAbsolutePath();
5548
boolean foundPython = false;
5649

57-
for (String lib : getLibraries(filesDir)) {
50+
for (String lib : getLibraries(activity)) {
5851
Log.v(TAG, "Loading library: " + lib);
5952
try {
6053
System.loadLibrary(lib);

0 commit comments

Comments
 (0)