From 19aa462701a1ec459d777dc97d7a838199bce61c Mon Sep 17 00:00:00 2001 From: Ryan Pessa Date: Fri, 1 Aug 2014 13:05:20 -0500 Subject: [PATCH 1/4] skip biglink for jb 4.3 --- distribute.sh | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/distribute.sh b/distribute.sh index e33404e4ed..421389dbd3 100755 --- a/distribute.sh +++ b/distribute.sh @@ -63,6 +63,8 @@ export BIGLINK="$ROOT_PATH/src/tools/biglink" export PIP=$PIP_NAME export VIRTUALENV=$VIRTUALENV_NAME +JELLYBEAN=0 + MD5SUM=$(which md5sum) if [ "X$MD5SUM" == "X" ]; then MD5SUM=$(which md5) @@ -218,6 +220,10 @@ function push_arm() { export STRIP="$TOOLCHAIN_PREFIX-strip --strip-unneeded" export MAKE="make -j5" + if [ "$JELLYBEAN" == "1" ]; then + export LIBLINK="$CC -shared $LDFLAGS" + fi + # Use ccache ? which ccache &>/dev/null if [ $? -eq 0 ]; then @@ -248,6 +254,7 @@ function usage() { echo echo " -d directory Name of the distribution directory" echo " -h Show this help" + echo " -j Create distribution for JB 4.3+" echo " -l Show a list of available modules" echo " -m 'mod1 mod2' Modules to include" echo " -f Restart from scratch (remove the current build)" @@ -345,6 +352,10 @@ function run_prepare() { fi done + if [ "$JELLYBEAN" == "1" ]; then + info "Distribution for Jelly Bean (4.3) or higher; biglink will be skipped" + fi + info "Distribution will be located at $DIST_PATH" if [ -e "$DIST_PATH" ]; then error "The distribution $DIST_PATH already exist" @@ -760,7 +771,10 @@ function run_distribute() { debug "Fill private directory" try cp -a python-install/lib private/ try mkdir -p private/include/python2.7 - try mv libs/$ARCH/libpymodules.so private/ + + if [ "$JELLYBEAN" == "0" ]; then + try mv libs/$ARCH/libpymodules.so private/ + fi try cp python-install/include/python2.7/pyconfig.h private/include/python2.7/ debug "Reduce private directory from unwanted files" @@ -786,9 +800,11 @@ function run_distribute() { } function run_biglink() { - push_arm - try $BIGLINK $LIBS_PATH/libpymodules.so $LIBLINK_PATH - pop_arm + if [ "$JELLYBEAN" == "0" ]; then + push_arm + try $BIGLINK $LIBS_PATH/libpymodules.so $LIBLINK_PATH + pop_arm + fi } function run() { @@ -829,11 +845,14 @@ function arm_deduplicate() { # Do the build -while getopts ":hvlfxm:u:d:s" opt; do +while getopts ":hjvlfxm:u:d:s" opt; do case $opt in h) usage ;; + j) + JELLYBEAN=1 + ;; l) list_modules ;; From 7606331eb89b99ed3a4894d93113c31554a937eb Mon Sep 17 00:00:00 2001 From: Ryan Pessa Date: Mon, 18 Aug 2014 13:06:21 -0500 Subject: [PATCH 2/4] add libcopy for jb 4.3 --- distribute.sh | 21 +++++--- src/tools/biglink-jb | 111 +++++++++++++++++++++++++++++++++++++++++++ src/tools/liblink-jb | 43 +++++++++++++++++ 3 files changed, 168 insertions(+), 7 deletions(-) create mode 100755 src/tools/biglink-jb create mode 100755 src/tools/liblink-jb diff --git a/distribute.sh b/distribute.sh index 421389dbd3..f9924e9a25 100755 --- a/distribute.sh +++ b/distribute.sh @@ -63,7 +63,7 @@ export BIGLINK="$ROOT_PATH/src/tools/biglink" export PIP=$PIP_NAME export VIRTUALENV=$VIRTUALENV_NAME -JELLYBEAN=0 +export JELLYBEAN=0 MD5SUM=$(which md5sum) if [ "X$MD5SUM" == "X" ]; then @@ -219,10 +219,11 @@ function push_arm() { export LD="$TOOLCHAIN_PREFIX-ld" export STRIP="$TOOLCHAIN_PREFIX-strip --strip-unneeded" export MAKE="make -j5" + export READELF="$TOOLCHAIN_PREFIX-readelf" - if [ "$JELLYBEAN" == "1" ]; then - export LIBLINK="$CC -shared $LDFLAGS" - fi + #if [ "$JELLYBEAN" == "1" ]; then + # export LIBLINK="$CC -shared $LDFLAGS" + #fi # Use ccache ? which ccache &>/dev/null @@ -772,7 +773,9 @@ function run_distribute() { try cp -a python-install/lib private/ try mkdir -p private/include/python2.7 - if [ "$JELLYBEAN" == "0" ]; then + if [ "$JELLYBEAN" == "1" ]; then + try sh -c "cat libs/$ARCH/copylibs | xargs -d'\n' cp -t private/" + else try mv libs/$ARCH/libpymodules.so private/ fi try cp python-install/include/python2.7/pyconfig.h private/include/python2.7/ @@ -800,11 +803,13 @@ function run_distribute() { } function run_biglink() { + push_arm if [ "$JELLYBEAN" == "0" ]; then - push_arm try $BIGLINK $LIBS_PATH/libpymodules.so $LIBLINK_PATH - pop_arm + else + try $BIGLINK $LIBS_PATH/copylibs $LIBLINK_PATH fi + pop_arm } function run() { @@ -852,6 +857,8 @@ while getopts ":hjvlfxm:u:d:s" opt; do ;; j) JELLYBEAN=1 + LIBLINK=${LIBLINK}-jb + BIGLINK=${BIGLINK}-jb ;; l) list_modules diff --git a/src/tools/biglink-jb b/src/tools/biglink-jb new file mode 100755 index 0000000000..e7b9609424 --- /dev/null +++ b/src/tools/biglink-jb @@ -0,0 +1,111 @@ +#!/usr/bin/env python + +from __future__ import print_function +import os +import sys +import subprocess +import re + +re_needso = re.compile(r'^.*\(NEEDED\)\s+Shared library: \[lib(.*)\.so\]\s*$') + +blacklist_libs = ( + 'c', + 'stdc++', + 'dl', + 'python2.7', + 'sdl', + 'sdl_image', + 'sdl_ttf', + 'z', + 'm', + 'GLESv2', + 'jpeg', + 'png', + 'log', +) + +found_libs = [] +sofiles = [ ] + +for directory in sys.argv[2:]: + + for fn in os.listdir(directory): + fn = os.path.join(directory, fn) + + if not fn.endswith(".libs"): + continue + + dirfn = fn[:-1] + 'dirs' + if not os.path.exists(dirfn): + continue + + with open(fn) as f: + needed_libs = [lib for lib in {ln.strip() for ln in f} if lib not in blacklist_libs and lib not in found_libs] + + while needed_libs: + print('need libs:\n\t' + '\n\t'.join(needed_libs)) + + start_needed_libs = needed_libs[:] + found_sofiles = [] + + with open(dirfn) as f: + for libdir in f: + if not needed_libs: + break + + libdir = libdir.strip() + print('scanning %s' % libdir) + for lib in needed_libs[:]: + if lib in found_libs: + continue + + if lib.endswith('.a'): + needed_libs.remove(lib) + found_libs.append(lib) + continue + + lib_a = 'lib' + lib + '.a' + libpath_a = os.path.join(libdir, lib_a) + lib_so = 'lib' + lib + '.so' + libpath_so = os.path.join(libdir, lib_so) + plain_so = lib + '.so' + plainpath_so = os.path.join(libdir, plain_so) + + sopath = None + if os.path.exists(libpath_so): + sopath = libpath_so + elif os.path.exists(plainpath_so): + sopath = plainpath_so + + if sopath: + print('found %s in %s' % (lib, libdir)) + found_sofiles.append(sopath) + needed_libs.remove(lib) + found_libs.append(lib) + continue + + if os.path.exists(libpath_a): + print('found %s (static) in %s' % (lib, libdir)) + needed_libs.remove(lib) + found_libs.append(lib) + continue + + for sofile in found_sofiles: + print('scanning dependencies for %s' % sofile) + out = subprocess.check_output([os.environ['READELF'], '-d', sofile]) + for line in out.splitlines(): + needso = re_needso.match(line) + if needso: + lib = needso.group(1) + if lib not in needed_libs and lib not in found_libs and lib not in blacklist_libs: + needed_libs.append(needso.group(1)) + + sofiles += found_sofiles + + if needed_libs == start_needed_libs: + raise RuntimeError('Failed to located needed libraries!\n\t' + '\n\t'.join(needed_libs)) + +output = sys.argv[1] + +with open(output, 'w') as f: + f.write('\n'.join(sofiles)) diff --git a/src/tools/liblink-jb b/src/tools/liblink-jb new file mode 100755 index 0000000000..46dc1ca325 --- /dev/null +++ b/src/tools/liblink-jb @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +from __future__ import print_function +import sys +import subprocess +from os import environ +from os.path import basename, join + +libs = [ ] +libdirs = [ ] +output = None + + +i = 1 +while i < len(sys.argv): + opt = sys.argv[i] + i += 1 + + if opt == "-o": + output = sys.argv[i] + i ++ 1 + continue + + if opt.startswith("-l"): + libs.append(opt[2:]) + continue + + if opt.startswith("-L"): + libdirs.append(opt[2:]) + continue + +output = join(environ.get('LIBLINK_PATH'), basename(output)) + +with open(output + ".libs", "w") as f: + f.write("\n".join(libs)) + +with open(output + ".libdirs", "w") as f: + f.write("\n".join(libdirs)) + + +libargs = ' '.join(["'%s'" % arg for arg in sys.argv[1:]]) +cmd = '%s -shared %s %s' % (environ['CC'], environ['LDFLAGS'], libargs) +sys.exit(subprocess.call(cmd, shell=True)) From a101685ba0a623cf5b335c7f4573f9918d5682d9 Mon Sep 17 00:00:00 2001 From: Ryan Pessa Date: Mon, 15 Sep 2014 17:27:47 -0500 Subject: [PATCH 3/4] rename option to copylibs and clean up --- distribute.sh | 34 ++++++++++++++++++++-------------- src/tools/biglink-jb | 2 +- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/distribute.sh b/distribute.sh index f9924e9a25..6b1b82f187 100755 --- a/distribute.sh +++ b/distribute.sh @@ -63,7 +63,7 @@ export BIGLINK="$ROOT_PATH/src/tools/biglink" export PIP=$PIP_NAME export VIRTUALENV=$VIRTUALENV_NAME -export JELLYBEAN=0 +export COPYLIBS=0 MD5SUM=$(which md5sum) if [ "X$MD5SUM" == "X" ]; then @@ -221,10 +221,6 @@ function push_arm() { export MAKE="make -j5" export READELF="$TOOLCHAIN_PREFIX-readelf" - #if [ "$JELLYBEAN" == "1" ]; then - # export LIBLINK="$CC -shared $LDFLAGS" - #fi - # Use ccache ? which ccache &>/dev/null if [ $? -eq 0 ]; then @@ -255,12 +251,15 @@ function usage() { echo echo " -d directory Name of the distribution directory" echo " -h Show this help" - echo " -j Create distribution for JB 4.3+" echo " -l Show a list of available modules" echo " -m 'mod1 mod2' Modules to include" echo " -f Restart from scratch (remove the current build)" echo " -x display expanded values (execute 'set -x')" echo + echo "Advanced:" + echo " -L Copy libraries instead of using biglink" + echo " (may not work before Android 4.3)" + echo echo "For developers:" echo " -u 'mod1 mod2' Modules to update (if already compiled)" echo @@ -353,8 +352,13 @@ function run_prepare() { fi done - if [ "$JELLYBEAN" == "1" ]; then - info "Distribution for Jelly Bean (4.3) or higher; biglink will be skipped" + if [ "$COPYLIBS" == "1" ]; then + info "Library files will be copied to the distribution (no biglink)" + error "NOTICE: This option is still beta!" + error "\tIf you encounter an error 'Failed to locate needed libraries!' and" + error "\tthe libraries listed are not supposed to be provided by your app or" + error "\tits dependencies, please submit a bug report at" + error "\thttps://github.com/kivy/python-for-android/issues" fi info "Distribution will be located at $DIST_PATH" @@ -773,8 +777,10 @@ function run_distribute() { try cp -a python-install/lib private/ try mkdir -p private/include/python2.7 - if [ "$JELLYBEAN" == "1" ]; then - try sh -c "cat libs/$ARCH/copylibs | xargs -d'\n' cp -t private/" + if [ "$COPYLIBS" == "1" ]; then + if [ -s "libs/$ARCH/copylibs" ]; then + try sh -c "cat libs/$ARCH/copylibs | xargs -d'\n' cp -t private/" + fi else try mv libs/$ARCH/libpymodules.so private/ fi @@ -804,7 +810,7 @@ function run_distribute() { function run_biglink() { push_arm - if [ "$JELLYBEAN" == "0" ]; then + if [ "$COPYLIBS" == "0" ]; then try $BIGLINK $LIBS_PATH/libpymodules.so $LIBLINK_PATH else try $BIGLINK $LIBS_PATH/copylibs $LIBLINK_PATH @@ -850,13 +856,13 @@ function arm_deduplicate() { # Do the build -while getopts ":hjvlfxm:u:d:s" opt; do +while getopts ":hCvlfxm:u:d:s" opt; do case $opt in h) usage ;; - j) - JELLYBEAN=1 + C) + COPYLIBS=1 LIBLINK=${LIBLINK}-jb BIGLINK=${BIGLINK}-jb ;; diff --git a/src/tools/biglink-jb b/src/tools/biglink-jb index e7b9609424..ef3004f5bc 100755 --- a/src/tools/biglink-jb +++ b/src/tools/biglink-jb @@ -103,7 +103,7 @@ for directory in sys.argv[2:]: sofiles += found_sofiles if needed_libs == start_needed_libs: - raise RuntimeError('Failed to located needed libraries!\n\t' + '\n\t'.join(needed_libs)) + raise RuntimeError('Failed to locate needed libraries!\n\t' + '\n\t'.join(needed_libs)) output = sys.argv[1] From 7a60a3a2c2f2d8e49295b29c3d2927c49d63fc1f Mon Sep 17 00:00:00 2001 From: Ryan Pessa Date: Mon, 15 Sep 2014 17:30:50 -0500 Subject: [PATCH 4/4] fix help --- distribute.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribute.sh b/distribute.sh index 6b1b82f187..9a8da04bef 100755 --- a/distribute.sh +++ b/distribute.sh @@ -257,7 +257,7 @@ function usage() { echo " -x display expanded values (execute 'set -x')" echo echo "Advanced:" - echo " -L Copy libraries instead of using biglink" + echo " -C Copy libraries instead of using biglink" echo " (may not work before Android 4.3)" echo echo "For developers:"