diff --git a/.travis.yml b/.travis.yml index e20a8cc75f..c2c38d018b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,35 +1,46 @@ sudo: required +#jdk: openjdk8 -language: generic +language: android + + +android: + components: + - build-tools-23.0.1 + - android-23 + + licenses: + - 'android-sdk-license-.+' + - 'google-gdk-license-.+' -services: - - docker before_install: - - sudo apt update -qq - - sudo apt install -qq --no-install-recommends python2.7 python3 - - sudo pip install tox>=2.0 - # https://github.com/travis-ci/travis-ci/issues/6069#issuecomment-266546552 - - git remote set-branches --add origin master - - git fetch + - sudo apt -qq update > /dev/null + - sudo apt -qq install --no-install-recommends build-essential ccache git zlib1g-dev python2.7 python2.7-dev libncurses5:i386 libstdc++6:i386 zlib1g:i386 openjdk-8-jdk unzip ant ccache autoconf libtool > /dev/null + - wget -q https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip + - unzip -qq android-ndk-r16b-linux-x86_64.zip -d $HOME + - export ANDROID_NDK_HOME=$HOME/android-ndk-r16b + - echo "ANDROID_NDK_HOME" $ANDROID_NDK_HOME + - export PATH=$PATH:$ANDROID_NDK_HOME + - export PIP_DISABLE_PIP_VERSION_CHECK=1 + - virtualenv --python=/usr/bin/python2.7 venv/bin/activate + - pip install --quiet --user -r requirements.txt + + +install: + - git clone --quiet --depth=50 --branch=master https://github.com/Redwid/youtube-dl.git ../youtube-dl + env: global: - ANDROID_SDK_HOME=/opt/android/android-sdk - - ANDROID_NDK_HOME=/opt/android/android-ndk - matrix: - - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python3' - # overrides requirements to skip `peewee` pure python module, see: - # https://github.com/kivy/python-for-android/issues/1263#issuecomment-390421054 - - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools' - - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --bootstrap sdl2 --requirements python2,numpy' - # builds only the recipes that moved - - COMMAND='. venv/bin/activate && ./ci/rebuild_updated_recipes.py' - -before_script: - # we want to fail fast on tox errors without having to `docker build` first - - tox + script: - - docker build --tag=p4a . - - docker run p4a /bin/sh -c "$COMMAND" + - sh clean-build-copy.sh + +deploy: +# dry_run: true + provider: script + skip_cleanup: true + script: cd deploy && sh gradlew --no-daemon bintrayUpload --info diff --git a/README.rst b/README.rst index 5a0455875f..e38156a63a 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,18 @@ python-for-android ================== + +Forked version +============== + +Changes: + +- Modified native start.c + +- Added bash script to easily build + + + |Build Status| .. |Build Status| image:: https://secure.travis-ci.org/kivy/python-for-android.png?branch=master diff --git a/clean-build-copy.sh b/clean-build-copy.sh new file mode 100644 index 0000000000..3879cc73a4 --- /dev/null +++ b/clean-build-copy.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# +# Set up instructions for MacOS (10.14) +# 1. Set up your ANDROID_HOME directory: +# export ANDROID_HOME=/Users/{your_user_name_here}/Library/Android/sdk +# source ~/.bash_profile +# 2. Install gcc: +# brew install gcc +# 3. Install the macOS_SDK_headers_for_macOS_10.14.pkg: https://github.com/frida/frida/issues/338#issuecomment-424595668 +# cd /Library/Developer/CommandLineTools/Packages/ +# open macOS_SDK_headers_for_macOS_10.14.pkg +# 4. Install llvm: +# brew install llvm +# 5. Install XCode +# 6. Download NDK r16b: https://developer.android.com/ndk/downloads/older_releases?hl=zh-tw +# set $ANDROID_NDK_HOME value below pointed to unpacked NDK r16b folder +# + + +#enviroment set up +export PYTHONIOENCODING=UTF-8 +export PYTHONUNBUFFERED=1 +export VERSIONER_PYTHON_VERSION=2.7 +export PYTHONPATH=$PWD +export ANDROIDAPI="23" +export NDKAPI="23" + +if [ -z "$ANDROID_NDK_HOME" ] +then + export ANDROID_NDK_HOME=~/Downloads/android-ndk-r16b +fi + +#script set up +p4a="$PWD/pythonforandroid/toolchain.py" +pythonApp="$(dirname "$PWD")/youtube-dl/youtube_dl" +pythonAppDistName="youtube_dl_wrapper" +pythonAppPackageName="org.youtube.dl" +pythonAppName="youtube_dl" + +#Check if we are running on travis +if [ -z "$TRAVIS" ] +then + distFolder="$HOME/.python-for-android/dists/$pythonAppDistName" +else + distFolder="$HOME/.local/share/python-for-android/dists/$pythonAppDistName" +fi + +echo "pythonApp: ${pythonApp}" +echo "distFolder: ${distFolder}" + +#Link for android-youtube-dl project to where script will copy build artifacts +androidYoutubeDlProject="$(dirname "$PWD")/android-youtube-dl" + + +clean() { + echo "[INFO] Clean" + python $p4a clean_builds + rm -rf $distFolder + rm -rf $PWD/build + echo "[INFO] Clean done" +} + + +build() { + echo "[INFO] Build [$1]" + python $p4a clean_builds + rm -rf $distFolder + python -W ignore $p4a apk --private=$pythonApp --dist_name=$pythonAppDistName --package=$pythonAppPackageName --name=$pythonAppName --version=1 --ndk_dir=$ANDROID_NDK_HOME --ndk_version=r16b --requirements=android,pyopenssl,pycrypto,openssl --android_api=$ANDROIDAPI --arch=$1 --java-build-tool=gradle +# if [ -z "$TRAVIS" ] +# then +# copy_libs +# copy_assets +# else + copy_assets_and_make_aar $1 +# fi + echo "[INFO] Build [$1] done" +} + + +copy_libs() { + mkdir $PWD/build + cp -rfv $distFolder/libs/ $PWD/build/libs/ +} + + +copy_assets() { + cp -rfv $distFolder/src/main/assets/ $PWD/build/assets + rm $PWD/build/assets/.gitkeep + + delete_unused_assets "armeabi-v7a" + #delete_unused_assets "x86" + delete_unused_assets "arm64-v8a" +} + + +delete_unused_assets() { + rm $PWD/build/libs/$1/libSDL2.so + rm $PWD/build/libs/$1/libSDL2_image.so + rm $PWD/build/libs/$1/libSDL2_mixer.so + rm $PWD/build/libs/$1/libSDL2_ttf.so +} + + +copy_assets_to_androidYoutubeDlProject() { + cp -rfv $PWD/build/assets/ $androidYoutubeDlProject/lib/src/main/assets + cp -rfv $PWD/build/libs/ $androidYoutubeDlProject/lib/src/main/jniLibs +} + + +copy_assets_and_make_aar() { + echo "[INFO] Copy assets [$1]" + #Copy *.so files + mkdir $PWD/build + mkdir $PWD/build/$1 + mkdir $PWD/build/$1/jni + mkdir $PWD/build/$1/jni/$1 + cp -rfv $distFolder/libs/$1 $PWD/build/$1/jni + rm $PWD/build/$1/jni/$1/libSDL2.so + rm $PWD/build/$1/jni/$1/libSDL2_image.so + rm $PWD/build/$1/jni/$1/libSDL2_mixer.so + rm $PWD/build/$1/jni/$1/libSDL2_ttf.so + + #Copy *.mp3 file + mkdir $PWD/build/$1/assets + cp -rfv $distFolder/src/main/assets/private.mp3 $PWD/build/$1/assets/private-$1.mp3 + echo "[INFO] Copy assets [$1] done" + + echo "[INFO] Make aar [$1]" + #Create aar file + root=$PWD + cd $root/build/$1 + zip $root/build/$1.aar -r ./* + cd $root + echo "[INFO] Make aar [$1] done" +} + + +clean + +build "armeabi-v7a" +#build "x86" +build "arm64-v8a" + +#copy_assets_to_androidYoutubeDlProject +echo "[INFO] Build done" diff --git a/deploy/build.gradle b/deploy/build.gradle new file mode 100644 index 0000000000..1cf1a5ce04 --- /dev/null +++ b/deploy/build.gradle @@ -0,0 +1,69 @@ +buildscript { + repositories { + jcenter() + } +} + +plugins { + id "com.jfrog.bintray" version "1.8.4" +} + +allprojects { + repositories { + jcenter() + } + apply plugin: 'maven' + apply plugin: 'maven-publish' + apply plugin: 'java' +} + +group = 'org.redwid.android.youtube.dl' +description = 'python' +version = '0.6.8' + +publishing { + publications { + MyPublication(MavenPublication) { + version = project.version + artifactId 'python' + artifacts = createArtifacts() + } + } + repositories { + maven { + url { "file://${System.env.HOME}/.m2/repository/" } + } + } +} + +def createArtifacts() { + def artifacts = [] + artifacts += [source: "../build/arm64-v8a.aar", classifier: "arm64-v8a", extension: "aar"] + artifacts += [source: "../build/armeabi-v7a.aar", classifier: "armeabi-v7a", extension: "aar"] + println("artifacts: " + artifacts) + return artifacts +} + +bintray { + user = System.getenv('BINTRAY_USER') + key = System.getenv('BINTRAY_API_KEY') + publications = ['MyPublication'] + pkg { + repo = 'maven' + name = "$project.group:$project.description" + userOrg = 'redwid' + licenses = ['MIT'] + vcsUrl = 'https://github.com/Redwid/python-for-android.git' + publish = true + publicDownloadNumbers = true + version { + name = project.version + desc = 'Python for android youtube-dl' + released = new Date() + } + } +} + +task wrapper(type: Wrapper) { + gradleVersion = '4.10.2' +} \ No newline at end of file diff --git a/deploy/gradle/gradle.properties b/deploy/gradle/gradle.properties new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deploy/gradle/wrapper/gradle-wrapper.jar b/deploy/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000..29953ea141 Binary files /dev/null and b/deploy/gradle/wrapper/gradle-wrapper.jar differ diff --git a/deploy/gradle/wrapper/gradle-wrapper.properties b/deploy/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000000..e0b3fb8d70 --- /dev/null +++ b/deploy/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.2-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/deploy/gradlew b/deploy/gradlew new file mode 100755 index 0000000000..cccdd3d517 --- /dev/null +++ b/deploy/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/deploy/gradlew.bat b/deploy/gradlew.bat new file mode 100644 index 0000000000..f9553162f1 --- /dev/null +++ b/deploy/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/pythonforandroid/bootstraps/pygame/build/build.py b/pythonforandroid/bootstraps/pygame/build/build.py index 1c08339413..1b4ff7f2c4 100755 --- a/pythonforandroid/bootstraps/pygame/build/build.py +++ b/pythonforandroid/bootstraps/pygame/build/build.py @@ -180,7 +180,7 @@ def select(fn): if select(x)] # create tar.gz of thoses files - tf = tarfile.open(tfn, 'w:gz', format=tarfile.USTAR_FORMAT) + tf = tarfile.open(tfn, 'w:gz', format=tarfile.GNU_FORMAT) dirs = [] for fn, afn in files: dn = dirname(afn) diff --git a/pythonforandroid/bootstraps/sdl2/__init__.py b/pythonforandroid/bootstraps/sdl2/__init__.py index e77d9c932a..f9c9628152 100644 --- a/pythonforandroid/bootstraps/sdl2/__init__.py +++ b/pythonforandroid/bootstraps/sdl2/__init__.py @@ -1,8 +1,10 @@ +from os.path import join, exists + +import sh + from pythonforandroid.toolchain import ( Bootstrap, shprint, current_directory, info, info_main) from pythonforandroid.util import ensure_dir -from os.path import join, exists -import sh class SDL2GradleBootstrap(Bootstrap): @@ -36,7 +38,7 @@ def run_distribute(self): hostpython = sh.Command(self.ctx.hostpython) if self.ctx.python_recipe.name == 'python2': try: - shprint(hostpython, '-OO', '-m', 'compileall', + shprint(hostpython, '-OO', '-m', 'compileall', '-q', python_install_dir, _tail=10, _filterout="^Listing") except sh.ErrorReturnCode: diff --git a/pythonforandroid/bootstraps/sdl2/build/build.py b/pythonforandroid/bootstraps/sdl2/build/build.py index d620cb46d6..487ff7158b 100644 --- a/pythonforandroid/bootstraps/sdl2/build/build.py +++ b/pythonforandroid/bootstraps/sdl2/build/build.py @@ -2,20 +2,20 @@ # coding: utf-8 from __future__ import print_function -from os.path import ( - dirname, join, isfile, realpath, relpath, split, exists, basename) -from os import makedirs, remove, listdir -import os -import tarfile -import time + import json -import subprocess +import os import shutil -from zipfile import ZipFile +import subprocess import sys +import tarfile +import time from distutils.version import LooseVersion - from fnmatch import fnmatch +from os import makedirs, remove, listdir +from os.path import ( + dirname, join, isfile, realpath, relpath, split, exists, basename) +from zipfile import ZipFile import jinja2 @@ -185,7 +185,7 @@ def select(fn): if select(x)] # create tar.gz of thoses files - tf = tarfile.open(tfn, 'w:gz', format=tarfile.USTAR_FORMAT) + tf = tarfile.open(tfn, 'w:gz', format=tarfile.GNU_FORMAT) dirs = [] for fn, afn in files: dn = dirname(afn) @@ -215,7 +215,7 @@ def compile_dir(dfn): # -OO = strip docstrings if PYTHON is None: return - subprocess.call([PYTHON, '-OO', '-m', 'compileall', '-f', dfn]) + subprocess.call([PYTHON, '-OO', '-m', 'compileall', '-f', '-q', dfn]) def make_package(args): diff --git a/pythonforandroid/bootstraps/sdl2/build/jni/src/Android.mk b/pythonforandroid/bootstraps/sdl2/build/jni/src/Android.mk index be2bb9b27a..6647dfcbbd 100644 --- a/pythonforandroid/bootstraps/sdl2/build/jni/src/Android.mk +++ b/pythonforandroid/bootstraps/sdl2/build/jni/src/Android.mk @@ -4,19 +4,14 @@ include $(CLEAR_VARS) LOCAL_MODULE := main -SDL_PATH := ../SDL - -LOCAL_C_INCLUDES := $(LOCAL_PATH)/$(SDL_PATH)/include - # Add your application source files here... -LOCAL_SRC_FILES := $(SDL_PATH)/src/main/android/SDL_android_main.c \ - start.c +LOCAL_SRC_FILES := start.c LOCAL_CFLAGS += -I$(PYTHON_INCLUDE_ROOT) $(EXTRA_CFLAGS) -LOCAL_SHARED_LIBRARIES := SDL2 python_shared +LOCAL_SHARED_LIBRARIES := python_shared -LOCAL_LDLIBS := -lGLESv1_CM -lGLESv2 -llog $(EXTRA_LDLIBS) +LOCAL_LDLIBS := -llog $(EXTRA_LDLIBS) LOCAL_LDFLAGS += -L$(PYTHON_LINK_ROOT) $(APPLICATION_ADDITIONAL_LDFLAGS) diff --git a/pythonforandroid/bootstraps/sdl2/build/jni/src/Android_static.mk b/pythonforandroid/bootstraps/sdl2/build/jni/src/Android_static.mk index faed669c0e..2de278ee00 100644 --- a/pythonforandroid/bootstraps/sdl2/build/jni/src/Android_static.mk +++ b/pythonforandroid/bootstraps/sdl2/build/jni/src/Android_static.mk @@ -6,7 +6,5 @@ LOCAL_MODULE := main LOCAL_SRC_FILES := YourSourceHere.c -LOCAL_STATIC_LIBRARIES := SDL2_static - include $(BUILD_SHARED_LIBRARY) $(call import-module,SDL)LOCAL_PATH := $(call my-dir) diff --git a/pythonforandroid/bootstraps/sdl2/build/jni/src/start.c b/pythonforandroid/bootstraps/sdl2/build/jni/src/start.c index c4cde305f2..472ac7555c 100644 --- a/pythonforandroid/bootstraps/sdl2/build/jni/src/start.c +++ b/pythonforandroid/bootstraps/sdl2/build/jni/src/start.c @@ -14,9 +14,7 @@ #include #include -#include "SDL.h" #include "android/log.h" -#include "SDL_opengles2.h" #define ENTRYPOINT_MAXLEN 128 #define LOG(n, x) __android_log_write(ANDROID_LOG_INFO, (n), (x)) @@ -68,6 +66,16 @@ int file_exists(const char *filename) { /* int main(int argc, char **argv) { */ int main(int argc, char *argv[]) { + LOGP("main() begin"); + + char out[1024]; + snprintf(out, 1024, "argc: %d", argc); + LOGP(out); + for (int i = 0; i < argc; i++) { + snprintf(out, 1024, " item[%d]: %s", i, argv[i]); + LOGP(out); + } + char *env_argument = NULL; char *env_entrypoint = NULL; @@ -295,9 +303,11 @@ int main(int argc, char *argv[]) { return -1; } + LOGP("run PyRun_SimpleFile begin"); /* run python ! */ ret = PyRun_SimpleFile(fd, entrypoint); + LOGP("run PyRun_SimpleFile end"); if (PyErr_Occurred() != NULL) { ret = 1; @@ -317,42 +327,53 @@ int main(int argc, char *argv[]) { return ret; } -JNIEXPORT void JNICALL Java_org_kivy_android_PythonService_nativeStart( +JNIEXPORT void JNICALL Java_org_redwid_android_youtube_dl_YoutubeDlWorker_nativeStart( JNIEnv *env, jobject thiz, jstring j_android_private, - jstring j_android_argument, jstring j_service_entrypoint, - jstring j_python_name, jstring j_python_home, jstring j_python_path, - jstring j_arg) { + jstring j_android_argument, jstring j_application_entrypoint, + jstring j_python_name, jstring j_python_home, jstring j_python_path, jobjectArray j_application_args) { + LOGP("Python Java_org_redwid_android_youtube_dl_YoutubeDlWorker_nativeStart() begin"); jboolean iscopy; const char *android_private = (*env)->GetStringUTFChars(env, j_android_private, &iscopy); const char *android_argument = (*env)->GetStringUTFChars(env, j_android_argument, &iscopy); - const char *service_entrypoint = - (*env)->GetStringUTFChars(env, j_service_entrypoint, &iscopy); + const char *application_entrypoint = + (*env)->GetStringUTFChars(env, j_application_entrypoint, &iscopy); const char *python_name = (*env)->GetStringUTFChars(env, j_python_name, &iscopy); const char *python_home = (*env)->GetStringUTFChars(env, j_python_home, &iscopy); const char *python_path = (*env)->GetStringUTFChars(env, j_python_path, &iscopy); - const char *arg = (*env)->GetStringUTFChars(env, j_arg, &iscopy); setenv("ANDROID_PRIVATE", android_private, 1); setenv("ANDROID_ARGUMENT", android_argument, 1); setenv("ANDROID_APP_PATH", android_argument, 1); - setenv("ANDROID_ENTRYPOINT", service_entrypoint, 1); + setenv("ANDROID_ENTRYPOINT", application_entrypoint, 1); setenv("PYTHONOPTIMIZE", "2", 1); setenv("PYTHON_NAME", python_name, 1); setenv("PYTHONHOME", python_home, 1); setenv("PYTHONPATH", python_path, 1); - setenv("PYTHON_SERVICE_ARGUMENT", arg, 1); setenv("P4A_BOOTSTRAP", "SDL2", 1); - char *argv[] = {"."}; + const char *argv[20]; + jsize stringCount = (*env)->GetArrayLength(env, j_application_args); + + char out[256]; + snprintf(out, 256, "stringCount: %d", stringCount); + LOGP(out); + + for (int i = 0; i < stringCount; i++) { + LOGP("processing j_application_args"); + jstring string = (jstring) (*env)->GetObjectArrayElement(env, j_application_args, i); + argv[i] = (*env)->GetStringUTFChars(env, string, &iscopy); + } + /* ANDROID_ARGUMENT points to service subdir, * so main() will run main.py from this dir */ - main(1, argv); + main(stringCount, argv); + LOGP("Python for nativeStart end."); } #endif diff --git a/pythonforandroid/bootstraps/service_only/__init__.py b/pythonforandroid/bootstraps/service_only/__init__.py index 088928c6c4..9990e46faa 100644 --- a/pythonforandroid/bootstraps/service_only/__init__.py +++ b/pythonforandroid/bootstraps/service_only/__init__.py @@ -1,7 +1,9 @@ import glob from os import walk from os.path import join, exists, curdir, abspath + import sh + from pythonforandroid.toolchain import Bootstrap, current_directory, info, info_main, shprint @@ -42,7 +44,7 @@ def run_distribute(self): hostpython = sh.Command(self.ctx.hostpython) if not self.ctx.python_recipe.from_crystax: try: - shprint(hostpython, '-OO', '-m', 'compileall', + shprint(hostpython, '-OO', '-m', 'compileall', '-q', self.ctx.get_python_install_dir(), _tail=10, _filterout="^Listing") except sh.ErrorReturnCode: diff --git a/pythonforandroid/bootstraps/service_only/build/build.py b/pythonforandroid/bootstraps/service_only/build/build.py index 29420cf23d..4183cab691 100755 --- a/pythonforandroid/bootstraps/service_only/build/build.py +++ b/pythonforandroid/bootstraps/service_only/build/build.py @@ -2,17 +2,16 @@ from __future__ import print_function -from os.path import dirname, join, isfile, realpath, relpath, split, exists -from os import makedirs import os -import tarfile -import subprocess +import shlex import shutil -from zipfile import ZipFile +import subprocess import sys -import shlex - +import tarfile from fnmatch import fnmatch +from os import makedirs +from os.path import dirname, join, isfile, realpath, relpath, split, exists +from zipfile import ZipFile import jinja2 @@ -175,7 +174,7 @@ def select(fn): if select(x)] # create tar.gz of thoses files - tf = tarfile.open(tfn, 'w:gz', format=tarfile.USTAR_FORMAT) + tf = tarfile.open(tfn, 'w:gz', format=tarfile.GNU_FORMAT) dirs = [] for fn, afn in files: dn = dirname(afn) @@ -205,7 +204,7 @@ def compile_dir(dfn): return # Currently leaving out the compile to pyo step because it's somehow broken # -OO = strip docstrings - subprocess.call([PYTHON, '-OO', '-m', 'compileall', '-f', dfn]) + subprocess.call([PYTHON, '-OO', '-m', 'compileall', '-f', '-q', dfn]) def make_package(args): diff --git a/pythonforandroid/bootstraps/webview/__init__.py b/pythonforandroid/bootstraps/webview/__init__.py index 041fe43c3e..b5fe4a8f81 100644 --- a/pythonforandroid/bootstraps/webview/__init__.py +++ b/pythonforandroid/bootstraps/webview/__init__.py @@ -1,9 +1,11 @@ -from pythonforandroid.toolchain import Bootstrap, current_directory, info, info_main, shprint -from os.path import join, exists, curdir, abspath -from os import walk import glob +from os import walk +from os.path import join, exists, curdir, abspath + import sh +from pythonforandroid.toolchain import Bootstrap, current_directory, info, info_main, shprint + class WebViewBootstrap(Bootstrap): name = 'webview' @@ -39,7 +41,7 @@ def run_distribute(self): hostpython = sh.Command(self.ctx.hostpython) if not self.ctx.python_recipe.from_crystax: try: - shprint(hostpython, '-OO', '-m', 'compileall', + shprint(hostpython, '-OO', '-m', 'compileall', '-q', self.ctx.get_python_install_dir(), _tail=10, _filterout="^Listing") except sh.ErrorReturnCode: diff --git a/pythonforandroid/bootstraps/webview/build/build.py b/pythonforandroid/bootstraps/webview/build/build.py index bc70b16c34..c4f8fabbcf 100755 --- a/pythonforandroid/bootstraps/webview/build/build.py +++ b/pythonforandroid/bootstraps/webview/build/build.py @@ -2,18 +2,17 @@ from __future__ import print_function -from os.path import dirname, join, isfile, realpath, relpath, split, exists -from os import makedirs import os -import tarfile -import time -import subprocess +import re import shutil -from zipfile import ZipFile +import subprocess import sys -import re - +import tarfile +import time from fnmatch import fnmatch +from os import makedirs +from os.path import dirname, join, isfile, realpath, relpath, split, exists +from zipfile import ZipFile import jinja2 @@ -176,7 +175,7 @@ def select(fn): if select(x)] # create tar.gz of thoses files - tf = tarfile.open(tfn, 'w:gz', format=tarfile.USTAR_FORMAT) + tf = tarfile.open(tfn, 'w:gz', format=tarfile.GNU_FORMAT) dirs = [] for fn, afn in files: dn = dirname(afn) @@ -206,7 +205,7 @@ def compile_dir(dfn): return # Currently leaving out the compile to pyo step because it's somehow broken # -OO = strip docstrings - subprocess.call([PYTHON, '-OO', '-m', 'compileall', '-f', dfn]) + subprocess.call([PYTHON, '-OO', '-m', 'compileall', '-f', '-q', dfn]) def make_package(args): diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index 9aa7212559..725c1c5eb7 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -1,19 +1,19 @@ from __future__ import print_function -from os.path import (join, realpath, dirname, expanduser, exists, - split, isdir) -from os import environ -import os import glob -import sys +import os import re import sh +import sys +from os import environ +from os.path import (join, realpath, dirname, expanduser, exists, + split, isdir) -from pythonforandroid.util import (ensure_dir, current_directory) +from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86 from pythonforandroid.logger import (info, warning, error, info_notify, Err_Fore, info_main, shprint) -from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86 from pythonforandroid.recipe import Recipe +from pythonforandroid.util import (ensure_dir, current_directory) DEFAULT_ANDROID_API = 15 diff --git a/pythonforandroid/logger.py b/pythonforandroid/logger.py index 38afba0e5f..26fe6bc85e 100644 --- a/pythonforandroid/logger.py +++ b/pythonforandroid/logger.py @@ -1,12 +1,13 @@ import logging import os import re -import sh -from sys import stdout, stderr -from math import log10 from collections import defaultdict -from colorama import Style as Colo_Style, Fore as Colo_Fore +from math import log10 +from sys import stdout, stderr + +import sh import six +from colorama import Style as Colo_Style, Fore as Colo_Fore # This codecs change fixes a bug with log output, but crashes under python3 if not six.PY3: @@ -176,19 +177,21 @@ def shprint(command, *args, **kwargs): for line in output: if isinstance(line, bytes): line = line.decode('utf-8', errors='replace') - if logger.level > logging.DEBUG: - msg = line.replace( - '\n', ' ').replace( - '\t', ' ').replace( - '\b', ' ').rstrip() - if msg: - stdout.write(u'{}\r{}{:<{width}}'.format( - Err_Style.RESET_ALL, msg_hdr, - shorten_string(msg, msg_width), width=msg_width)) - stdout.flush() - need_closing_newline = True - else: - logger.debug(''.join(['\t', line.rstrip()])) + # Workaround for travis ci + if 'TRAVIS' not in os.environ: + if logger.level > logging.DEBUG: + msg = line.replace( + '\n', ' ').replace( + '\t', ' ').replace( + '\b', ' ').rstrip() + if msg: + stdout.write(u'{}\r{}{:<{width}}'.format( + Err_Style.RESET_ALL, msg_hdr, + shorten_string(msg, msg_width), width=msg_width)) + stdout.flush() + need_closing_newline = True + else: + logger.debug(''.join(['\t', line.rstrip()])) if need_closing_newline: stdout.write('{}\r{:>{width}}\r'.format( Err_Style.RESET_ALL, ' ', width=(columns - 1))) diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 8bc68c036b..ca1ddb2712 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -1,17 +1,18 @@ -from os.path import basename, dirname, exists, isdir, isfile, join, realpath, split -import importlib +import fnmatch import glob -from shutil import rmtree -from six import PY2, with_metaclass - import hashlib -from re import match - -import sh +import importlib +import os import shutil -import fnmatch from os import listdir, unlink, environ, mkdir, curdir, walk +from os.path import basename, dirname, exists, isdir, isfile, join, realpath, split +from re import match +from shutil import rmtree from sys import stdout + +import sh +from six import PY2, with_metaclass + try: from urlparse import urlparse except ImportError: @@ -144,7 +145,19 @@ def report_hook(index, blksize, size): if exists(target): unlink(target) - urlretrieve(url, target, report_hook) + # Download item with multiple attempts (for bad connections): + attempts = 0 + while True: + try: + urlretrieve(url, target, report_hook) + except OSError: + attempts += 1 + if attempts >= 5: + raise + stdout.write('Download failed retrying in a second...') + time.sleep(1) + continue + break return target elif parsed_url.scheme in ('git', 'git+file', 'git+ssh', 'git+http', 'git+https'): if isdir(target): @@ -361,6 +374,8 @@ def unpack(self, arch): extraction_filename = join( self.ctx.packages_path, self.name, filename) if isfile(extraction_filename): + info('File type for {}'.format(extraction_filename)) + os.system('file ' + extraction_filename) if extraction_filename.endswith('.zip'): try: sh.unzip(extraction_filename) @@ -375,17 +390,12 @@ def unpack(self, arch): root_directory = fileh.filelist[0].filename.split('/')[0] if root_directory != basename(directory_name): shprint(sh.mv, root_directory, directory_name) - elif (extraction_filename.endswith('.tar.gz') or - extraction_filename.endswith('.tgz') or - extraction_filename.endswith('.tar.bz2') or - extraction_filename.endswith('.tbz2') or - extraction_filename.endswith('.tar.xz') or - extraction_filename.endswith('.txz')): - sh.tar('xf', extraction_filename) - root_directory = shprint( - sh.tar, 'tf', extraction_filename).stdout.decode( - 'utf-8').split('\n')[0].split('/')[0] - if root_directory != directory_name: + elif extraction_filename.endswith( + ('.tar.gz', '.tgz', '.tar.bz2', '.tbz2', '.tar.xz', '.txz')): + sh.tar('xvf', extraction_filename) + root_directory = sh.tar('tf', extraction_filename).stdout.decode( + 'utf-8').split('\n')[0].split('/')[0] + if root_directory != basename(directory_name): shprint(sh.mv, root_directory, directory_name) else: raise Exception( diff --git a/pythonforandroid/recipes/openssl/__init__.py b/pythonforandroid/recipes/openssl/__init__.py index b8256e8662..7c70b04d0a 100644 --- a/pythonforandroid/recipes/openssl/__init__.py +++ b/pythonforandroid/recipes/openssl/__init__.py @@ -1,12 +1,14 @@ from functools import partial -from pythonforandroid.toolchain import Recipe, shprint, current_directory import sh +from pythonforandroid.toolchain import Recipe, shprint, current_directory + class OpenSSLRecipe(Recipe): version = '1.0.2h' - url = 'https://www.openssl.org/source/openssl-{version}.tar.gz' + #url = 'https://www.openssl.org/source/openssl-{version}.tar.gz' + url = 'https://github.com/openssl/openssl/archive/OpenSSL_1_0_2h.tar.gz' def should_build(self, arch): return not self.has_libs(arch, 'libssl' + self.version + '.so', diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index 3aa187e111..2f23b8151b 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -7,6 +7,7 @@ """ from __future__ import print_function + from pythonforandroid import __version__ from pythonforandroid.build import DEFAULT_NDK_API, DEFAULT_ANDROID_API diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..0be947becf --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +colorama==0.3.3 +appdirs==1.4.3 +sh==1.10 +jinja2==2.10 +Cython==0.29.7 +six==1.4.1