From 7dd7d17b36c688407516632567fbbcfb13d24881 Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Sun, 7 Oct 2018 00:41:49 +0200 Subject: [PATCH] Conditional recipe build fixes #1382 This is a first iteration addressing recipes conditional build. In this version all modified recipes are build in batch with no dependency resolution whatsoever. Also recipes are only built against `python3crystax` target and `sdl2` bootstrap. The list of broken recipes is not fully known, but this script will help finding out. If a recipe is known to be broken, it should be added in the `BROKEN_RECIPES` set to be skipped until it gets fixed. --- .travis.yml | 27 +++-- ci/__init__.py | 0 ci/rebuild_updated_recipes.py | 124 +++++++++++++++++++++ pythonforandroid/recipes/babel/__init__.py | 2 +- pythonforandroid/recipes/cffi/__init__.py | 3 + setup.py | 9 +- tox.ini | 2 +- 7 files changed, 150 insertions(+), 17 deletions(-) create mode 100644 ci/__init__.py create mode 100755 ci/rebuild_updated_recipes.py diff --git a/.travis.yml b/.travis.yml index f24ac149c5..b74297e22a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,18 +6,27 @@ services: - docker before_install: - - sudo apt-get update -qq - - sudo apt-get install -qq python2.7 python3 + - sudo apt update -qq + - sudo apt install -qq python2.7 python3 - sudo pip install tox>=2.0 + # https://github.com/travis-ci/travis-ci/issues/6069#issuecomment-266546552 + - git remote set-branches --add origin master + - git fetch env: - - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2.py apk --sdk-dir /opt/android/android-sdk --ndk-dir /opt/android/android-ndk' - # overrides requirements to skip `peewee` pure python module, see: - # https://github.com/kivy/python-for-android/issues/1263#issuecomment-390421054 - - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir /opt/android/android-sdk --ndk-dir /opt/android/android-ndk --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools' - - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2.py apk --sdk-dir /opt/android/android-sdk --ndk-dir /opt/android/android-ndk --bootstrap sdl2 --requirements python2,numpy' - - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3.py apk --sdk-dir /opt/android/android-sdk --ndk-dir /opt/android/crystax-ndk' - - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3.py apk --sdk-dir /opt/android/android-sdk --ndk-dir /opt/android/crystax-ndk --requirements python3crystax,setuptools,android' + global: + - ANDROID_SDK_HOME=/opt/android/android-sdk + - ANDROID_NDK_HOME=/opt/android/android-ndk + - CRYSTAX_NDK_HOME=/opt/android/crystax-ndk + matrix: + - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME' + # overrides requirements to skip `peewee` pure python module, see: + # https://github.com/kivy/python-for-android/issues/1263#issuecomment-390421054 + - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools' + - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --bootstrap sdl2 --requirements python2,numpy' + - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $CRYSTAX_NDK_HOME --requirements python3crystax,setuptools,android,sdl2,pyjnius,kivy' + # builds only the recipes that moved + - COMMAND='. venv/bin/activate && ./ci/rebuild_updated_recipes.py' before_script: # we want to fail fast on tox errors without having to `docker build` first diff --git a/ci/__init__.py b/ci/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ci/rebuild_updated_recipes.py b/ci/rebuild_updated_recipes.py new file mode 100755 index 0000000000..82e80600a7 --- /dev/null +++ b/ci/rebuild_updated_recipes.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Continuous Integration helper script. +Automatically detects recipes modified in a changeset (compares with master) +and recompiles them. + +To run locally, set the environment variables before running: +``` +ANDROID_SDK_HOME=~/.buildozer/android/platform/android-sdk-20 +ANDROID_NDK_HOME=~/.buildozer/android/platform/android-ndk-r9c +CRYSTAX_NDK_HOME=~/.buildozer/crystax-ndk +./ci/rebuild_update_recipes.py +``` + +Current limitations: +- handle the case with conflicting requirements + e.g. https://travis-ci.org/AndreMiras/python-for-android/builds/438840800 + the list was huge and result was: + [ERROR]: Didn't find any valid dependency graphs. + [ERROR]: This means that some of your requirements pull in conflicting dependencies. +- currently only builds on target python3crystax (even though python2 is supported) +- comprehensive list of working/broken recipes is not yet known +- only rebuilds on sdl2 bootstrap +""" +import sh +import os +from enum import Enum +from pythonforandroid.toolchain import current_directory + + +class TargetPython(Enum): + python2 = 0 + python3crystax = 1 + + +# recipes that currently break the build +# a recipe could be broken for a target Python and not for the other, +# hence we're maintaining one list per Python target +BROKEN_RECIPES_PYTHON2 = set([]) +BROKEN_RECIPES_PYTHON3_CRYSTAX = set([ + # not yet python3crystax compatible + 'apsw', 'atom', 'boost', 'brokenrecipe', 'cdecimal', 'cherrypy', + 'coverage', + # https://github.com/kivy/python-for-android/issues/550 + 'audiostream', + # enum34 is not compatible with Python 3.6 standard library + # https://stackoverflow.com/a/45716067/185510 + 'enum34', + # https://github.com/kivy/python-for-android/issues/1398 + 'ifaddrs', + # https://github.com/kivy/python-for-android/issues/1399 + 'libglob', + # cannot find -lcrystax + 'cffi', +]) +BROKEN_RECIPES = { + TargetPython.python2: BROKEN_RECIPES_PYTHON2, + TargetPython.python3crystax: BROKEN_RECIPES_PYTHON3_CRYSTAX, +} +# recipes that are were already built +CORE_RECIPES = set([ + 'pyjnius', 'kivy', 'openssl', 'requests', 'sqlite3', 'setuptools', + 'numpy', 'android', +]) + + +def modified_recipes(branch='origin/master'): + """ + Returns a set of modified recipes between the current branch and the one + in param. + """ + # using the contrib version on purpose rather than sh.git, since it comes + # with a bunch of fixes, e.g. disabled TTY, see: + # https://stackoverflow.com/a/20128598/185510 + git_diff = sh.contrib.git.diff('--name-only', branch) + recipes = set() + for file_path in git_diff: + if 'pythonforandroid/recipes/' in file_path: + recipe = file_path.split('/')[2] + recipes.add(recipe) + return recipes + + +def build(target_python, requirements): + """ + Builds an APK given a target Python and a set of requirements. + """ + if not requirements: + return + testapp = 'setup_testapp_python2.py' + android_sdk_home = os.environ['ANDROID_SDK_HOME'] + android_ndk_home = os.environ['ANDROID_NDK_HOME'] + crystax_ndk_home = os.environ['CRYSTAX_NDK_HOME'] + if target_python == TargetPython.python3crystax: + android_ndk_home = crystax_ndk_home + testapp = 'setup_testapp_python3.py' + requirements.add(target_python.name) + requirements = ','.join(requirements) + print('requirements:', requirements) + with current_directory('testapps/'): + try: + for line in sh.python( + testapp, 'apk', '--sdk-dir', android_sdk_home, + '--ndk-dir', android_ndk_home, '--bootstrap', 'sdl2', '--requirements', + requirements, _err_to_out=True, _iter=True): + print(line) + except sh.ErrorReturnCode as e: + raise + + +def main(): + target_python = TargetPython.python3crystax + recipes = modified_recipes() + print('recipes modified:', recipes) + broken_recipes = BROKEN_RECIPES[target_python] + recipes = recipes - (CORE_RECIPES | broken_recipes) + print('recipes to build:', recipes) + build(target_python, recipes) + print(recipes) + + +if __name__ == '__main__': + main() diff --git a/pythonforandroid/recipes/babel/__init__.py b/pythonforandroid/recipes/babel/__init__.py index 5585577eb0..dc1fad86f1 100644 --- a/pythonforandroid/recipes/babel/__init__.py +++ b/pythonforandroid/recipes/babel/__init__.py @@ -3,7 +3,7 @@ class BabelRecipe(PythonRecipe): name = 'babel' - version = '2.1.1' + version = '2.2.0' url = 'https://pypi.python.org/packages/source/B/Babel/Babel-{version}.tar.gz' depends = [('python2', 'python3crystax'), 'setuptools', 'pytz'] diff --git a/pythonforandroid/recipes/cffi/__init__.py b/pythonforandroid/recipes/cffi/__init__.py index cb3ff4077e..67e6b35529 100644 --- a/pythonforandroid/recipes/cffi/__init__.py +++ b/pythonforandroid/recipes/cffi/__init__.py @@ -3,6 +3,9 @@ class CffiRecipe(CompiledComponentsPythonRecipe): + """ + Extra system dependencies: autoconf, automake and libtool. + """ name = 'cffi' version = '1.4.2' url = 'https://pypi.python.org/packages/source/c/cffi/cffi-{version}.tar.gz' diff --git a/setup.py b/setup.py index 94a2225a83..4dce3bb89f 100644 --- a/setup.py +++ b/setup.py @@ -16,12 +16,9 @@ data_files = [] -if os.name == 'nt': - install_reqs = ['appdirs', 'colorama>=0.3.3', 'jinja2', - 'six'] -else: - install_reqs = ['appdirs', 'colorama>=0.3.3', 'sh>=1.10', 'jinja2', - 'six'] +install_reqs = ['appdirs', 'colorama>=0.3.3', 'jinja2', 'six', 'enum34'] +if os.name != 'nt': + install_reqs.append('sh>=1.10') # By specifying every file manually, package_data will be able to # include them in binary distributions. Note that we have to add diff --git a/tox.ini b/tox.ini index 706bf73254..e27c4015d8 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ commands = pytest {posargs:tests/} [testenv:pep8] deps = flake8 -commands = flake8 pythonforandroid/ tests/ +commands = flake8 pythonforandroid/ tests/ ci/ [flake8] ignore =