From f5e78b7685fbf3c09428a8b97cdea5325882e15e Mon Sep 17 00:00:00 2001 From: opacam Date: Sun, 30 Jun 2019 19:20:53 +0200 Subject: [PATCH 01/41] [docs] Add documentation: testing a pull request --- doc/source/index.rst | 1 + doc/source/testing_pull_requests.rst | 226 +++++++++++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 doc/source/testing_pull_requests.rst diff --git a/doc/source/index.rst b/doc/source/index.rst index 16d6162dc3..84a02b962e 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -38,6 +38,7 @@ Contents troubleshooting docker contribute + testing_pull_requests Indices and tables diff --git a/doc/source/testing_pull_requests.rst b/doc/source/testing_pull_requests.rst new file mode 100644 index 0000000000..603828db77 --- /dev/null +++ b/doc/source/testing_pull_requests.rst @@ -0,0 +1,226 @@ +Testing an python-for-android pull request +========================================== + +In order to test a pull request, we recommend to consider the following points: + + #. of course, check if the overall thing makes sense + #. is the CI passing? if not what specifically fails + #. is it working locally at compile time? + #. is it working on device at runtime? + +This document will focus on the third point: +`is it working locally at compile time?` so we will give some hints about how +to proceed in order to create a local copy of the pull requests and build an +apk. We expect that the contributors has enough criteria/knowledge to perform +the other steps mentioned, so let's begin... + +To create an apk from a python-for-android pull request we contemplate three +possible scenarios: + + - using python-for-android commands directly from the pull request files + that we want to test, without installing it (the recommended way for most + of the test cases) + - installing python-for-android using the github's branch of the pull request + - using buildozer and a custom app + +We will explain the first two methods using one of the distributed +python-for-android test apps and we assume that you already have the +python-for-android dependencies installed. For the `buildozer` method we also +expect that you already have a a properly working app to test and a working +installation/configuration of `buildozer`. There is one step that it's shared +with all the testing methods that we propose in here...we named it +`Common steps`. + + +Common steps +^^^^^^^^^^^^ +The first step to do it's to get a copy of the pull request, we can do it of +several ways, and that it will depend of the circumstances but all the methods +presented here will do the job, so... + +Fetch the pull request by number +-------------------------------- +For the example, we will use `1901` for the example) and the pull request +branch that we will use is `feature-fix-numpy`, then you will use a variation +of the following git command: +`git fetch origin pull/<#>/head:`, eg.:: + + .. codeblock:: bash + + git fetch upstream pull/1901/head:feature-fix-numpy + +.. note:: Notice that we fetch from `upstream`, since that is the original + project, where the pull request is supposed to be + +.. tip:: The amount of work of some users maybe worth it to add his remote + to your fork's git configuration, to do so with the imaginary + github user `Obi-Wan Kenobi` which nickname is `obiwankenobi`, you + will do:: + + .. codeblock:: bash + + git remote add obiwankenobi https://github.com/obiwankenobi/python-for-android.git + + And to fetch the pull request branch that we put as example, you + would do:: + + .. codeblock:: bash + + git fetch obiwankenobi + git checkout obiwankenobi/feature-fix-numpy + + +Clone the pull request branch from the user's fork +-------------------------------------------------- +Sometimes you may prefer to use directly the fork of the user, so you will get +the nickname of the user who created the pull request, let's take the same +imaginary user than before `obiwankenobi`:: + + .. codeblock:: bash + + git clone -b feature-fix-numpy \ + --single-branch \ + https://github.com/obiwankenobi/python-for-android.git \ + p4a-feature-fix-numpy + +Here's the above command explained line by line: + +- `git clone -b feature-fix-numpy`: we tell git that we want to clone the + branch named `feature-fix-numpy` +- `--single-branch`: we tell git that we only want that branch +- `https://github.com/obiwankenobi/python-for-android.git`: noticed the + nickname of the user that created the pull request: `obiwankenobi` in the + middle of the line? that should be changed as needed for each pull + request that you want to test +- `p4a-feature-fix-numpy`: the name of the cloned repository, so we can + have multiple clones of different prs in the same folder + +.. note:: You can view the author/branch information looking at the + subtitle of the pull request, near the pull request status (expected + an `open` status) + +Using python-for-android commands directly from the pull request files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Enter inside the directory of the cloned repository in the above + step and run p4a command with proper args, eg:: + + .. codeblock:: bash + + cd p4a-feature-fix-numpy + PYTHONPATH=. python3 -m pythonforandroid.toolchain apk \ + --private=testapps/testapp_sqlite_openssl \ + --dist-name=dist_test_app_python3_libs \ + --package=org.kivy \ + --name=test_app_python3_sqlite_openssl \ + --version=0.1 \ + --requirements=requests,peewee,sdl2,pyjnius,kivy,python3 \ + --ndk-dir=/media/DEVEL/Android/android-ndk-r20 \ + --sdk-dir=/media/DEVEL/Android/android-sdk-linux \ + --android-api=27 \ + --arch=arm64-v8a \ + --permission=INTERNET \ + --debug + +Things that you should know: + + + - The example above will build an testapp we will make use of the files of + the testapp named `testapp_sqlite_openssl.py` but we don't use the setup + file to build it so we must tell python-for-android what we want via + arguments + - be sure to at least edit the following arguments when running the above + command, since the default set in there it's unlikely that match your + installation: + + - `--ndk-dir`: An absolute path to your android's NDK dir + - `--sdk-dir`: An absolute path to your android's SDK dir + - `--debug`: this one enables the debug mode of python-for-android, + which will show all log messages of the build. You can omit this + one but it's worth it to be mentioned, since this it's useful to us + when trying to find the source of the problem when things goes + wrong + - The apk generated by the above command should be located at the root of + of the cloned repository, were you run the command to build the apk + - The testapps distributed with python-for-android are located at + `testapps` folder under the main folder project + - All the builds of python-for-android are located at + `~/.local/share/python-for-android` + - You should have a downloaded copy of the android's NDK and SDK + +Installing python-for-android using the github's branch of the pull request +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Enter inside the directory of the cloned repository mentioned in + `Common steps` and install it via pip, eg.:: + + .. codeblock:: bash + + cd p4a-feature-fix-numpy + pip3 install . --upgrade --user + +- Now, go inside the `testapps` directory (we assume that you still are inside + the cloned repository):: + + .. codeblock:: bash + + cd testapps + +- Run the build of the apk via the freshly installed copy of python-for-android + by running a similar command than below:: + + .. code-block:: bash + + python3 setup_testapp_python3_sqlite_openssl.py apk \ + --ndk-dir=/media/DEVEL/Android/android-ndk-r20 \ + --sdk-dir=/media/DEVEL/Android/android-sdk-linux \ + --android-api=27 \ + --arch=arm64-v8a \ + --debug + + +Things that you should know: + + - In the example above, we override some variables that are set in + `setup_testapp_python3_sqlite_openssl.py`, you could also override them + by editing this file + - be sure to at least edit the following arguments when running the above + command, since the default set in there it's unlikely that match your + installation: + + - `--ndk-dir`: An absolute path to your android's NDK dir + - `--sdk-dir`: An absolute path to your android's SDK dir + +.. tip:: if you don't want to mess up with the system's python, you could do + the same steps but inside a virtualenv + +.. warning:: Once you finish the pull request tests remember to go back to the + master or develop versions of python-for-android, since you just + installed the python-for-android files of the `pull request` + +Using buildozer with a custom app +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Edit your `buildozer.spec` file. You should search for the key + `p4a.source_dir` and set the right value so in the example posted in + `Common steps` it would look like this:: + + p4a.source_dir = /home/user/p4a_pull_requests/p4a-feature-fix-numpy + +- Run you buildozer command as usual, eg.:: + + buildozer android debug p4a --dist-name=dist-test-feature-fix-numpy + +.. note:: this method has the advantage, can be run without installing the + pull request version of python-for-android nor the android's + dependencies but has one problem...when things goes wrong you must + determine if it's a buildozer issue or a python-for-android one + +.. warning:: Once you finish the pull request tests remember to comment/edit + the `p4a.source_dir` constant that you just edited to test the + pull request + +.. tip:: this method it's useful for developing pull requests since you can + edit `p4a.source_dir` to point to your python-for-android fork and you + can test any branch you want only switching branches with: + `git checkout ` from inside your python-for-android fork \ No newline at end of file From 98cfb8ae6f53f72dbedb7b13df1bdcb4c8e5d428 Mon Sep 17 00:00:00 2001 From: Jonas Thiem Date: Wed, 26 Jun 2019 21:12:00 +0200 Subject: [PATCH 02/41] Make test_get_bootstraps_from_recipes() deterministic and better --- pythonforandroid/bootstrap.py | 138 +++++++++++++++++++++++++++++----- tests/test_bootstrap.py | 121 +++++++++++++++++++++++++++-- 2 files changed, 231 insertions(+), 28 deletions(-) diff --git a/pythonforandroid/bootstrap.py b/pythonforandroid/bootstrap.py index 3480b3241b..8c8419b274 100755 --- a/pythonforandroid/bootstrap.py +++ b/pythonforandroid/bootstrap.py @@ -1,10 +1,11 @@ +import functools +import glob +import importlib +import os from os.path import (join, dirname, isdir, normpath, splitext, basename) from os import listdir, walk, sep import sh import shlex -import glob -import importlib -import os import shutil from pythonforandroid.logger import (warning, shprint, info, logger, @@ -34,6 +35,35 @@ def copy_files(src_root, dest_root, override=True): os.makedirs(dest_file) +default_recipe_priorities = [ + "webview", "sdl2", "service_only" # last is highest +] +# ^^ NOTE: these are just the default priorities if no special rules +# apply (which you can find in the code below), so basically if no +# known graphical lib or web lib is used - in which case service_only +# is the most reasonable guess. + + +def _cmp_bootstraps_by_priority(a, b): + def rank_bootstrap(bootstrap): + """ Returns a ranking index for each bootstrap, + with higher priority ranked with higher number. """ + if bootstrap.name in default_recipe_priorities: + return default_recipe_priorities.index(bootstrap.name) + 1 + return 0 + + # Rank bootstraps in order: + rank_a = rank_bootstrap(a) + rank_b = rank_bootstrap(b) + if rank_a != rank_b: + return (rank_b - rank_a) + else: + if a.name < b.name: # alphabetic sort for determinism + return -1 + else: + return 1 + + class Bootstrap(object): '''An Android project template, containing recipe stuff for compilation and templated fields for APK info. @@ -138,36 +168,43 @@ def run_distribute(self): self.distribution.save_info(self.dist_dir) @classmethod - def list_bootstraps(cls): + def all_bootstraps(cls): '''Find all the available bootstraps and return them.''' forbidden_dirs = ('__pycache__', 'common') bootstraps_dir = join(dirname(__file__), 'bootstraps') + result = set() for name in listdir(bootstraps_dir): if name in forbidden_dirs: continue filen = join(bootstraps_dir, name) if isdir(filen): - yield name + result.add(name) + return result @classmethod - def get_bootstrap_from_recipes(cls, recipes, ctx): - '''Returns a bootstrap whose recipe requirements do not conflict with - the given recipes.''' + def get_usable_bootstraps_for_recipes(cls, recipes, ctx): + '''Returns all bootstrap whose recipe requirements do not conflict + with the given recipes, in no particular order.''' info('Trying to find a bootstrap that matches the given recipes.') bootstraps = [cls.get_bootstrap(name, ctx) - for name in cls.list_bootstraps()] - acceptable_bootstraps = [] + for name in cls.all_bootstraps()] + acceptable_bootstraps = set() + + # Find out which bootstraps are acceptable: for bs in bootstraps: if not bs.can_be_chosen_automatically: continue - possible_dependency_lists = expand_dependencies(bs.recipe_depends) + possible_dependency_lists = expand_dependencies(bs.recipe_depends, ctx) for possible_dependencies in possible_dependency_lists: ok = True + # Check if the bootstap's dependencies have an internal conflict: for recipe in possible_dependencies: recipe = Recipe.get_recipe(recipe, ctx) if any([conflict in recipes for conflict in recipe.conflicts]): ok = False break + # Check if bootstrap's dependencies conflict with chosen + # packages: for recipe in recipes: try: recipe = Recipe.get_recipe(recipe, ctx) @@ -180,14 +217,58 @@ def get_bootstrap_from_recipes(cls, recipes, ctx): ok = False break if ok and bs not in acceptable_bootstraps: - acceptable_bootstraps.append(bs) + acceptable_bootstraps.add(bs) + info('Found {} acceptable bootstraps: {}'.format( len(acceptable_bootstraps), [bs.name for bs in acceptable_bootstraps])) - if acceptable_bootstraps: - info('Using the first of these: {}' - .format(acceptable_bootstraps[0].name)) - return acceptable_bootstraps[0] + return acceptable_bootstraps + + @classmethod + def get_bootstrap_from_recipes(cls, recipes, ctx): + '''Picks a single recommended default bootstrap out of + all_usable_bootstraps_from_recipes() for the given reicpes, + and returns it.''' + + known_web_packages = {"flask"} # to pick webview over service_only + recipes_with_deps_lists = expand_dependencies(recipes, ctx) + acceptable_bootstraps = cls.get_usable_bootstraps_for_recipes( + recipes, ctx + ) + + def have_dependency_in_recipes(dep): + for dep_list in recipes_with_deps_lists: + if dep in dep_list: + return True + return False + + # Special rule: return SDL2 bootstrap if there's an sdl2 dep: + if (have_dependency_in_recipes("sdl2") and + "sdl2" in [b.name for b in acceptable_bootstraps] + ): + info('Using sdl2 bootstrap since it is in dependencies') + return cls.get_bootstrap("sdl2", ctx) + + # Special rule: return "webview" if we depend on common web recipe: + for possible_web_dep in known_web_packages: + if have_dependency_in_recipes(possible_web_dep): + # We have a web package dep! + if "webview" in [b.name for b in acceptable_bootstraps]: + info('Using webview bootstrap since common web packages ' + 'were found {}'.format( + known_web_packages.intersection(recipes) + )) + return cls.get_bootstrap("webview", ctx) + + prioritized_acceptable_bootstraps = sorted( + list(acceptable_bootstraps), + key=functools.cmp_to_key(_cmp_bootstraps_by_priority) + ) + + if prioritized_acceptable_bootstraps: + info('Using the highest ranked/first of these: {}' + .format(prioritized_acceptable_bootstraps[0].name)) + return prioritized_acceptable_bootstraps[0] return None @classmethod @@ -299,9 +380,26 @@ def fry_eggs(self, sitepackages): shprint(sh.rm, '-rf', d) -def expand_dependencies(recipes): +def expand_dependencies(recipes, ctx): + """ This function expands to lists of all different available + alternative recipe combinations, with the dependencies added in + ONLY for all the not-with-alternative recipes. + (So this is like the deps graph very simplified and incomplete, but + hopefully good enough for most basic bootstrap compatibility checks) + """ + + # Add in all the deps of recipes where there is no alternative: + recipes_with_deps = list(recipes) + for entry in recipes: + if not isinstance(entry, (tuple, list)) or len(entry) == 1: + if isinstance(entry, (tuple, list)): + entry = entry[0] + recipe = Recipe.get_recipe(entry, ctx) + recipes_with_deps += recipe.depends + + # Split up lists by available alternatives: recipe_lists = [[]] - for recipe in recipes: + for recipe in recipes_with_deps: if isinstance(recipe, (tuple, list)): new_recipe_lists = [] for alternative in recipe: @@ -311,6 +409,6 @@ def expand_dependencies(recipes): new_recipe_lists.append(new_list) recipe_lists = new_recipe_lists else: - for old_list in recipe_lists: - old_list.append(recipe) + for existing_list in recipe_lists: + existing_list.append(recipe) return recipe_lists diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index f8f625132d..f5435da08d 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -1,6 +1,6 @@ + import os import sh - import unittest try: @@ -9,12 +9,16 @@ # `Python 2` or lower than `Python 3.3` does not # have the `unittest.mock` module built-in import mock -from pythonforandroid.bootstrap import Bootstrap +from pythonforandroid.bootstrap import ( + _cmp_bootstraps_by_priority, Bootstrap, expand_dependencies, +) from pythonforandroid.distribution import Distribution from pythonforandroid.recipe import Recipe from pythonforandroid.archs import ArchARMv7_a from pythonforandroid.build import Context +from test_graph import get_fake_recipe + class BaseClassSetupBootstrap(object): """ @@ -90,7 +94,7 @@ def test_build_dist_dirs(self): - :meth:`~pythonforandroid.bootstrap.Bootstrap.get_dist_dir` - :meth:`~pythonforandroid.bootstrap.Bootstrap.get_common_dir` """ - bs = Bootstrap().get_bootstrap("sdl2", self.ctx) + bs = Bootstrap.get_bootstrap("sdl2", self.ctx) self.assertTrue( bs.get_build_dir().endswith("build/bootstrap_builds/sdl2-python3") @@ -100,32 +104,133 @@ def test_build_dist_dirs(self): bs.get_common_dir().endswith("pythonforandroid/bootstraps/common") ) - def test_list_bootstraps(self): + def test__cmp_bootstraps_by_priority(self): + # Test service_only has higher priority than sdl2: + # (higher priority = smaller number/comes first) + self.assertTrue(_cmp_bootstraps_by_priority( + Bootstrap.get_bootstrap("service_only", self.ctx), + Bootstrap.get_bootstrap("sdl2", self.ctx) + ) < 0) + + # Test a random bootstrap is always lower priority than sdl2: + class _FakeBootstrap(object): + def __init__(self, name): + self.name = name + bs1 = _FakeBootstrap("alpha") + bs2 = _FakeBootstrap("zeta") + self.assertTrue(_cmp_bootstraps_by_priority( + bs1, + Bootstrap.get_bootstrap("sdl2", self.ctx) + ) > 0) + self.assertTrue(_cmp_bootstraps_by_priority( + bs2, + Bootstrap.get_bootstrap("sdl2", self.ctx) + ) > 0) + + # Test bootstraps that aren't otherwise recognized are ranked + # alphabetically: + self.assertTrue(_cmp_bootstraps_by_priority( + bs2, + bs1, + ) > 0) + self.assertTrue(_cmp_bootstraps_by_priority( + bs1, + bs2, + ) < 0) + + def test_all_bootstraps(self): """A test which will initialize a bootstrap and will check if the method :meth:`~pythonforandroid.bootstrap.Bootstrap.list_bootstraps` returns the expected values, which should be: `empty", `service_only`, `webview` and `sdl2` """ expected_bootstraps = {"empty", "service_only", "webview", "sdl2"} - set_of_bootstraps = set(Bootstrap().list_bootstraps()) + set_of_bootstraps = Bootstrap.all_bootstraps() self.assertEqual( expected_bootstraps, expected_bootstraps & set_of_bootstraps ) self.assertEqual(len(expected_bootstraps), len(set_of_bootstraps)) + def test_expand_dependencies(self): + # Test dependency expansion of a recipe with no alternatives: + expanded_result_1 = expand_dependencies(["pysdl2"], self.ctx) + self.assertTrue( + {"sdl2", "pysdl2", "python3"} in + [set(s) for s in expanded_result_1] + ) + + # Test expansion of a single element but as tuple: + expanded_result_1 = expand_dependencies([("pysdl2",)], self.ctx) + self.assertTrue( + {"sdl2", "pysdl2", "python3"} in + [set(s) for s in expanded_result_1] + ) + + # Test all alternatives are listed (they won't have dependencies + # expanded since expand_dependencies() is too simplistic): + expanded_result_2 = expand_dependencies([("pysdl2", "kivy")], self.ctx) + self.assertEqual([["pysdl2"], ["kivy"]], expanded_result_2) + def test_get_bootstraps_from_recipes(self): """A test which will initialize a bootstrap and will check if the method :meth:`~pythonforandroid.bootstrap.Bootstrap. get_bootstraps_from_recipes` returns the expected values """ + + import pythonforandroid.recipe + original_get_recipe = pythonforandroid.recipe.Recipe.get_recipe + + # Test that SDL2 works with kivy: recipes_sdl2 = {"sdl2", "python3", "kivy"} - bs = Bootstrap().get_bootstrap_from_recipes(recipes_sdl2, self.ctx) + bs = Bootstrap.get_bootstrap_from_recipes(recipes_sdl2, self.ctx) + self.assertEqual(bs.name, "sdl2") + # Test that pysdl2 or kivy alone will also yield SDL2 (dependency): + recipes_pysdl2_only = {"pysdl2"} + bs = Bootstrap.get_bootstrap_from_recipes( + recipes_pysdl2_only, self.ctx + ) self.assertEqual(bs.name, "sdl2") + recipes_kivy_only = {"kivy"} + bs = Bootstrap.get_bootstrap_from_recipes( + recipes_kivy_only, self.ctx + ) + self.assertEqual(bs.name, "sdl2") + + with mock.patch("pythonforandroid.recipe.Recipe.get_recipe") as \ + mock_get_recipe: + # Test that something conflicting with sdl2 won't give sdl2: + def _add_sdl2_conflicting_recipe(name, ctx): + if name == "conflictswithsdl2": + if name not in pythonforandroid.recipe.Recipe.recipes: + pythonforandroid.recipe.Recipe.recipes[name] = ( + get_fake_recipe("sdl2", conflicts=["sdl2"]) + ) + return original_get_recipe(name, ctx) + mock_get_recipe.side_effect = _add_sdl2_conflicting_recipe + recipes_with_sdl2_conflict = {"python3", "conflictswithsdl2"} + bs = Bootstrap.get_bootstrap_from_recipes( + recipes_with_sdl2_conflict, self.ctx + ) + self.assertNotEqual(bs.name, "sdl2") + + # Test using flask will default to webview: + recipes_with_flask = {"python3", "flask"} + bs = Bootstrap.get_bootstrap_from_recipes( + recipes_with_flask, self.ctx + ) + self.assertEqual(bs.name, "webview") + + # Test using random packages will default to service_only: + recipes_with_no_sdl2_or_web = {"python3", "numpy"} + bs = Bootstrap.get_bootstrap_from_recipes( + recipes_with_no_sdl2_or_web, self.ctx + ) + self.assertEqual(bs.name, "service_only") - # test wrong recipes + # Test wrong recipes wrong_recipes = {"python2", "python3", "pyjnius"} - bs = Bootstrap().get_bootstrap_from_recipes(wrong_recipes, self.ctx) + bs = Bootstrap.get_bootstrap_from_recipes(wrong_recipes, self.ctx) self.assertIsNone(bs) @mock.patch("pythonforandroid.bootstrap.ensure_dir") From 1d505334b885dabe7efe63a36308081d94ee277a Mon Sep 17 00:00:00 2001 From: opacam Date: Tue, 9 Jul 2019 10:28:59 +0200 Subject: [PATCH 03/41] [docker] Update android's sdk tools to `28.0.2` Also: - remove unneeded calls to android's sdkmanager - remove download of android's platform 19 (because we don't need it) --- Dockerfile.py2 | 14 ++++++-------- Dockerfile.py3 | 14 ++++++-------- doc/source/quickstart.rst | 4 ++-- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/Dockerfile.py2 b/Dockerfile.py2 index 05f956aa22..5b9f2e2744 100644 --- a/Dockerfile.py2 +++ b/Dockerfile.py2 @@ -58,8 +58,8 @@ RUN ${RETRY} curl --location --progress-bar --insecure \ ENV ANDROID_SDK_HOME="${ANDROID_HOME}/android-sdk" # get the latest version from https://developer.android.com/studio/index.html -ENV ANDROID_SDK_TOOLS_VERSION="3859397" -ENV ANDROID_SDK_BUILD_TOOLS_VERSION="26.0.2" +ENV ANDROID_SDK_TOOLS_VERSION="4333796" +ENV ANDROID_SDK_BUILD_TOOLS_VERSION="28.0.2" ENV ANDROID_SDK_TOOLS_ARCHIVE="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip" ENV ANDROID_SDK_TOOLS_DL_URL="https://dl.google.com/android/repository/${ANDROID_SDK_TOOLS_ARCHIVE}" @@ -76,16 +76,14 @@ RUN mkdir --parents "${ANDROID_SDK_HOME}/.android/" \ && echo '### User Sources for Android SDK Manager' \ > "${ANDROID_SDK_HOME}/.android/repositories.cfg" -# accept Android licenses (JDK necessary!) +# Download and accept Android licenses (JDK necessary!) RUN ${RETRY} apt -y install -qq --no-install-recommends openjdk-8-jdk \ && apt -y autoremove RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;${ANDROID_SDK_BUILD_TOOLS_VERSION}" > /dev/null +RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-27" > /dev/null -# download platforms, API, build tools -RUN "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-19" && \ - "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-27" && \ - "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;${ANDROID_SDK_BUILD_TOOLS_VERSION}" && \ - chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager" +# Set avdmanager permissions (executable) +RUN chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager" ENV USER="user" diff --git a/Dockerfile.py3 b/Dockerfile.py3 index 6a8286e9fc..e441417112 100644 --- a/Dockerfile.py3 +++ b/Dockerfile.py3 @@ -58,8 +58,8 @@ RUN ${RETRY} curl --location --progress-bar --insecure \ ENV ANDROID_SDK_HOME="${ANDROID_HOME}/android-sdk" # get the latest version from https://developer.android.com/studio/index.html -ENV ANDROID_SDK_TOOLS_VERSION="3859397" -ENV ANDROID_SDK_BUILD_TOOLS_VERSION="26.0.2" +ENV ANDROID_SDK_TOOLS_VERSION="4333796" +ENV ANDROID_SDK_BUILD_TOOLS_VERSION="28.0.2" ENV ANDROID_SDK_TOOLS_ARCHIVE="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip" ENV ANDROID_SDK_TOOLS_DL_URL="https://dl.google.com/android/repository/${ANDROID_SDK_TOOLS_ARCHIVE}" @@ -76,16 +76,14 @@ RUN mkdir --parents "${ANDROID_SDK_HOME}/.android/" \ && echo '### User Sources for Android SDK Manager' \ > "${ANDROID_SDK_HOME}/.android/repositories.cfg" -# accept Android licenses (JDK necessary!) +# Download and accept Android licenses (JDK necessary!) RUN ${RETRY} apt -y install -qq --no-install-recommends openjdk-8-jdk \ && apt -y autoremove RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;${ANDROID_SDK_BUILD_TOOLS_VERSION}" > /dev/null +RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-27" > /dev/null -# download platforms, API, build tools -RUN "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-19" && \ - "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-27" && \ - "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;${ANDROID_SDK_BUILD_TOOLS_VERSION}" && \ - chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager" +# Set avdmanager permissions (executable) +RUN chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager" ENV USER="user" diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index f92b475542..1f50579dde 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -131,9 +131,9 @@ API/NDK API level 21**: Second, install the build-tools. You can use ``$SDK_DIR/tools/bin/sdkmanager --list`` to see all the -possibilities, but 26.0.2 is the latest version at the time of writing:: +possibilities, but 28.0.2 is the latest version at the time of writing:: - $SDK_DIR/tools/bin/sdkmanager "build-tools;26.0.2" + $SDK_DIR/tools/bin/sdkmanager "build-tools;28.0.2" Configure p4a to use your SDK/NDK ````````````````````````````````` From 6d6ab20e7307877486154e47b61dd197cb34d20a Mon Sep 17 00:00:00 2001 From: opacam Date: Wed, 10 Jul 2019 11:12:43 +0200 Subject: [PATCH 04/41] [bootstrap] Fix crash when guessing Bootstrap (expand_dependencies) When a pure python package it's supplied inside the dependency list (for the method `expand_dependencies`), we will get a crash because we don't have a recipe for it, unless we contemplate that situation --- pythonforandroid/bootstrap.py | 9 +++++++-- tests/test_bootstrap.py | 12 ++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/pythonforandroid/bootstrap.py b/pythonforandroid/bootstrap.py index 8c8419b274..d58e20f2bf 100755 --- a/pythonforandroid/bootstrap.py +++ b/pythonforandroid/bootstrap.py @@ -394,8 +394,13 @@ def expand_dependencies(recipes, ctx): if not isinstance(entry, (tuple, list)) or len(entry) == 1: if isinstance(entry, (tuple, list)): entry = entry[0] - recipe = Recipe.get_recipe(entry, ctx) - recipes_with_deps += recipe.depends + try: + recipe = Recipe.get_recipe(entry, ctx) + recipes_with_deps += recipe.depends + except ValueError: + # it's a pure python package without a recipe, so we + # don't know the dependencies...skipping for now + pass # Split up lists by available alternatives: recipe_lists = [[]] diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index f5435da08d..ad62a45b15 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -171,6 +171,18 @@ def test_expand_dependencies(self): expanded_result_2 = expand_dependencies([("pysdl2", "kivy")], self.ctx) self.assertEqual([["pysdl2"], ["kivy"]], expanded_result_2) + def test_expand_dependencies_with_pure_python_package(self): + """Check that `expanded_dependencies`, with a pure python package as + one of the dependencies, returns a list of dependencies + """ + expanded_result = expand_dependencies( + ["python3", "kivy", "peewee"], self.ctx + ) + self.assertEqual(len(expanded_result), 3) + self.assertIsInstance(expanded_result, list) + for i in expanded_result: + self.assertIsInstance(i, list) + def test_get_bootstraps_from_recipes(self): """A test which will initialize a bootstrap and will check if the method :meth:`~pythonforandroid.bootstrap.Bootstrap. From b5d9b611be233e9d47daf04e143c5eff715d5a04 Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Wed, 10 Jul 2019 18:31:26 +0200 Subject: [PATCH 05/41] Feature gitignore additions (#1911) * [gitignore] Add test/coverage entries to gitignore * [gitignore] Add `testapps/build/` to gitignore * [gitignore] Add `.directory` to gitignore Those files are autogenerated by `Dolphin`, the KDE's file manager --- .gitignore | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6a8964aa0f..8b3efa2f6f 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,20 @@ __pycache__/ #idea/pycharm .idea/ -.tox \ No newline at end of file + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.pytest_cache/ + +# testapp's build folder +testapps/build/ + +# Dolphin (the KDE file manager autogenerates the file `.directory`) +.directory From a948a99e689c8a0fe71b7d83b7b6b5f22b13a3d5 Mon Sep 17 00:00:00 2001 From: opacam Date: Wed, 10 Jul 2019 20:42:13 +0200 Subject: [PATCH 06/41] [gitignore] Remove `nosetests.xml` from gitignore Because we don't use it (we are using pytest), so better keep it simple Note: this was introduced by mistake in b5d9b611 --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8b3efa2f6f..05196a8acb 100644 --- a/.gitignore +++ b/.gitignore @@ -28,7 +28,6 @@ htmlcov/ .coverage .coverage.* .cache -nosetests.xml coverage.xml *.cover .pytest_cache/ From 7338ae4306c6f045bbbfde589f26a6a917e538df Mon Sep 17 00:00:00 2001 From: Mirko Date: Thu, 11 Jul 2019 09:52:35 +0200 Subject: [PATCH 07/41] Fixes ffmpeg and libx264 recipes for arm64-v8 (#1916) libx264 and ffmpeg recipes fixes for arm64-v8 --- pythonforandroid/recipes/ffmpeg/__init__.py | 26 ++++++++++++++------ pythonforandroid/recipes/libx264/__init__.py | 6 ++++- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/pythonforandroid/recipes/ffmpeg/__init__.py b/pythonforandroid/recipes/ffmpeg/__init__.py index 85bdb8ad06..4a04f4844f 100644 --- a/pythonforandroid/recipes/ffmpeg/__init__.py +++ b/pythonforandroid/recipes/ffmpeg/__init__.py @@ -98,20 +98,29 @@ def build_arch(self, arch): '--enable-shared', ] + if 'arm64' in arch.arch: + cross_prefix = 'aarch64-linux-android-' + arch_flag = 'aarch64' + else: + cross_prefix = 'arm-linux-androideabi-' + arch_flag = 'arm' + # android: flags += [ '--target-os=android', - '--cross-prefix=arm-linux-androideabi-', - '--arch=arm', + '--cross-prefix={}'.format(cross_prefix), + '--arch={}'.format(arch_flag), '--sysroot=' + self.ctx.ndk_platform, '--enable-neon', '--prefix={}'.format(realpath('.')), ] - cflags += [ - '-mfpu=vfpv3-d16', - '-mfloat-abi=softfp', - '-fPIC', - ] + + if arch_flag == 'arm': + cflags += [ + '-mfpu=vfpv3-d16', + '-mfloat-abi=softfp', + '-fPIC', + ] env['CFLAGS'] += ' ' + ' '.join(cflags) env['LDFLAGS'] += ' ' + ' '.join(ldflags) @@ -121,7 +130,8 @@ def build_arch(self, arch): shprint(sh.make, '-j4', _env=env) shprint(sh.make, 'install', _env=env) # copy libs: - sh.cp('-a', sh.glob('./lib/lib*.so'), self.ctx.get_libs_dir(arch.arch)) + sh.cp('-a', sh.glob('./lib/lib*.so'), + self.ctx.get_libs_dir(arch.arch)) recipe = FFMpegRecipe() diff --git a/pythonforandroid/recipes/libx264/__init__.py b/pythonforandroid/recipes/libx264/__init__.py index c139b4ce74..89d48c8410 100644 --- a/pythonforandroid/recipes/libx264/__init__.py +++ b/pythonforandroid/recipes/libx264/__init__.py @@ -14,9 +14,13 @@ def should_build(self, arch): def build_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): env = self.get_recipe_env(arch) + if 'arm64' in arch.arch: + cross_prefix = 'aarch64-linux-android-' + else: + cross_prefix = 'arm-linux-androideabi-' configure = sh.Command('./configure') shprint(configure, - '--cross-prefix=arm-linux-androideabi-', + '--cross-prefix={}'.format(cross_prefix), '--host=arm-linux', '--disable-asm', '--disable-cli', From e3f2d81a35a3047e4b1b1622a51a31735456ba36 Mon Sep 17 00:00:00 2001 From: opacam Date: Tue, 9 Jul 2019 14:10:02 +0200 Subject: [PATCH 08/41] [crystax] Drop CrystaX support and code base Closes #1905 --- ci/constants.py | 2 - doc/source/buildoptions.rst | 33 +++--- doc/source/docker.rst | 11 +- pythonforandroid/archs.py | 11 +- pythonforandroid/bootstrap.py | 8 +- .../bootstraps/common/build/build.py | 19 ++-- .../build/jni/application/src/Android.mk | 4 - .../common/build/jni/application/src/start.c | 40 ++----- .../java/org/kivy/android/PythonUtil.java | 1 - .../java/org/kivy/android/PythonUtil.java | 1 - .../build/jni/application/src/Android.mk | 4 - .../build/jni/application/src/Android.mk | 4 - pythonforandroid/build.py | 7 +- pythonforandroid/python.py | 14 +-- pythonforandroid/recipe.py | 67 +++--------- pythonforandroid/recipes/cymunk/__init__.py | 2 - pythonforandroid/recipes/flask/__init__.py | 2 +- .../recipes/genericndkbuild/__init__.py | 2 +- .../recipes/hostpython2/__init__.py | 2 +- .../recipes/hostpython3/__init__.py | 2 +- .../recipes/hostpython3crystax/__init__.py | 44 -------- pythonforandroid/recipes/jedi/__init__.py | 2 - pythonforandroid/recipes/numpy/__init__.py | 5 - pythonforandroid/recipes/openal/__init__.py | 3 - pythonforandroid/recipes/openssl/__init__.py | 5 - pythonforandroid/recipes/pysha3/__init__.py | 7 +- pythonforandroid/recipes/python2/__init__.py | 2 +- pythonforandroid/recipes/python3/__init__.py | 2 +- .../recipes/python3crystax/__init__.py | 102 ------------------ .../recipes/secp256k1/__init__.py | 11 +- .../recipes/setuptools/__init__.py | 1 - pythonforandroid/recipes/six/__init__.py | 1 - pythonforandroid/toolchain.py | 2 +- testapps/setup_keyboard.py | 2 +- testapps/setup_testapp_flask.py | 2 +- testapps/setup_testapp_python2.py | 2 +- .../setup_testapp_python2_sqlite_openssl.py | 2 +- testapps/setup_testapp_python3crystax.py | 30 ------ testapps/setup_vispy.py | 2 +- tests/test_graph.py | 9 +- 40 files changed, 79 insertions(+), 393 deletions(-) delete mode 100644 pythonforandroid/recipes/hostpython3crystax/__init__.py delete mode 100644 pythonforandroid/recipes/python3crystax/__init__.py delete mode 100644 testapps/setup_testapp_python3crystax.py diff --git a/ci/constants.py b/ci/constants.py index 27e9d31a1e..56ab5bff62 100644 --- a/ci/constants.py +++ b/ci/constants.py @@ -3,7 +3,6 @@ class TargetPython(Enum): python2 = 0 - python3crystax = 1 python3 = 2 @@ -22,7 +21,6 @@ class TargetPython(Enum): 'ffpyplayer', 'flask', 'groestlcoin_hash', - 'hostpython3crystax', # https://github.com/kivy/python-for-android/issues/1354 'kiwisolver', 'libmysqlclient', diff --git a/doc/source/buildoptions.rst b/doc/source/buildoptions.rst index f2d7b26f64..ddb81be092 100644 --- a/doc/source/buildoptions.rst +++ b/doc/source/buildoptions.rst @@ -33,25 +33,20 @@ e.g. ``--requirements=python3``. CrystaX python3 -############### - -.. warning:: python-for-android originally supported Python 3 using the CrystaX - NDK. This support is now being phased out as CrystaX is no longer - actively developed. - -.. note:: You must manually download the `CrystaX NDK - `__ and tell - python-for-android to use it with ``--ndk-dir /path/to/NDK``. - -Select this by adding the ``python3crystax`` recipe to your -requirements, e.g. ``--requirements=python3crystax``. - -This uses the prebuilt Python from the `CrystaX NDK -`__, a drop-in replacement for -Google's official NDK which includes many improvements. You -*must* use the CrystaX NDK 10.3.0 or higher when building with -python3. You can get it `here -`__. +~~~~~~~~~~~~~~~ + +python-for-android originally supported Python 3 using the CrystaX NDK. Since +we have a working python3 recipe, we don't support CrystaX NDK anymore. If you +were using `python3crystax`, we recommend to give it a try to the new `python3` +recipe. + +.. note:: Since we don't support `python3crystax` anymore, the old instructions + has been removed from here. If you, still have the need to make use + of this old recipe, you should do it with an old `python-for-android` + release. Probably, a got starting point would be `version 0.7.0 + `__ or + if that doesn't work , then go for `version 0.6.0 + `__ .. _bootstrap_build_options: diff --git a/doc/source/docker.rst b/doc/source/docker.rst index 3ff9267f73..28db26d49c 100644 --- a/doc/source/docker.rst +++ b/doc/source/docker.rst @@ -17,12 +17,11 @@ already have Docker preinstalled and set up. .. warning:: This approach is highly space unfriendly! The more layers (``commit``) or even Docker images (``build``) you create the more space it'll consume. - Within the Docker image there is Android + Crystax SDK and NDK + various - dependencies. Within the custom diff made by building the distribution - there is another big chunk of space eaten. The very basic stuff such as - a distribution with: CPython 3, setuptools, Python for Android ``android`` - module, SDL2 (+ deps), PyJNIus and Kivy takes almost 13 GB. Check your free - space first! + Within the Docker image there is Android SDK and NDK + various dependencies. + Within the custom diff made by building the distribution there is another + big chunk of space eaten. The very basic stuff such as a distribution with: + CPython 3, setuptools, Python for Android ``android`` module, SDL2 (+ deps), + PyJNIus and Kivy takes almost 13 GB. Check your free space first! 1. Clone the repository:: diff --git a/pythonforandroid/archs.py b/pythonforandroid/archs.py index a27067ab1a..42f143ed21 100644 --- a/pythonforandroid/archs.py +++ b/pythonforandroid/archs.py @@ -92,9 +92,6 @@ def get_env(self, with_flags_in_cc=True, clang=False): env["LDFLAGS"] += " ".join(['-lm', '-L' + self.ctx.get_libs_dir(self.arch)]) - if self.ctx.ndk == 'crystax': - env['LDFLAGS'] += ' -L{}/sources/crystax/libs/{} -lcrystax'.format(self.ctx.ndk_dir, self.arch) - toolchain_prefix = self.ctx.toolchain_prefix toolchain_version = self.ctx.toolchain_version command_prefix = self.command_prefix @@ -154,10 +151,7 @@ def get_env(self, with_flags_in_cc=True, clang=False): env['LD'] = '{}-ld'.format(command_prefix) env['LDSHARED'] = env["CC"] + " -pthread -shared " +\ "-Wl,-O1 -Wl,-Bsymbolic-functions " - if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax: - # For crystax python, we can't use the host python headers: - env["CFLAGS"] += ' -I{}/sources/python/{}/include/python/'.\ - format(self.ctx.ndk_dir, self.ctx.python_recipe.version[0:3]) + env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix) env['MAKE'] = 'make -j5' env['READELF'] = '{}-readelf'.format(command_prefix) @@ -180,9 +174,6 @@ def get_env(self, with_flags_in_cc=True, clang=False): env['ARCH'] = self.arch env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api)) - if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax: - env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version - return env diff --git a/pythonforandroid/bootstrap.py b/pythonforandroid/bootstrap.py index d58e20f2bf..cd11c6e1a9 100755 --- a/pythonforandroid/bootstrap.py +++ b/pythonforandroid/bootstrap.py @@ -80,10 +80,7 @@ class Bootstrap(object): distribution = None # All bootstraps should include Python in some way: - recipe_depends = [ - ("python2", "python3", "python3crystax"), - 'android', - ] + recipe_depends = [("python2", "python3"), 'android'] can_be_chosen_automatically = True '''Determines whether the bootstrap can be chosen as one that @@ -345,9 +342,6 @@ def _unpack_aar(self, aar, arch): def strip_libraries(self, arch): info('Stripping libraries') - if self.ctx.python_recipe.from_crystax: - info('Python was loaded from CrystaX, skipping strip') - return env = arch.get_env() tokens = shlex.split(env['STRIP']) strip = sh.Command(tokens[0]) diff --git a/pythonforandroid/bootstraps/common/build/build.py b/pythonforandroid/bootstraps/common/build/build.py index 2b69082f14..ed5e708892 100644 --- a/pythonforandroid/bootstraps/common/build/build.py +++ b/pythonforandroid/bootstraps/common/build/build.py @@ -332,9 +332,7 @@ def make_package(args): shutil.copyfile(join(args.private, "main.py"), join(main_py_only_dir, "main.py")) tar_dirs.append(main_py_only_dir) - for python_bundle_dir in ('private', - 'crystax_python', - '_python_bundle'): + for python_bundle_dir in ('private', '_python_bundle'): if exists(python_bundle_dir): tar_dirs.append(python_bundle_dir) if get_bootstrap_name() == "webview": @@ -783,14 +781,13 @@ def _read_configuration(): if args.try_system_python_compile: # Hardcoding python2.7 is okay for now, as python3 skips the # compilation anyway - if not exists('crystax_python'): - python_executable = 'python2.7' - try: - subprocess.call([python_executable, '--version']) - except (OSError, subprocess.CalledProcessError): - pass - else: - PYTHON = python_executable + python_executable = 'python2.7' + try: + subprocess.call([python_executable, '--version']) + except (OSError, subprocess.CalledProcessError): + pass + else: + PYTHON = python_executable if args.no_compile_pyo: PYTHON = None diff --git a/pythonforandroid/bootstraps/common/build/jni/application/src/Android.mk b/pythonforandroid/bootstraps/common/build/jni/application/src/Android.mk index 4a442eeb32..fb2b17719d 100644 --- a/pythonforandroid/bootstraps/common/build/jni/application/src/Android.mk +++ b/pythonforandroid/bootstraps/common/build/jni/application/src/Android.mk @@ -21,7 +21,3 @@ LOCAL_LDLIBS := -lGLESv1_CM -lGLESv2 -llog $(EXTRA_LDLIBS) LOCAL_LDFLAGS += -L$(PYTHON_LINK_ROOT) $(APPLICATION_ADDITIONAL_LDFLAGS) include $(BUILD_SHARED_LIBRARY) - -ifdef CRYSTAX_PYTHON_VERSION - $(call import-module,python/$(CRYSTAX_PYTHON_VERSION)) -endif diff --git a/pythonforandroid/bootstraps/common/build/jni/application/src/start.c b/pythonforandroid/bootstraps/common/build/jni/application/src/start.c index a88ec74c74..24297accdb 100644 --- a/pythonforandroid/bootstraps/common/build/jni/application/src/start.c +++ b/pythonforandroid/bootstraps/common/build/jni/application/src/start.c @@ -165,26 +165,14 @@ int main(int argc, char *argv[]) { // Set up the python path char paths[256]; - char crystax_python_dir[256]; - snprintf(crystax_python_dir, 256, - "%s/crystax_python", getenv("ANDROID_UNPACK")); char python_bundle_dir[256]; snprintf(python_bundle_dir, 256, "%s/_python_bundle", getenv("ANDROID_UNPACK")); - if (dir_exists(crystax_python_dir) || dir_exists(python_bundle_dir)) { - if (dir_exists(crystax_python_dir)) { - LOGP("crystax_python exists"); - snprintf(paths, 256, - "%s/stdlib.zip:%s/modules", - crystax_python_dir, crystax_python_dir); - } - - if (dir_exists(python_bundle_dir)) { - LOGP("_python_bundle dir exists"); - snprintf(paths, 256, - "%s/stdlib.zip:%s/modules", - python_bundle_dir, python_bundle_dir); - } + if (dir_exists(python_bundle_dir)) { + LOGP("_python_bundle dir exists"); + snprintf(paths, 256, + "%s/stdlib.zip:%s/modules", + python_bundle_dir, python_bundle_dir); LOGP("calculated paths to be..."); LOGP(paths); @@ -196,10 +184,8 @@ int main(int argc, char *argv[]) { LOGP("set wchar paths..."); } else { - // We do not expect to see crystax_python any more, so no point - // reminding the user about it. If it does exist, we'll have - // logged it earlier. - LOGP("_python_bundle does not exist"); + LOGP("_python_bundle does not exist...this not looks good, all python" + " recipes should have this folder, should we expect a crash soon?"); } Py_Initialize(); @@ -234,18 +220,6 @@ int main(int argc, char *argv[]) { PyRun_SimpleString("import sys, posix\n"); char add_site_packages_dir[256]; - if (dir_exists(crystax_python_dir)) { - snprintf(add_site_packages_dir, 256, - "sys.path.append('%s/site-packages')", - crystax_python_dir); - - PyRun_SimpleString("import sys\n" - "sys.argv = ['notaninterpreterreally']\n" - "from os.path import realpath, join, dirname"); - PyRun_SimpleString(add_site_packages_dir); - /* "sys.path.append(join(dirname(realpath(__file__)), 'site-packages'))") */ - PyRun_SimpleString("sys.path = ['.'] + sys.path"); - } if (dir_exists(python_bundle_dir)) { snprintf(add_site_packages_dir, 256, diff --git a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java index 1f2673850d..2bb1cf3607 100644 --- a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java +++ b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java @@ -32,7 +32,6 @@ protected static void addLibraryIfExists(ArrayList libsList, String patt protected static ArrayList getLibraries(File libsDir) { ArrayList libsList = new ArrayList(); - addLibraryIfExists(libsList, "crystax", libsDir); addLibraryIfExists(libsList, "sqlite3", libsDir); addLibraryIfExists(libsList, "ffi", libsDir); addLibraryIfExists(libsList, "ssl.*", libsDir); diff --git a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonUtil.java b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonUtil.java index c1180addfa..d7eb704f12 100644 --- a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonUtil.java +++ b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonUtil.java @@ -31,7 +31,6 @@ protected static void addLibraryIfExists(ArrayList libsList, String patt protected static ArrayList getLibraries(File libsDir) { ArrayList libsList = new ArrayList(); - addLibraryIfExists(libsList, "crystax", libsDir); addLibraryIfExists(libsList, "sqlite3", libsDir); addLibraryIfExists(libsList, "ffi", libsDir); addLibraryIfExists(libsList, "png16", libsDir); diff --git a/pythonforandroid/bootstraps/service_only/build/jni/application/src/Android.mk b/pythonforandroid/bootstraps/service_only/build/jni/application/src/Android.mk index 0bc42bfb89..dc351a3319 100644 --- a/pythonforandroid/bootstraps/service_only/build/jni/application/src/Android.mk +++ b/pythonforandroid/bootstraps/service_only/build/jni/application/src/Android.mk @@ -16,7 +16,3 @@ LOCAL_LDLIBS := -llog $(EXTRA_LDLIBS) LOCAL_LDFLAGS += -L$(PYTHON_LINK_ROOT) $(APPLICATION_ADDITIONAL_LDFLAGS) include $(BUILD_SHARED_LIBRARY) - -ifdef CRYSTAX_PYTHON_VERSION - $(call import-module,python/$(CRYSTAX_PYTHON_VERSION)) -endif diff --git a/pythonforandroid/bootstraps/webview/build/jni/application/src/Android.mk b/pythonforandroid/bootstraps/webview/build/jni/application/src/Android.mk index 20399573c9..5fbc4cd365 100644 --- a/pythonforandroid/bootstraps/webview/build/jni/application/src/Android.mk +++ b/pythonforandroid/bootstraps/webview/build/jni/application/src/Android.mk @@ -18,7 +18,3 @@ LOCAL_LDLIBS := -llog $(EXTRA_LDLIBS) LOCAL_LDFLAGS += -L$(PYTHON_LINK_ROOT) $(APPLICATION_ADDITIONAL_LDFLAGS) include $(BUILD_SHARED_LIBRARY) - -ifdef CRYSTAX_PYTHON_VERSION - $(call import-module,python/$(CRYSTAX_PYTHON_VERSION)) -endif diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index 19893d1619..f0fbc86e9d 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -561,10 +561,13 @@ def build_recipes(build_order, python_modules, ctx, project_dir, # 4) biglink everything info_main('# Biglinking object files') - if not ctx.python_recipe or not ctx.python_recipe.from_crystax: + if not ctx.python_recipe: biglink(ctx, arch) else: - info('NDK is crystax, skipping biglink (will this work?)') + warning( + "Context's python recipe found, " + "skipping biglink (will this work?)" + ) # 5) postbuild packages info_main('# Postbuilding recipes') diff --git a/pythonforandroid/python.py b/pythonforandroid/python.py index 3a214ee714..ab9035e11a 100755 --- a/pythonforandroid/python.py +++ b/pythonforandroid/python.py @@ -49,13 +49,9 @@ class GuestPythonRecipe(TargetPythonRecipe): this limitation. ''' - from_crystax = False - '''True if the python is used from CrystaX, False otherwise (i.e. if - it is built by p4a).''' - configure_args = () '''The configure arguments needed to build the python recipe. Those are - used in method :meth:`build_arch` (if not overwritten like python3crystax's + used in method :meth:`build_arch` (if not overwritten like python3's recipe does). .. note:: This variable should be properly set in subclass. @@ -108,10 +104,6 @@ def __init__(self, *args, **kwargs): super(GuestPythonRecipe, self).__init__(*args, **kwargs) def get_recipe_env(self, arch=None, with_flags_in_cc=True): - if self.from_crystax: - return super(GuestPythonRecipe, self).get_recipe_env( - arch=arch, with_flags_in_cc=with_flags_in_cc) - env = environ.copy() android_host = env['HOSTARCH'] = arch.command_prefix @@ -215,10 +207,6 @@ def add_flags(include_flags, link_dirs, link_libs): def prebuild_arch(self, arch): super(TargetPythonRecipe, self).prebuild_arch(arch) - if self.from_crystax and self.ctx.ndk != 'crystax': - raise BuildInterruptingException( - 'The {} recipe can only be built when using the CrystaX NDK. ' - 'Exiting.'.format(self.name)) self.ctx.python_recipe = self def build_arch(self, arch): diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 9b07a3cf1b..6429c6ea97 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -730,7 +730,7 @@ class PythonRecipe(Recipe): def __init__(self, *args, **kwargs): super(PythonRecipe, self).__init__(*args, **kwargs) depends = self.depends - depends.append(('python2', 'python3', 'python3crystax')) + depends.append(('python2', 'python3')) depends = list(set(depends)) self.depends = depends @@ -753,8 +753,6 @@ def real_hostpython_location(self): host_build = Recipe.get_recipe(host_name, self.ctx).get_build_dir() if host_name in ['hostpython2', 'hostpython3']: return join(host_build, 'native-build', 'python') - elif host_name in ['hostpython3crystax']: - return join(host_build, 'hostpython') else: python_recipe = self.ctx.python_recipe return 'python{}'.format(python_recipe.version) @@ -783,27 +781,16 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): env['LANG'] = "en_GB.UTF-8" if not self.call_hostpython_via_targetpython: - # sets python headers/linkages...depending on python's recipe python_name = self.ctx.python_recipe.name - python_version = self.ctx.python_recipe.version - python_short_version = '.'.join(python_version.split('.')[:2]) - if not self.ctx.python_recipe.from_crystax: - env['CFLAGS'] += ' -I{}'.format( - self.ctx.python_recipe.include_root(arch.arch)) - env['LDFLAGS'] += ' -L{} -lpython{}'.format( - self.ctx.python_recipe.link_root(arch.arch), - self.ctx.python_recipe.major_minor_version_string) - if python_name == 'python3': - env['LDFLAGS'] += 'm' - else: - ndk_dir_python = join(self.ctx.ndk_dir, 'sources', - 'python', python_version) - env['CFLAGS'] += ' -I{} '.format( - join(ndk_dir_python, 'include', - 'python')) - env['LDFLAGS'] += ' -L{}'.format( - join(ndk_dir_python, 'libs', arch.arch)) - env['LDFLAGS'] += ' -lpython{}m'.format(python_short_version) + env['CFLAGS'] += ' -I{}'.format( + self.ctx.python_recipe.include_root(arch.arch) + ) + env['LDFLAGS'] += ' -L{} -lpython{}'.format( + self.ctx.python_recipe.link_root(arch.arch), + self.ctx.python_recipe.major_minor_version_string, + ) + if python_name == 'python3': + env['LDFLAGS'] += 'm' hppath = [] hppath.append(join(dirname(self.hostpython_location), 'Lib')) @@ -954,7 +941,7 @@ class CythonRecipe(PythonRecipe): def __init__(self, *args, **kwargs): super(CythonRecipe, self).__init__(*args, **kwargs) depends = self.depends - depends.append(('python2', 'python3', 'python3crystax')) + depends.append(('python2', 'python3')) depends = list(set(depends)) self.depends = depends @@ -1021,8 +1008,7 @@ def cythonize_file(self, env, build_dir, filename): del cyenv['PYTHONPATH'] if 'PYTHONNOUSERSITE' in cyenv: cyenv.pop('PYTHONNOUSERSITE') - cython = 'cython' if self.ctx.python_recipe.from_crystax else self.ctx.cython - cython_command = sh.Command(cython) + cython_command = sh.Command(self.ctx.cython) shprint(cython_command, filename, *self.cython_args, _env=cyenv) def cythonize_build(self, env, build_dir="."): @@ -1041,9 +1027,6 @@ def get_recipe_env(self, arch, with_flags_in_cc=True): ' -L{} '.format(self.ctx.libs_dir) + ' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'obj', 'local', arch.arch))) - if self.ctx.python_recipe.from_crystax: - env['LDFLAGS'] = (env['LDFLAGS'] + - ' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'libs', arch.arch))) env['LDSHARED'] = env['CC'] + ' -shared' # shprint(sh.whereis, env['LDSHARED'], _env=env) @@ -1059,24 +1042,6 @@ def get_recipe_env(self, arch, with_flags_in_cc=True): env['LIBLINK_PATH'] = liblink_path ensure_dir(liblink_path) - # Add crystax-specific site packages: - if self.ctx.python_recipe.from_crystax: - command = sh.Command('python{}'.format(self.ctx.python_recipe.version)) - site_packages_dirs = command( - '-c', 'import site; print("\\n".join(site.getsitepackages()))') - site_packages_dirs = site_packages_dirs.stdout.decode('utf-8').split('\n') - if 'PYTHONPATH' in env: - env['PYTHONPATH'] = env['PYTHONPATH'] +\ - ':{}'.format(':'.join(site_packages_dirs)) - else: - env['PYTHONPATH'] = ':'.join(site_packages_dirs) - while env['PYTHONPATH'].find("::") > 0: - env['PYTHONPATH'] = env['PYTHONPATH'].replace("::", ":") - if env['PYTHONPATH'].endswith(":"): - env['PYTHONPATH'] = env['PYTHONPATH'][:-1] - if env['PYTHONPATH'].startswith(":"): - env['PYTHONPATH'] = env['PYTHONPATH'][1:] - return env @@ -1084,20 +1049,12 @@ class TargetPythonRecipe(Recipe): '''Class for target python recipes. Sets ctx.python_recipe to point to itself, so as to know later what kind of Python was built or used.''' - from_crystax = False - '''True if the python is used from CrystaX, False otherwise (i.e. if - it is built by p4a).''' - def __init__(self, *args, **kwargs): self._ctx = None super(TargetPythonRecipe, self).__init__(*args, **kwargs) def prebuild_arch(self, arch): super(TargetPythonRecipe, self).prebuild_arch(arch) - if self.from_crystax and self.ctx.ndk != 'crystax': - raise BuildInterruptingException( - 'The {} recipe can only be built when ' - 'using the CrystaX NDK. Exiting.'.format(self.name)) self.ctx.python_recipe = self def include_root(self, arch): diff --git a/pythonforandroid/recipes/cymunk/__init__.py b/pythonforandroid/recipes/cymunk/__init__.py index 96d4169710..272c18f9e6 100644 --- a/pythonforandroid/recipes/cymunk/__init__.py +++ b/pythonforandroid/recipes/cymunk/__init__.py @@ -6,7 +6,5 @@ class CymunkRecipe(CythonRecipe): url = 'https://github.com/tito/cymunk/archive/{version}.zip' name = 'cymunk' - depends = [('python2', 'python3crystax', 'python3')] - recipe = CymunkRecipe() diff --git a/pythonforandroid/recipes/flask/__init__.py b/pythonforandroid/recipes/flask/__init__.py index 1a9b685256..05d59eebdf 100644 --- a/pythonforandroid/recipes/flask/__init__.py +++ b/pythonforandroid/recipes/flask/__init__.py @@ -9,7 +9,7 @@ class FlaskRecipe(PythonRecipe): version = '0.10.1' url = 'https://github.com/pallets/flask/archive/{version}.zip' - depends = [('python2', 'python3', 'python3crystax'), 'setuptools'] + depends = ['setuptools'] python_depends = ['jinja2', 'werkzeug', 'markupsafe', 'itsdangerous', 'click'] diff --git a/pythonforandroid/recipes/genericndkbuild/__init__.py b/pythonforandroid/recipes/genericndkbuild/__init__.py index d91f946c88..e6cccb6e8d 100644 --- a/pythonforandroid/recipes/genericndkbuild/__init__.py +++ b/pythonforandroid/recipes/genericndkbuild/__init__.py @@ -7,7 +7,7 @@ class GenericNDKBuildRecipe(BootstrapNDKRecipe): version = None url = None - depends = [('python2', 'python3', 'python3crystax')] + depends = [('python2', 'python3')] conflicts = ['sdl2'] def should_build(self, arch): diff --git a/pythonforandroid/recipes/hostpython2/__init__.py b/pythonforandroid/recipes/hostpython2/__init__.py index 08d45ba564..36eb178d8a 100644 --- a/pythonforandroid/recipes/hostpython2/__init__.py +++ b/pythonforandroid/recipes/hostpython2/__init__.py @@ -12,7 +12,7 @@ class Hostpython2Recipe(HostPythonRecipe): ''' version = '2.7.15' name = 'hostpython2' - conflicts = ['hostpython3', 'hostpython3crystax'] + conflicts = ['hostpython3'] recipe = Hostpython2Recipe() diff --git a/pythonforandroid/recipes/hostpython3/__init__.py b/pythonforandroid/recipes/hostpython3/__init__.py index 8b268bdd4f..a23f0b9fa2 100644 --- a/pythonforandroid/recipes/hostpython3/__init__.py +++ b/pythonforandroid/recipes/hostpython3/__init__.py @@ -11,7 +11,7 @@ class Hostpython3Recipe(HostPythonRecipe): ''' version = '3.7.1' name = 'hostpython3' - conflicts = ['hostpython2', 'hostpython3crystax'] + conflicts = ['hostpython2'] recipe = Hostpython3Recipe() diff --git a/pythonforandroid/recipes/hostpython3crystax/__init__.py b/pythonforandroid/recipes/hostpython3crystax/__init__.py deleted file mode 100644 index 88cee35938..0000000000 --- a/pythonforandroid/recipes/hostpython3crystax/__init__.py +++ /dev/null @@ -1,44 +0,0 @@ -from pythonforandroid.toolchain import Recipe, shprint -from os.path import join -import sh - - -class Hostpython3CrystaXRecipe(Recipe): - version = 'auto' # the version is taken from the python3crystax recipe - name = 'hostpython3crystax' - - conflicts = ['hostpython2'] - - def get_build_container_dir(self, arch=None): - choices = self.check_recipe_choices() - dir_name = '-'.join([self.name] + choices) - return join(self.ctx.build_dir, 'other_builds', dir_name, 'desktop') - - # def prebuild_armeabi(self): - # # Override hostpython Setup? - # shprint(sh.cp, join(self.get_recipe_dir(), 'Setup'), - # join(self.get_build_dir('armeabi'), 'Modules', 'Setup')) - - def get_build_dir(self, arch=None): - return join(self.get_build_container_dir(), self.name) - - def build_arch(self, arch): - """ - Creates expected build and symlinks system Python version. - """ - self.ctx.hostpython = '/usr/bin/false' - # creates the sub buildir (used by other recipes) - # https://github.com/kivy/python-for-android/issues/1154 - sub_build_dir = join(self.get_build_dir(), 'build') - shprint(sh.mkdir, '-p', sub_build_dir) - python3crystax = self.get_recipe('python3crystax', self.ctx) - system_python = sh.which("python" + python3crystax.version) - if system_python is None: - raise OSError( - ('Trying to use python3crystax=={} but this Python version ' - 'is not installed locally.').format(python3crystax.version)) - link_dest = join(self.get_build_dir(), 'hostpython') - shprint(sh.ln, '-sf', system_python, link_dest) - - -recipe = Hostpython3CrystaXRecipe() diff --git a/pythonforandroid/recipes/jedi/__init__.py b/pythonforandroid/recipes/jedi/__init__.py index 6338a52f24..17168e85a3 100644 --- a/pythonforandroid/recipes/jedi/__init__.py +++ b/pythonforandroid/recipes/jedi/__init__.py @@ -5,8 +5,6 @@ class JediRecipe(PythonRecipe): version = 'v0.9.0' url = 'https://github.com/davidhalter/jedi/archive/{version}.tar.gz' - depends = [('python2', 'python3crystax', 'python3')] - patches = ['fix_MergedNamesDict_get.patch'] # This apparently should be fixed in jedi 0.10 (not released to # pypi yet), but it still occurs on Android, I could not reproduce diff --git a/pythonforandroid/recipes/numpy/__init__.py b/pythonforandroid/recipes/numpy/__init__.py index 97df43524d..4e47e9d890 100644 --- a/pythonforandroid/recipes/numpy/__init__.py +++ b/pythonforandroid/recipes/numpy/__init__.py @@ -8,7 +8,6 @@ class NumpyRecipe(CompiledComponentsPythonRecipe): version = '1.16.4' url = 'https://pypi.python.org/packages/source/n/numpy/numpy-{version}.zip' site_packages_name = 'numpy' - depends = [('python2', 'python3', 'python3crystax')] patches = [ join('patches', 'add_libm_explicitly_to_build.patch'), @@ -42,10 +41,6 @@ def get_recipe_env(self, arch): py_ver = self.ctx.python_recipe.major_minor_version_string py_inc_dir = self.ctx.python_recipe.include_root(arch.arch) py_lib_dir = self.ctx.python_recipe.link_root(arch.arch) - if self.ctx.ndk == 'crystax': - src_dir = join(self.ctx.ndk_dir, 'sources') - flags += " -I{}".format(join(src_dir, 'crystax', 'include')) - flags += " -L{}".format(join(src_dir, 'crystax', 'libs', arch.arch)) flags += ' -I{}'.format(py_inc_dir) flags += ' -L{} -lpython{}'.format(py_lib_dir, py_ver) if 'python3' in self.ctx.python_recipe.name: diff --git a/pythonforandroid/recipes/openal/__init__.py b/pythonforandroid/recipes/openal/__init__.py index ad93065f4d..cfb62f6148 100644 --- a/pythonforandroid/recipes/openal/__init__.py +++ b/pythonforandroid/recipes/openal/__init__.py @@ -24,9 +24,6 @@ def build_arch(self, arch): '-DCMAKE_TOOLCHAIN_FILE={}'.format('XCompile-Android.txt'), '-DHOST={}'.format(self.ctx.toolchain_prefix) ] - if self.ctx.ndk == 'crystax': - # avoids a segfault in libcrystax when calling lrintf - cmake_args += ['-DHAVE_LRINTF=0'] shprint( sh.cmake, '.', *cmake_args, diff --git a/pythonforandroid/recipes/openssl/__init__.py b/pythonforandroid/recipes/openssl/__init__.py index 38dbaeeb42..d3033a3594 100644 --- a/pythonforandroid/recipes/openssl/__init__.py +++ b/pythonforandroid/recipes/openssl/__init__.py @@ -20,11 +20,6 @@ class OpenSSLRecipe(Recipe): using the methods: :meth:`include_flags`, :meth:`link_dirs_flags` and :meth:`link_libs_flags`. - .. note:: the python2legacy version is too old to support openssl 1.1+, so - we must use version 1.0.x. Also python3crystax is not building - successfully with openssl libs 1.1+ so we use the legacy version as - we do with python2legacy. - .. warning:: This recipe is very sensitive because is used for our core recipes, the python recipes. The used API should match with the one used in our python build, otherwise we will be unable to build the diff --git a/pythonforandroid/recipes/pysha3/__init__.py b/pythonforandroid/recipes/pysha3/__init__.py index 35cfff84a8..c171c3f662 100644 --- a/pythonforandroid/recipes/pysha3/__init__.py +++ b/pythonforandroid/recipes/pysha3/__init__.py @@ -13,16 +13,11 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = super(Pysha3Recipe, self).get_recipe_env(arch, with_flags_in_cc) # CFLAGS may only be used to specify C compiler flags, for macro definitions use CPPFLAGS env['CPPFLAGS'] = env['CFLAGS'] - if self.ctx.ndk == 'crystax': - env['CPPFLAGS'] += ' -I{}/sources/python/{}/include/python/'.format( - self.ctx.ndk_dir, self.ctx.python_recipe.version[0:3]) env['CFLAGS'] = '' # LDFLAGS may only be used to specify linker flags, for libraries use LIBS - env['LDFLAGS'] = env['LDFLAGS'].replace('-lm', '').replace('-lcrystax', '') + env['LDFLAGS'] = env['LDFLAGS'].replace('-lm', '') env['LDFLAGS'] += ' -L{}'.format(os.path.join(self.ctx.bootstrap.build_dir, 'libs', arch.arch)) env['LIBS'] = ' -lm' - if self.ctx.ndk == 'crystax': - env['LIBS'] += ' -lcrystax -lpython{}m'.format(self.ctx.python_recipe.version[0:3]) env['LDSHARED'] += env['LIBS'] return env diff --git a/pythonforandroid/recipes/python2/__init__.py b/pythonforandroid/recipes/python2/__init__.py index ba1fa9a671..78e666fa2a 100644 --- a/pythonforandroid/recipes/python2/__init__.py +++ b/pythonforandroid/recipes/python2/__init__.py @@ -21,7 +21,7 @@ class Python2Recipe(GuestPythonRecipe): name = 'python2' depends = ['hostpython2'] - conflicts = ['python3crystax', 'python3'] + conflicts = ['python3'] patches = [ # new 2.7.15 patches diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index f22dce8e91..963fad635f 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -28,7 +28,7 @@ class Python3Recipe(GuestPythonRecipe): patches = patches + ["patches/remove-fix-cortex-a8.patch"] depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi'] - conflicts = ['python3crystax', 'python2'] + conflicts = ['python2'] configure_args = ( '--host={android_host}', diff --git a/pythonforandroid/recipes/python3crystax/__init__.py b/pythonforandroid/recipes/python3crystax/__init__.py deleted file mode 100644 index 805be0dd12..0000000000 --- a/pythonforandroid/recipes/python3crystax/__init__.py +++ /dev/null @@ -1,102 +0,0 @@ - -from pythonforandroid.recipe import TargetPythonRecipe -from pythonforandroid.toolchain import shprint -from pythonforandroid.logger import info, error -from pythonforandroid.util import ensure_dir, temp_directory -from os.path import exists, join -import sh - -prebuilt_download_locations = { - '3.6': ('https://github.com/inclement/crystax_python_builds/' - 'releases/download/0.1/crystax_python_3.6_armeabi_armeabi-v7a.tar.gz')} - - -class Python3CrystaXRecipe(TargetPythonRecipe): - version = '3.6' - url = '' - name = 'python3crystax' - - depends = ['hostpython3crystax'] - conflicts = ['python3', 'python2'] - - from_crystax = True - - def get_dir_name(self): - name = super(Python3CrystaXRecipe, self).get_dir_name() - name += '-version{}'.format(self.version) - return name - - def build_arch(self, arch): - # We don't have to actually build anything as CrystaX comes - # with the necessary modules. They are included by modifying - # the Android.mk in the jni folder. - - # If the Python version to be used is not prebuilt with the CrystaX - # NDK, we do have to download it. - - crystax_python_dir = join(self.ctx.ndk_dir, 'sources', 'python') - if not exists(join(crystax_python_dir, self.version)): - info(('The NDK does not have a prebuilt Python {}, trying ' - 'to obtain one.').format(self.version)) - - if self.version not in prebuilt_download_locations: - error(('No prebuilt version for Python {} could be found, ' - 'the built cannot continue.')) - exit(1) - - with temp_directory() as td: - self.download_file(prebuilt_download_locations[self.version], - join(td, 'downloaded_python')) - shprint(sh.tar, 'xf', join(td, 'downloaded_python'), - '--directory', crystax_python_dir) - - if not exists(join(crystax_python_dir, self.version)): - error(('Something went wrong, the directory at {} should ' - 'have been created but does not exist.').format( - join(crystax_python_dir, self.version))) - - if not exists(join( - crystax_python_dir, self.version, 'libs', arch.arch)): - error(('The prebuilt Python for version {} does not contain ' - 'binaries for your chosen architecture "{}".').format( - self.version, arch.arch)) - exit(1) - - # TODO: We should have an option to build a new Python. This - # would also allow linking to openssl and sqlite from CrystaX. - - dirn = self.ctx.get_python_install_dir() - ensure_dir(dirn) - - # Instead of using a locally built hostpython, we use the - # user's Python for now. They must have the right version - # available. Using e.g. pyenv makes this easy. - self.ctx.hostpython = 'python{}'.format(self.version) - - def create_python_bundle(self, dirn, arch): - ndk_dir = self.ctx.ndk_dir - py_recipe = self.ctx.python_recipe - python_dir = join(ndk_dir, 'sources', 'python', - py_recipe.version, 'libs', arch.arch) - shprint(sh.cp, '-r', join(python_dir, - 'stdlib.zip'), dirn) - shprint(sh.cp, '-r', join(python_dir, - 'modules'), dirn) - shprint(sh.cp, '-r', self.ctx.get_python_install_dir(), - join(dirn, 'site-packages')) - - info('Renaming .so files to reflect cross-compile') - self.reduce_object_file_names(join(dirn, "site-packages")) - - return join(dirn, 'site-packages') - - def include_root(self, arch_name): - return join(self.ctx.ndk_dir, 'sources', 'python', self.major_minor_version_string, - 'include', 'python') - - def link_root(self, arch_name): - return join(self.ctx.ndk_dir, 'sources', 'python', self.major_minor_version_string, - 'libs', arch_name) - - -recipe = Python3CrystaXRecipe() diff --git a/pythonforandroid/recipes/secp256k1/__init__.py b/pythonforandroid/recipes/secp256k1/__init__.py index 889803100f..338c7b90c3 100644 --- a/pythonforandroid/recipes/secp256k1/__init__.py +++ b/pythonforandroid/recipes/secp256k1/__init__.py @@ -10,9 +10,14 @@ class Secp256k1Recipe(CppCompiledComponentsPythonRecipe): call_hostpython_via_targetpython = False depends = [ - 'openssl', ('hostpython3', 'hostpython2', 'hostpython3crystax'), - ('python2', 'python3', 'python3crystax'), 'setuptools', - 'libffi', 'cffi', 'libsecp256k1'] + 'openssl', + ('hostpython3', 'hostpython2'), + ('python2', 'python3'), + 'setuptools', + 'libffi', + 'cffi', + 'libsecp256k1' + ] patches = [ "cross_compile.patch", "drop_setup_requires.patch", diff --git a/pythonforandroid/recipes/setuptools/__init__.py b/pythonforandroid/recipes/setuptools/__init__.py index 512d61a843..8e248bd874 100644 --- a/pythonforandroid/recipes/setuptools/__init__.py +++ b/pythonforandroid/recipes/setuptools/__init__.py @@ -6,7 +6,6 @@ class SetuptoolsRecipe(PythonRecipe): url = 'https://pypi.python.org/packages/source/s/setuptools/setuptools-{version}.zip' call_hostpython_via_targetpython = False install_in_hostpython = True - depends = [('python2', 'python3', 'python3crystax')] recipe = SetuptoolsRecipe() diff --git a/pythonforandroid/recipes/six/__init__.py b/pythonforandroid/recipes/six/__init__.py index 2e00432280..064b8e2871 100644 --- a/pythonforandroid/recipes/six/__init__.py +++ b/pythonforandroid/recipes/six/__init__.py @@ -5,7 +5,6 @@ class SixRecipe(PythonRecipe): version = '1.10.0' url = 'https://pypi.python.org/packages/source/s/six/six-{version}.tar.gz' - depends = [('python2', 'python3', 'python3crystax')] recipe = SixRecipe() diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index b61c9ac54a..6cfb300763 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -982,7 +982,7 @@ def apk(self, args): gradlew = sh.Command('./gradlew') if exists('/usr/bin/dos2unix'): # .../dists/bdisttest_python3/gradlew - # .../build/bootstrap_builds/sdl2-python3crystax/gradlew + # .../build/bootstrap_builds/sdl2-python3/gradlew # if docker on windows, gradle contains CRLF output = shprint( sh.Command('dos2unix'), gradlew._path.decode('utf8'), diff --git a/testapps/setup_keyboard.py b/testapps/setup_keyboard.py index 026847764d..26499a639a 100644 --- a/testapps/setup_keyboard.py +++ b/testapps/setup_keyboard.py @@ -7,7 +7,7 @@ 'blacklist-requirements': 'openssl,sqlite3', 'android-api': 27, 'ndk-api': 21, - 'ndk-dir': '/home/asandy/android/crystax-ndk-10.3.2', + 'ndk-dir': '/home/asandy/android/android-ndk-r17c', 'dist-name': 'bdisttest', 'ndk-version': '10.3.2', 'permission': 'VIBRATE', diff --git a/testapps/setup_testapp_flask.py b/testapps/setup_testapp_flask.py index 3302e8595c..3b2536e579 100644 --- a/testapps/setup_testapp_flask.py +++ b/testapps/setup_testapp_flask.py @@ -7,7 +7,7 @@ 'blacklist-requirements': 'openssl,sqlite3', 'android-api': 27, 'ndk-api': 21, - 'ndk-dir': '/home/asandy/android/crystax-ndk-10.3.2', + 'ndk-dir': '/home/asandy/android/android-ndk-r17c', 'dist-name': 'testapp_flask', 'ndk-version': '10.3.2', 'bootstrap': 'webview', diff --git a/testapps/setup_testapp_python2.py b/testapps/setup_testapp_python2.py index 5aed64a44a..9499c80c73 100644 --- a/testapps/setup_testapp_python2.py +++ b/testapps/setup_testapp_python2.py @@ -5,7 +5,7 @@ options = {'apk': {'requirements': 'sdl2,numpy,pyjnius,kivy,python2', 'android-api': 27, 'ndk-api': 21, - 'ndk-dir': '/home/asandy/android/crystax-ndk-10.3.2', + 'ndk-dir': '/home/asandy/android/android-ndk-r17c', 'dist-name': 'bdisttest_python2', 'ndk-version': '10.3.2', 'permission': 'VIBRATE', diff --git a/testapps/setup_testapp_python2_sqlite_openssl.py b/testapps/setup_testapp_python2_sqlite_openssl.py index 18ce7c4fcd..c1dcf53efc 100644 --- a/testapps/setup_testapp_python2_sqlite_openssl.py +++ b/testapps/setup_testapp_python2_sqlite_openssl.py @@ -5,7 +5,7 @@ options = {'apk': {'requirements': 'sdl2,pyjnius,kivy,python2,openssl,requests,peewee,sqlite3', 'android-api': 27, 'ndk-api': 21, - 'ndk-dir': '/home/sandy/android/crystax-ndk-10.3.2', + 'ndk-dir': '/home/sandy/android/android-ndk-r17c', 'dist-name': 'bdisttest_python2_sqlite_openssl', 'ndk-version': '10.3.2', 'permissions': ['INTERNET', 'VIBRATE'], diff --git a/testapps/setup_testapp_python3crystax.py b/testapps/setup_testapp_python3crystax.py deleted file mode 100644 index 08ed0afa09..0000000000 --- a/testapps/setup_testapp_python3crystax.py +++ /dev/null @@ -1,30 +0,0 @@ - -from distutils.core import setup -from setuptools import find_packages - -options = {'apk': {'requirements': 'sdl2,pyjnius,kivy,python3crystax', - 'android-api': 19, - 'ndk-api': 19, - 'ndk-dir': '/home/asandy/android/crystax-ndk-10.3.2', - 'dist-name': 'bdisttest_python3', - 'ndk-version': '10.3.2', - 'permission': 'VIBRATE', - }} - -package_data = {'': ['*.py', - '*.png'] - } - -packages = find_packages() -print('packages are', packages) - -setup( - name='testapp_python3', - version='1.1', - description='p4a setup.py test', - author='Alexander Taylor', - author_email='alexanderjohntaylor@gmail.com', - packages=find_packages(), - options=options, - package_data={'testapp': ['*.py', '*.png']} -) diff --git a/testapps/setup_vispy.py b/testapps/setup_vispy.py index a0863d0a1c..49ad47fda3 100644 --- a/testapps/setup_vispy.py +++ b/testapps/setup_vispy.py @@ -7,7 +7,7 @@ 'blacklist-requirements': 'openssl,sqlite3', 'android-api': 27, 'ndk-api': 21, - 'ndk-dir': '/home/asandy/android/crystax-ndk-10.3.2', + 'ndk-dir': '/home/asandy/android/android-ndk-r17c', 'dist-name': 'bdisttest', 'ndk-version': '10.3.2', 'permission': 'VIBRATE', diff --git a/tests/test_graph.py b/tests/test_graph.py index 0534d58290..ccade98561 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -19,14 +19,14 @@ Bootstrap.get_bootstrap('sdl2', ctx)] valid_combinations = list(product(name_sets, bootstraps)) valid_combinations.extend( - [(['python3crystax'], Bootstrap.get_bootstrap('sdl2', ctx)), - (['kivy', 'python3crystax'], Bootstrap.get_bootstrap('sdl2', ctx)), + [(['python3'], Bootstrap.get_bootstrap('sdl2', ctx)), + (['kivy', 'python3'], Bootstrap.get_bootstrap('sdl2', ctx)), (['flask'], Bootstrap.get_bootstrap('webview', ctx)), (['pysdl2'], None), # auto-detect bootstrap! important corner case ] ) invalid_combinations = [ - [['python2', 'python3crystax'], None], + [['python2', 'python3'], None], [['pysdl2', 'genericndkbuild'], None], ] invalid_combinations_simple = list(invalid_combinations) @@ -39,12 +39,11 @@ # non-tuple/non-ambiguous dependencies, e.g.: # # dependencies_1st = ["python2", "pillow"] -# dependencies_2nd = ["python3crystax", "pillow"] +# dependencies_2nd = ["python3", "pillow"] # # This however won't work: # # dependencies_1st = [("python2", "python3"), "pillow"] -# dependencies_2nd = [("python2legacy", "python3crystax"), "pillow"] # # (This is simply because the conflict checker doesn't resolve this to # keep the code ismple enough) From 21f93d2acae619eaa9d69bb2ed1afe0633cdfe05 Mon Sep 17 00:00:00 2001 From: opacam Date: Wed, 10 Jul 2019 12:21:47 +0200 Subject: [PATCH 09/41] [crystax] Text corrections for doc files - Fix misspelled word - update docker image size to the current situation Thanks @AndreMiras!! --- doc/source/buildoptions.rst | 2 +- doc/source/docker.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/buildoptions.rst b/doc/source/buildoptions.rst index ddb81be092..a3ab67a8af 100644 --- a/doc/source/buildoptions.rst +++ b/doc/source/buildoptions.rst @@ -43,7 +43,7 @@ recipe. .. note:: Since we don't support `python3crystax` anymore, the old instructions has been removed from here. If you, still have the need to make use of this old recipe, you should do it with an old `python-for-android` - release. Probably, a got starting point would be `version 0.7.0 + release. Probably, a good starting point would be `version 0.7.0 `__ or if that doesn't work , then go for `version 0.6.0 `__ diff --git a/doc/source/docker.rst b/doc/source/docker.rst index 28db26d49c..623e0e6883 100644 --- a/doc/source/docker.rst +++ b/doc/source/docker.rst @@ -21,7 +21,7 @@ already have Docker preinstalled and set up. Within the custom diff made by building the distribution there is another big chunk of space eaten. The very basic stuff such as a distribution with: CPython 3, setuptools, Python for Android ``android`` module, SDL2 (+ deps), - PyJNIus and Kivy takes almost 13 GB. Check your free space first! + PyJNIus and Kivy takes almost 2 GB. Check your free space first! 1. Clone the repository:: From aad27302fc2b019b17e57f12a0d580be58f42196 Mon Sep 17 00:00:00 2001 From: opacam Date: Thu, 11 Jul 2019 12:36:41 +0200 Subject: [PATCH 10/41] [crystax] Fix doc string typo for `PythonRecipe` --- pythonforandroid/recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 6429c6ea97..a4a1c859d4 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -725,7 +725,7 @@ class PythonRecipe(Recipe): This is almost always what you want to do.''' setup_extra_args = [] - '''List of extra arugments to pass to setup.py''' + '''List of extra arguments to pass to setup.py''' def __init__(self, *args, **kwargs): super(PythonRecipe, self).__init__(*args, **kwargs) From dc589c43ef5aaff6d36cc77c23c283c2b8bae9ea Mon Sep 17 00:00:00 2001 From: opacam Date: Thu, 11 Jul 2019 11:19:53 +0200 Subject: [PATCH 11/41] [crystax] Fix the dependencies resolution --- pythonforandroid/recipe.py | 45 ++++++++++++++++++------ pythonforandroid/recipes/six/__init__.py | 1 + tests/test_bootstrap.py | 3 +- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index a4a1c859d4..d294dcba9a 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -727,12 +727,42 @@ class PythonRecipe(Recipe): setup_extra_args = [] '''List of extra arguments to pass to setup.py''' + depends = [('python2', 'python3')] + ''' + .. note:: it's important to keep this depends as a class attribute, outside + `__init__` because, sometimes, we only initialize the object, so + the `__init__` call it won't be called, which will lead to not + have the python versions as a dependencies and it will cause a + tremendous `test_graph` error (difficult to track) and also, the + build order for dependencies will not be computed as expected (if + computed...). So be very careful with this line!! + + .. warning:: this `depends` may be overwrote in inherited classes of + `PythonRecipe`, so we make sure that any sub class will + contain python as a dependency. We do this by checking the + dependencies in meth:`PythonRecipe.__init__` method and adding + them again in case that is necessary, so don't forget to call + `super` in any inherited class of this class. + ''' + def __init__(self, *args, **kwargs): super(PythonRecipe, self).__init__(*args, **kwargs) - depends = self.depends - depends.append(('python2', 'python3')) - depends = list(set(depends)) - self.depends = depends + if not any( + [ + d + for d in {'python2', 'python3', ('python2', 'python3')} + if d in self.depends + ] + ): + # we overwrote `depends` in inherited recipe, so we must add it + # again the python versions as dependencies, but we only do this in + # case that the sub classes recipe does not contain any python + # version as dependency because it may be some recipes only + # compatible with a single version of python + depends = self.depends + depends.append(('python2', 'python3')) + depends = list(set(depends)) + self.depends = depends def clean_build(self, arch=None): super(PythonRecipe, self).clean_build(arch=arch) @@ -938,13 +968,6 @@ class CythonRecipe(PythonRecipe): cython_args = [] call_hostpython_via_targetpython = False - def __init__(self, *args, **kwargs): - super(CythonRecipe, self).__init__(*args, **kwargs) - depends = self.depends - depends.append(('python2', 'python3')) - depends = list(set(depends)) - self.depends = depends - def build_arch(self, arch): '''Build any cython components, then install the Python module by calling setup.py install with the target Python dir. diff --git a/pythonforandroid/recipes/six/__init__.py b/pythonforandroid/recipes/six/__init__.py index 064b8e2871..ca68e189f8 100644 --- a/pythonforandroid/recipes/six/__init__.py +++ b/pythonforandroid/recipes/six/__init__.py @@ -5,6 +5,7 @@ class SixRecipe(PythonRecipe): version = '1.10.0' url = 'https://pypi.python.org/packages/source/s/six/six-{version}.tar.gz' + depends = ['setuptools'] recipe = SixRecipe() diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index ad62a45b15..6f66925818 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -178,7 +178,8 @@ def test_expand_dependencies_with_pure_python_package(self): expanded_result = expand_dependencies( ["python3", "kivy", "peewee"], self.ctx ) - self.assertEqual(len(expanded_result), 3) + # we expect to have two results (one for python2 and one for python3) + self.assertEqual(len(expanded_result), 2) self.assertIsInstance(expanded_result, list) for i in expanded_result: self.assertIsInstance(i, list) From 7c1223e6a5313cf783c09eb1bc9fc037649b8636 Mon Sep 17 00:00:00 2001 From: opacam Date: Thu, 11 Jul 2019 14:23:05 +0200 Subject: [PATCH 12/41] [crystax] Add hostpython recipes to CORE_RECIPES Because we already build them whenever we build our python recipes, so this way we will avoid to make it fail the `CI` test `rebuild_updated_recipes` in the case that we have to touch both hostpython recipes at the same time --- ci/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/constants.py b/ci/constants.py index 56ab5bff62..c5ab61099d 100644 --- a/ci/constants.py +++ b/ci/constants.py @@ -86,5 +86,5 @@ class TargetPython(Enum): # recipes that were already built will be skipped CORE_RECIPES = set([ 'pyjnius', 'kivy', 'openssl', 'requests', 'sqlite3', 'setuptools', - 'numpy', 'android', 'python2', 'python3', + 'numpy', 'android', 'hostpython2', 'hostpython3', 'python2', 'python3', ]) From ac8df83e48a072a6fff88218a19bfd05d9136f9b Mon Sep 17 00:00:00 2001 From: opacam Date: Thu, 11 Jul 2019 14:33:52 +0200 Subject: [PATCH 13/41] [crystax] Detect removed recipes for rebuild_updated_recipes --- ci/rebuild_updated_recipes.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ci/rebuild_updated_recipes.py b/ci/rebuild_updated_recipes.py index 32b0fda0f6..f17df3fe62 100755 --- a/ci/rebuild_updated_recipes.py +++ b/ci/rebuild_updated_recipes.py @@ -28,6 +28,7 @@ from pythonforandroid.graph import get_recipe_order_and_bootstrap from pythonforandroid.toolchain import current_directory from pythonforandroid.util import BuildInterruptingException +from pythonforandroid.recipe import Recipe from ci.constants import TargetPython, CORE_RECIPES, BROKEN_RECIPES @@ -78,6 +79,18 @@ def main(): recipes -= CORE_RECIPES logger.info('recipes to build: {}'.format(recipes)) context = Context() + + # removing the deleted recipes for the given target (if any) + for recipe_name in recipes.copy(): + try: + Recipe.get_recipe(recipe_name, context) + except ValueError: + # recipe doesn't exist, so probably we remove it + recipes.remove(recipe_name) + logger.warning( + 'removed {} from recipes because deleted'.format(recipe_name) + ) + # forces the default target recipes_and_target = recipes | set([target_python.name]) try: From 185936e7be0cb84401b5909854fe0068ba42a8f2 Mon Sep 17 00:00:00 2001 From: opacam Date: Thu, 11 Jul 2019 18:04:24 +0200 Subject: [PATCH 14/41] [crystax] Remove hardcoded sdl2 bootstrap in rebuild_updated_recipes Since now we have an smartest way to determine the right bootstrap based on recipes, it's time to rely on it for our `rebuild_updated_recipes` Closes: #1594 --- ci/rebuild_updated_recipes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/rebuild_updated_recipes.py b/ci/rebuild_updated_recipes.py index f17df3fe62..54f62ac768 100755 --- a/ci/rebuild_updated_recipes.py +++ b/ci/rebuild_updated_recipes.py @@ -67,7 +67,7 @@ def build(target_python, requirements): # iterates to stream the output for line in sh.python( testapp, 'apk', '--sdk-dir', android_sdk_home, - '--ndk-dir', android_ndk_home, '--bootstrap', 'sdl2', '--requirements', + '--ndk-dir', android_ndk_home, '--requirements', requirements, _err_to_out=True, _iter=True): print(line) From 5ad8ad733f7ad97b7dadb1894ecb2454128f9db8 Mon Sep 17 00:00:00 2001 From: opacam Date: Sun, 14 Jul 2019 21:31:56 +0200 Subject: [PATCH 15/41] [crystax] Improve docs and inline comments By shortening a little some long texts, thanks @JonasT and @inclement!! --- doc/source/buildoptions.rst | 18 ++++++------------ pythonforandroid/recipe.py | 31 ++++++++++++------------------- 2 files changed, 18 insertions(+), 31 deletions(-) diff --git a/doc/source/buildoptions.rst b/doc/source/buildoptions.rst index a3ab67a8af..5d80020728 100644 --- a/doc/source/buildoptions.rst +++ b/doc/source/buildoptions.rst @@ -35,18 +35,12 @@ e.g. ``--requirements=python3``. CrystaX python3 ~~~~~~~~~~~~~~~ -python-for-android originally supported Python 3 using the CrystaX NDK. Since -we have a working python3 recipe, we don't support CrystaX NDK anymore. If you -were using `python3crystax`, we recommend to give it a try to the new `python3` -recipe. - -.. note:: Since we don't support `python3crystax` anymore, the old instructions - has been removed from here. If you, still have the need to make use - of this old recipe, you should do it with an old `python-for-android` - release. Probably, a good starting point would be `version 0.7.0 - `__ or - if that doesn't work , then go for `version 0.6.0 - `__ +python-for-android no longer supports building for Python 3 using the CrystaX +NDK. Instead, use the python3 recipe, which can be built using the normal +Google NDK. + +.. note:: The last python-for-android version supporting CrystaX was `0.7.0. + `__ .. _bootstrap_build_options: diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index d294dcba9a..4de888306a 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -729,20 +729,14 @@ class PythonRecipe(Recipe): depends = [('python2', 'python3')] ''' - .. note:: it's important to keep this depends as a class attribute, outside - `__init__` because, sometimes, we only initialize the object, so - the `__init__` call it won't be called, which will lead to not - have the python versions as a dependencies and it will cause a - tremendous `test_graph` error (difficult to track) and also, the - build order for dependencies will not be computed as expected (if - computed...). So be very careful with this line!! - - .. warning:: this `depends` may be overwrote in inherited classes of - `PythonRecipe`, so we make sure that any sub class will - contain python as a dependency. We do this by checking the - dependencies in meth:`PythonRecipe.__init__` method and adding - them again in case that is necessary, so don't forget to call - `super` in any inherited class of this class. + .. note:: it's important to keep this depends as a class attribute outside + `__init__` because sometimes we only initialize the class, so the + `__init__` call won't be called and the deps would be missing + (which breaks the dependency graph computation) + + .. warning:: don't forget to call `super().__init__()` in any recipe's + `__init__`, or otherwise it may not be ensured that it depends + on python2 or python3 which can break the dependency graph ''' def __init__(self, *args, **kwargs): @@ -754,11 +748,10 @@ def __init__(self, *args, **kwargs): if d in self.depends ] ): - # we overwrote `depends` in inherited recipe, so we must add it - # again the python versions as dependencies, but we only do this in - # case that the sub classes recipe does not contain any python - # version as dependency because it may be some recipes only - # compatible with a single version of python + # We ensure here that the recipe depends on python even it overrode + # `depends`. We only do this if it doesn't already depend on any + # python, since some recipes intentionally don't depend on/work + # with all python variants depends = self.depends depends.append(('python2', 'python3')) depends = list(set(depends)) From 11b4a975060ce9c848ae9b5f9e43af6a97e14e9e Mon Sep 17 00:00:00 2001 From: Alexander Taylor Date: Sun, 14 Jul 2019 21:58:36 +0100 Subject: [PATCH 16/41] Updated release number for develop branch --- pythonforandroid/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonforandroid/__init__.py b/pythonforandroid/__init__.py index 5b93b35870..2272e310e3 100644 --- a/pythonforandroid/__init__.py +++ b/pythonforandroid/__init__.py @@ -1 +1 @@ -__version__ = '2019.07.08' +__version__ = '2019.07.08.1.dev0' From 8122126dcbf04a170f6012f67211328833551ade Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Thu, 9 May 2019 10:56:28 +0100 Subject: [PATCH 17/41] feat: Allows registering the onRequestPermissionsResult callback. Adds an interface in PythonActivity and a method to register a Python function which will be called when the onRequestPermissionsResult callback is received. In android/permissions.py, a new function 'register_permissions_callback' is added to register a Python function (that takes three arguments) which will receive the three arguments of onRequestPermissionsResult. --- .../java/org/kivy/android/PythonActivity.java | 35 ++++++++++++++++-- .../android/src/android/permissions.py | 37 ++++++++++++++++++- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java index 425923433f..8193dbbd3e 100644 --- a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java @@ -581,7 +581,34 @@ public void onWindowFocusChanged(boolean hasFocus) { // call native function (since it's not yet loaded) } considerLoadingScreenRemoval(); - } + } + + /** + * Used by android.permissions p4a module to register a call back after + * requesting runtime permissions + **/ + public interface PermissionsCallback { + void onRequestPermissionsResult(int requestCode, String[] permissions, int[] grantResults); + } + + private PermissionsCallback permissionCallback; + private boolean havePermissionsCallback = false; + + public void addPermissionsCallback(PermissionsCallback callback) { + permissionCallback = callback; + havePermissionsCallback = true; + Log.v(TAG, "addPermissionsCallback(): Added callback for onRequestPermissionsResult"); + } + + @Override + public void onRequestPermissionsResult(int requestCode, String[] permissions, int[] grantResults) { + Log.v(TAG, "onRequestPermissionsResult()"); + if (havePermissionsCallback) { + Log.v(TAG, "onRequestPermissionsResult passed to callback"); + permissionCallback.onRequestPermissionsResult(requestCode, permissions, grantResults); + } + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + } /** * Used by android.permissions p4a module to check a permission @@ -592,9 +619,9 @@ public boolean checkCurrentPermission(String permission) { try { java.lang.reflect.Method methodCheckPermission = - Activity.class.getMethod("checkSelfPermission", java.lang.String.class); + Activity.class.getMethod("checkSelfPermission", java.lang.String.class); Object resultObj = methodCheckPermission.invoke(this, permission); - int result = Integer.parseInt(resultObj.toString()); + int result = Integer.parseInt(resultObj.toString()); if (result == PackageManager.PERMISSION_GRANTED) return true; } catch (IllegalAccessException | NoSuchMethodException | @@ -612,7 +639,7 @@ public void requestPermissions(String[] permissions) { try { java.lang.reflect.Method methodRequestPermission = Activity.class.getMethod("requestPermissions", - java.lang.String[].class, int.class); + java.lang.String[].class, int.class); methodRequestPermission.invoke(this, permissions, 1); } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { diff --git a/pythonforandroid/recipes/android/src/android/permissions.py b/pythonforandroid/recipes/android/src/android/permissions.py index 46dfc04287..3c42ef5fb0 100644 --- a/pythonforandroid/recipes/android/src/android/permissions.py +++ b/pythonforandroid/recipes/android/src/android/permissions.py @@ -1,6 +1,6 @@ try: - from jnius import autoclass + from jnius import autoclass, PythonJavaClass, java_method except ImportError: # To allow importing by build/manifest-creating code without # pyjnius being present: @@ -422,6 +422,41 @@ class Permission: ) +class onRequestPermissionsCallback(PythonJavaClass): + """Callback class for registering a Python callback from + onRequestPermissionsResult in PythonActivity. + """ + __javainterfaces__ = ['org.kivy.android.PythonActivity$PermissionsCallback'] + __javacontext__ = 'app' + _callback = None # To avoid garbage collection + + def __init__(self, func): + self.func = func + onRequestPermissionsCallback._callback = self + super().__init__() + + @java_method('(I[Ljava/lang/String;[I)V') + def onRequestPermissionsResult(self, requestCode, permissions, grantResults): + self.func(requestCode, permissions, grantResults) + + +def register_permissions_callback(callback): + """Register a callback. This will asynchronously receive arguments from + onRequestPermissionsResult on PythonActivity after request_permission(s) + is called. + + The callback must accept three arguments: requestCode, permissions and + grantResults. + + Note that calling request_permission on SDK_INT < 23 will return + immediately (as run-time permissions are not required), and so this + callback will never happen. + """ + java_callback = onRequestPermissionsCallback(callback) + python_activity = autoclass('org.kivy.android.PythonActivity') + python_activity.addPermissionsCallback(java_callback) + + def request_permissions(permissions): python_activity = autoclass('org.kivy.android.PythonActivity') python_activity.requestPermissions(permissions) From a0fbab64cc47c5d1cdd6004b3fdd496299b4b408 Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Mon, 15 Jul 2019 19:30:47 +0100 Subject: [PATCH 18/41] feat: Update Android permissions to allow passing a callback with each request. --- .../java/org/kivy/android/PythonActivity.java | 8 +- .../android/src/android/permissions.py | 84 +++++++++++++++---- 2 files changed, 72 insertions(+), 20 deletions(-) diff --git a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java index 8193dbbd3e..8e2f996da0 100644 --- a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java @@ -633,16 +633,20 @@ public boolean checkCurrentPermission(String permission) { /** * Used by android.permissions p4a module to request runtime permissions **/ - public void requestPermissions(String[] permissions) { + public void requestPermissionsWithRequestCode(String[] permissions, int requestCode) { if (android.os.Build.VERSION.SDK_INT < 23) return; try { java.lang.reflect.Method methodRequestPermission = Activity.class.getMethod("requestPermissions", java.lang.String[].class, int.class); - methodRequestPermission.invoke(this, permissions, 1); + methodRequestPermission.invoke(this, permissions, requestCode); } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { } } + + public void requestPermissions(String[] permissions) { + requestPermissionsWithRequestCode(permissions, 1); + } } diff --git a/pythonforandroid/recipes/android/src/android/permissions.py b/pythonforandroid/recipes/android/src/android/permissions.py index 3c42ef5fb0..b7d3cd7e9b 100644 --- a/pythonforandroid/recipes/android/src/android/permissions.py +++ b/pythonforandroid/recipes/android/src/android/permissions.py @@ -1,3 +1,4 @@ +import threading try: from jnius import autoclass, PythonJavaClass, java_method @@ -428,11 +429,10 @@ class onRequestPermissionsCallback(PythonJavaClass): """ __javainterfaces__ = ['org.kivy.android.PythonActivity$PermissionsCallback'] __javacontext__ = 'app' - _callback = None # To avoid garbage collection def __init__(self, func): self.func = func - onRequestPermissionsCallback._callback = self + onRequestPermissionsCallback._java_callback = self super().__init__() @java_method('(I[Ljava/lang/String;[I)V') @@ -440,30 +440,78 @@ def onRequestPermissionsResult(self, requestCode, permissions, grantResults): self.func(requestCode, permissions, grantResults) -def register_permissions_callback(callback): - """Register a callback. This will asynchronously receive arguments from - onRequestPermissionsResult on PythonActivity after request_permission(s) - is called. +class onRequestPermissionsManager: + """Class for requesting Android permissions via requestPermissions, + including registering callbacks to requestPermissions. - The callback must accept three arguments: requestCode, permissions and - grantResults. + Permissions are requested through the method 'request_permissions' which + accepts a list of permissions and an optional callback. + + Any callback will asynchronously receive arguments from + onRequestPermissionsResult on PythonActivity after requestPermissions is + called. + + The callback supplied must accept two arguments: 'permissions' and + 'grantResults' (as supplied to onPermissionsCallbackResult). Note that calling request_permission on SDK_INT < 23 will return - immediately (as run-time permissions are not required), and so this - callback will never happen. + immediately (as run-time permissions are not required), and so the callback + will never happen. Therefore, request_permissions should only be called + with a callback if calling check_permission has indicated that it is + necessary. + + The attribute '_java_callback' is initially None, but is set when the first + permissions request is made. It is set to an instance of + onRequestPermissionsCallback, which allows the Java callback to be + propagated to the class method 'python_callback'. This is then, in turn, + used to call an application callback if provided to request_permissions. """ - java_callback = onRequestPermissionsCallback(callback) - python_activity = autoclass('org.kivy.android.PythonActivity') - python_activity.addPermissionsCallback(java_callback) + _java_callback = None + _callbacks = {1: None} + _callback_id = 1 + _lock = threading.Lock() + @classmethod + def register_callback(cls): + """Register Java callback for requestPermissions.""" + cls._java_callback = onRequestPermissionsCallback(cls.python_callback) + python_activity = autoclass('org.kivy.android.PythonActivity') + python_activity.addPermissionsCallback(cls._java_callback) -def request_permissions(permissions): - python_activity = autoclass('org.kivy.android.PythonActivity') - python_activity.requestPermissions(permissions) + @classmethod + def request_permissions(cls, permissions, callback=None): + """Requests Android permissions from PythonActivity. + If 'callback' is supplied, the request is made with a new requestCode + and the callback is stored in the _callbacks dict. When a Java callback + with the matching requestCode is received, callback will be called + with arguments of 'permissions' and 'grantResults'. + """ + with cls._lock: + if not cls._java_callback: + cls.register_callback() + python_activity = autoclass('org.kivy.android.PythonActivity') + if not callback: + python_activity.requestPermissions(permissions) + else: + cls._callback_id += 1 + python_activity.requestPermissionsWithRequestCode( + permissions, cls._callback_id) + cls._callbacks[cls._callback_id] = callback + + @classmethod + def python_callback(cls, requestCode, permissions, grantResults): + """Calls the relevant callback with arguments of 'permissions' + and 'grantResults'.""" + if cls._callbacks.get(requestCode): + cls._callbacks[requestCode](permissions, grantResults) + + +def request_permissions(permissions, callback=None): + onRequestPermissionsManager.request_permissions(permissions, callback) -def request_permission(permission): - request_permissions([permission]) +def request_permission(permission, callback=None): + request_permissions([permission], callback) def check_permission(permission): From fba062a11fb2b2d283d342f89ac1c1dcd02d8be3 Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Tue, 16 Jul 2019 08:40:37 +0100 Subject: [PATCH 19/41] fix: A bit of renaming, and removed a redundant GC-preventing attribute. --- .../recipes/android/src/android/permissions.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pythonforandroid/recipes/android/src/android/permissions.py b/pythonforandroid/recipes/android/src/android/permissions.py index b7d3cd7e9b..88b5bcacf0 100644 --- a/pythonforandroid/recipes/android/src/android/permissions.py +++ b/pythonforandroid/recipes/android/src/android/permissions.py @@ -423,7 +423,7 @@ class Permission: ) -class onRequestPermissionsCallback(PythonJavaClass): +class _onRequestPermissionsCallback(PythonJavaClass): """Callback class for registering a Python callback from onRequestPermissionsResult in PythonActivity. """ @@ -432,17 +432,17 @@ class onRequestPermissionsCallback(PythonJavaClass): def __init__(self, func): self.func = func - onRequestPermissionsCallback._java_callback = self super().__init__() @java_method('(I[Ljava/lang/String;[I)V') - def onRequestPermissionsResult(self, requestCode, permissions, grantResults): + def onRequestPermissionsResult(self, requestCode, + permissions, grantResults): self.func(requestCode, permissions, grantResults) -class onRequestPermissionsManager: - """Class for requesting Android permissions via requestPermissions, - including registering callbacks to requestPermissions. +class _request_permissions_manager: + """Internal class for requesting Android permissions via + requestPermissions, including registering callbacks to requestPermissions. Permissions are requested through the method 'request_permissions' which accepts a list of permissions and an optional callback. @@ -474,7 +474,7 @@ class onRequestPermissionsManager: @classmethod def register_callback(cls): """Register Java callback for requestPermissions.""" - cls._java_callback = onRequestPermissionsCallback(cls.python_callback) + cls._java_callback = _onRequestPermissionsCallback(cls.python_callback) python_activity = autoclass('org.kivy.android.PythonActivity') python_activity.addPermissionsCallback(cls._java_callback) @@ -505,9 +505,11 @@ def python_callback(cls, requestCode, permissions, grantResults): if cls._callbacks.get(requestCode): cls._callbacks[requestCode](permissions, grantResults) +# Public API methods for requesting permissions + def request_permissions(permissions, callback=None): - onRequestPermissionsManager.request_permissions(permissions, callback) + _request_permissions_manager.request_permissions(permissions, callback) def request_permission(permission, callback=None): From 0d889bd5a739b4bec99d3b8e32a9ffed7422c4b3 Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Tue, 16 Jul 2019 15:00:04 +0100 Subject: [PATCH 20/41] style: Revert mistaken name change... --- pythonforandroid/recipes/android/src/android/permissions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pythonforandroid/recipes/android/src/android/permissions.py b/pythonforandroid/recipes/android/src/android/permissions.py index 88b5bcacf0..b8af351a57 100644 --- a/pythonforandroid/recipes/android/src/android/permissions.py +++ b/pythonforandroid/recipes/android/src/android/permissions.py @@ -440,7 +440,7 @@ def onRequestPermissionsResult(self, requestCode, self.func(requestCode, permissions, grantResults) -class _request_permissions_manager: +class _RequestPermissionsManager: """Internal class for requesting Android permissions via requestPermissions, including registering callbacks to requestPermissions. @@ -505,11 +505,11 @@ def python_callback(cls, requestCode, permissions, grantResults): if cls._callbacks.get(requestCode): cls._callbacks[requestCode](permissions, grantResults) -# Public API methods for requesting permissions +# Public API methods for requesting permissions def request_permissions(permissions, callback=None): - _request_permissions_manager.request_permissions(permissions, callback) + _RequestPermissionsManager.request_permissions(permissions, callback) def request_permission(permission, callback=None): From 3e26beee94f216508a8eb86b05e2711f7fba8205 Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Wed, 17 Jul 2019 14:08:48 +0100 Subject: [PATCH 21/41] fix: Convert Android permission int values to True/False Add docstrings --- .../android/src/android/permissions.py | 68 +++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/pythonforandroid/recipes/android/src/android/permissions.py b/pythonforandroid/recipes/android/src/android/permissions.py index b8af351a57..776fbd8981 100644 --- a/pythonforandroid/recipes/android/src/android/permissions.py +++ b/pythonforandroid/recipes/android/src/android/permissions.py @@ -423,6 +423,10 @@ class Permission: ) +PERMISSION_GRANTED = 0 +PERMISSION_DENIED = -1 + + class _onRequestPermissionsCallback(PythonJavaClass): """Callback class for registering a Python callback from onRequestPermissionsResult in PythonActivity. @@ -441,8 +445,7 @@ def onRequestPermissionsResult(self, requestCode, class _RequestPermissionsManager: - """Internal class for requesting Android permissions via - requestPermissions, including registering callbacks to requestPermissions. + """Internal class for requesting Android permissions. Permissions are requested through the method 'request_permissions' which accepts a list of permissions and an optional callback. @@ -465,10 +468,18 @@ class _RequestPermissionsManager: onRequestPermissionsCallback, which allows the Java callback to be propagated to the class method 'python_callback'. This is then, in turn, used to call an application callback if provided to request_permissions. + + The attribute '_callback_id' is incremented with each call to + request_permissions which has a callback (the value '1' is used for any + call which does not pass a callback). This is passed to requestCode in + the Java call, and used to identify (via the _callbacks dictionary) + the matching call. """ _java_callback = None _callbacks = {1: None} _callback_id = 1 + # Lock to prevent multiple calls to request_permissions being handled + # simultaneously (as incrementing _callback_id is not atomic) _lock = threading.Lock() @classmethod @@ -484,7 +495,7 @@ def request_permissions(cls, permissions, callback=None): If 'callback' is supplied, the request is made with a new requestCode and the callback is stored in the _callbacks dict. When a Java callback with the matching requestCode is received, callback will be called - with arguments of 'permissions' and 'grantResults'. + with arguments of 'permissions' and 'grant_results'. """ with cls._lock: if not cls._java_callback: @@ -502,13 +513,54 @@ def request_permissions(cls, permissions, callback=None): def python_callback(cls, requestCode, permissions, grantResults): """Calls the relevant callback with arguments of 'permissions' and 'grantResults'.""" + # Convert from Android codes to True/False + grant_results = [x == PERMISSION_GRANTED for x in grantResults] if cls._callbacks.get(requestCode): - cls._callbacks[requestCode](permissions, grantResults) + cls._callbacks[requestCode](permissions, grant_results) # Public API methods for requesting permissions def request_permissions(permissions, callback=None): + """Requests Android permissions. + + Args: + permissions (str): A list of permissions to requests (str) + callback (callable, optional): A function to call when the request + is completed (callable) + + Returns: + None + + Notes: + + Permission strings can be imported from the 'Permission' class in this + module. For example: + + from android import Permission + permissions_list = [Permission.CAMERA, + Permission.WRITE_EXTERNAL_STORAGE] + + See the p4a source file 'permissions.py' for a list of valid permission + strings (pythonforandroid/recipes/android/src/android/permissions.py). + + Any callback supplied must accept two arguments: + permissions (list of str): A list of permission strings + grant_results (list of bool): A list of bools indicating whether the + respective permission was granted. + See Android documentation for onPermissionsCallbackResult for + further information. + + Note that if the request is interupted the callback may contain an empty + list of permissions, without permissions being granted; the App should + check that each permission requested has been granted. + + Also note that calling request_permission on SDK_INT < 23 will return + immediately (as run-time permissions are not required), and so the callback + will never happen. Therefore, request_permissions should only be called + with a callback if calling check_permission has indicated that it is + necessary. + """ _RequestPermissionsManager.request_permissions(permissions, callback) @@ -517,6 +569,14 @@ def request_permission(permission, callback=None): def check_permission(permission): + """Checks if an app holds the passed permission. + + Args: + - permission An Android permission (str) + + Returns: + bool: True if the app holds the permission given, False otherwise. + """ python_activity = autoclass('org.kivy.android.PythonActivity') result = bool(python_activity.checkCurrentPermission( permission + "" From 736c639ada3cdbc5322d9aff56a5dad45f3c1798 Mon Sep 17 00:00:00 2001 From: opacam Date: Sat, 22 Jun 2019 11:33:21 +0200 Subject: [PATCH 22/41] [ndk] Make it raise an error if an old ndk is used In order to avoid issues with old android's NDKs, we force to raise an error in case that an NDK version lower than the specified is detected. We also add the ability to extract the android's NDK letter version, which for now, will only be used to inform the user of the version which is using --- pythonforandroid/recommendations.py | 70 +++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/pythonforandroid/recommendations.py b/pythonforandroid/recommendations.py index fd2fd3a8be..09ebb587d3 100644 --- a/pythonforandroid/recommendations.py +++ b/pythonforandroid/recommendations.py @@ -10,28 +10,80 @@ MAX_NDK_VERSION = 17 RECOMMENDED_NDK_VERSION = '17c' -OLD_NDK_MESSAGE = 'Older NDKs may not be compatible with all p4a features.' NEW_NDK_MESSAGE = 'Newer NDKs may not be fully supported by p4a.' +NDK_DOWNLOAD_URL = 'https://developer.android.com/ndk/downloads/' def check_ndk_version(ndk_dir): - # Check the NDK version against what is currently recommended + """ + Check the NDK version against what is currently recommended and raise an + exception of :class:`~pythonforandroid.util.BuildInterruptingException` in + case that the user tries to use an NDK lower than minimum supported, + specified via attribute `MIN_NDK_VERSION`. + + .. versionchanged:: 2019.06.06.1.dev0 + Added the ability to get android's ndk `letter version` and also + rewrote to raise an exception in case that an NDK version lower than + the minimum supported is detected. + """ version = read_ndk_version(ndk_dir) if version is None: - return # if we failed to read the version, just don't worry about it + warning( + 'Unable to read the ndk version, assuming that you are using an' + ' NDK greater than {min_supported} (the minimum ndk required to' + ' use p4a successfully).\n' + 'Note: If you got build errors, consider to download the' + ' recommended ndk version which is {rec_version} and try' + ' it again (after removing all the files generated with this' + ' build). To download the android NDK visit the following page: ' + '{ndk_url}'.format( + min_supported=MIN_NDK_VERSION, + rec_version=RECOMMENDED_NDK_VERSION, + ndk_url=NDK_DOWNLOAD_URL, + ) + ) + return + + # create a dictionary which will describe the relationship of the android's + # ndk minor version with the `human readable` letter version, egs: + # Pkg.Revision = 17.1.4828580 => ndk-17b + # Pkg.Revision = 17.2.4988734 => ndk-17c + # Pkg.Revision = 19.0.5232133 => ndk-19 (No letter) + minor_to_letter = {0: ''} + minor_to_letter.update( + {n + 1: chr(i) for n, i in enumerate(range(ord('b'), ord('b') + 25))} + ) major_version = version.version[0] + letter_version = minor_to_letter[version.version[1]] + string_version = '{major_version}{letter_version}'.format( + major_version=major_version, letter_version=letter_version + ) - info('Found NDK revision {}'.format(version)) + info('Found NDK version {}'.format(string_version)) if major_version < MIN_NDK_VERSION: - warning('Minimum recommended NDK version is {}'.format( - RECOMMENDED_NDK_VERSION)) - warning(OLD_NDK_MESSAGE) + raise BuildInterruptingException( + 'Unsupported NDK version detected {user_version}\n' + '* Note: Minimum supported NDK version is {min_supported}'.format( + user_version=string_version, min_supported=MIN_NDK_VERSION + ), + instructions=( + 'Please, go to the android ndk page ({ndk_url}) and download a' + ' supported version.\n*** The currently recommended NDK' + ' version is {rec_version} ***'.format( + ndk_url=NDK_DOWNLOAD_URL, + rec_version=RECOMMENDED_NDK_VERSION, + ) + ), + ) elif major_version > MAX_NDK_VERSION: - warning('Maximum recommended NDK version is {}'.format( - RECOMMENDED_NDK_VERSION)) + warning( + 'Maximum recommended NDK version is {}'.format( + RECOMMENDED_NDK_VERSION + ) + ) warning(NEW_NDK_MESSAGE) From 19972b430a23c61a9c98f82c083fb1834d75b131 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 19 Jul 2019 11:26:38 +0200 Subject: [PATCH 23/41] [test] Add unittest for `pythonforandroid.recommendations` --- tests/test_recommendations.py | 191 ++++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 tests/test_recommendations.py diff --git a/tests/test_recommendations.py b/tests/test_recommendations.py new file mode 100644 index 0000000000..9567b12848 --- /dev/null +++ b/tests/test_recommendations.py @@ -0,0 +1,191 @@ +import unittest +from os.path import join +from sys import version as py_version + +try: + from unittest import mock +except ImportError: + # `Python 2` or lower than `Python 3.3` does not + # have the `unittest.mock` module built-in + import mock +from pythonforandroid.recommendations import ( + check_ndk_api, + check_ndk_version, + check_target_api, + read_ndk_version, + MAX_NDK_VERSION, + RECOMMENDED_NDK_VERSION, + RECOMMENDED_TARGET_API, + MIN_NDK_API, + MIN_NDK_VERSION, + NDK_DOWNLOAD_URL, + ARMEABI_MAX_TARGET_API, + MIN_TARGET_API, +) +from pythonforandroid.util import BuildInterruptingException +running_in_py2 = int(py_version[0]) < 3 + + +class TestRecommendations(unittest.TestCase): + """ + An inherited class of `unittest.TestCase`to test the module + :mod:`~pythonforandroid.recommendations`. + """ + + def setUp(self): + self.ndk_dir = "/opt/android/android-ndk" + + @unittest.skipIf(running_in_py2, "`assertLogs` requires Python 3.4+") + @mock.patch("pythonforandroid.recommendations.read_ndk_version") + def test_check_ndk_version_greater_than_recommended(self, mock_read_ndk): + mock_read_ndk.return_value.version = [MAX_NDK_VERSION + 1, 0, 5232133] + with self.assertLogs(level="INFO") as cm: + check_ndk_version(self.ndk_dir) + mock_read_ndk.assert_called_once_with(self.ndk_dir) + self.assertEqual( + cm.output, + [ + "INFO:p4a:[INFO]: Found NDK version {ndk_current}".format( + ndk_current=MAX_NDK_VERSION + 1 + ), + "WARNING:p4a:[WARNING]:" + " Maximum recommended NDK version is {ndk_recommended}".format( + ndk_recommended=RECOMMENDED_NDK_VERSION + ), + "WARNING:p4a:[WARNING]:" + " Newer NDKs may not be fully supported by p4a.", + ], + ) + + @mock.patch("pythonforandroid.recommendations.read_ndk_version") + def test_check_ndk_version_lower_than_recommended(self, mock_read_ndk): + mock_read_ndk.return_value.version = [MIN_NDK_VERSION - 1, 0, 5232133] + with self.assertRaises(BuildInterruptingException) as e: + check_ndk_version(self.ndk_dir) + self.assertEqual( + e.exception.args[0], + "Unsupported NDK version detected {ndk_current}" + "\n* Note: Minimum supported NDK version is {ndk_min}".format( + ndk_current=MIN_NDK_VERSION - 1, ndk_min=MIN_NDK_VERSION + ), + ) + mock_read_ndk.assert_called_once_with(self.ndk_dir) + + @unittest.skipIf(running_in_py2, "`assertLogs` requires Python 3.4+") + def test_check_ndk_version_error(self): + """ + Test that a fake ndk dir give us two messages: + - first should be an `INFO` log + - second should be an `WARNING` log + """ + with self.assertLogs(level="INFO") as cm: + check_ndk_version(self.ndk_dir) + self.assertEqual( + cm.output, + [ + "INFO:p4a:[INFO]: Could not determine NDK version, " + "no source.properties in the NDK dir", + "WARNING:p4a:[WARNING]: Unable to read the ndk version, " + "assuming that you are using an NDK greater than 17 (the " + "minimum ndk required to use p4a successfully).\n" + "Note: If you got build errors, consider to download the " + "recommended ndk version which is 17c and try it again (after " + "removing all the files generated with this build). To " + "download the android NDK visit the following " + "page: {download_url}".format(download_url=NDK_DOWNLOAD_URL), + ], + ) + + @mock.patch("pythonforandroid.recommendations.open") + def test_read_ndk_version(self, mock_open_src_prop): + mock_open_src_prop.side_effect = [ + mock.mock_open( + read_data="Pkg.Revision = 17.2.4988734" + ).return_value + ] + version = read_ndk_version(self.ndk_dir) + mock_open_src_prop.assert_called_once_with( + join(self.ndk_dir, "source.properties") + ) + assert version == "17.2.4988734" + + @unittest.skipIf(running_in_py2, "`assertLogs` requires Python 3.4+") + @mock.patch("pythonforandroid.recommendations.open") + def test_read_ndk_version_error(self, mock_open_src_prop): + mock_open_src_prop.side_effect = [ + mock.mock_open(read_data="").return_value + ] + with self.assertLogs(level="INFO") as cm: + version = read_ndk_version(self.ndk_dir) + self.assertEqual( + cm.output, + [ + "INFO:p4a:[INFO]: Could not parse " + "$NDK_DIR/source.properties, not checking NDK version" + ], + ) + mock_open_src_prop.assert_called_once_with( + join(self.ndk_dir, "source.properties") + ) + assert version is None + + def test_check_target_api_error_arch_armeabi(self): + + with self.assertRaises(BuildInterruptingException) as e: + check_target_api(RECOMMENDED_TARGET_API, "armeabi") + self.assertEqual( + e.exception.args[0], + "Asked to build for armeabi architecture with API {ndk_api}, but " + "API {max_target_api} or greater does not support armeabi".format( + ndk_api=RECOMMENDED_TARGET_API, + max_target_api=ARMEABI_MAX_TARGET_API, + ), + ) + + @unittest.skipIf(running_in_py2, "`assertLogs` requires Python 3.4+") + def test_check_target_api_warning_target_api(self): + + with self.assertLogs(level="INFO") as cm: + check_target_api(MIN_TARGET_API - 1, MIN_TARGET_API) + self.assertEqual( + cm.output, + [ + "WARNING:p4a:[WARNING]: Target API 25 < 26", + "WARNING:p4a:[WARNING]: Target APIs lower than 26 are no " + "longer supported on Google Play, and are not recommended. " + "Note that the Target API can be higher than your device " + "Android version, and should usually be as high as possible.", + ], + ) + + def test_check_ndk_api_error_android_api(self): + """ + Given an `android api` greater than an `ndk_api`, we should get an + `BuildInterruptingException`. + """ + ndk_api = MIN_NDK_API + 1 + android_api = MIN_NDK_API + with self.assertRaises(BuildInterruptingException) as e: + check_ndk_api(ndk_api, android_api) + self.assertEqual( + e.exception.args[0], + "Target NDK API is {ndk_api}, higher than the target Android " + "API {android_api}.".format( + ndk_api=ndk_api, android_api=android_api + ), + ) + + @unittest.skipIf(running_in_py2, "`assertLogs` requires Python 3.4+") + def test_check_ndk_api_warning_old_ndk(self): + """ + Given an `android api` lower than the supported by p4a, we should + get an `BuildInterruptingException`. + """ + ndk_api = MIN_NDK_API - 1 + android_api = RECOMMENDED_TARGET_API + with self.assertLogs(level="INFO") as cm: + check_ndk_api(ndk_api, android_api) + self.assertEqual( + cm.output, + ["WARNING:p4a:[WARNING]: NDK API less than 21 is not supported"], + ) From cbfb51f8dd5a5394c000fcc3f16e8dd59615f046 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 19 Jul 2019 20:20:46 +0200 Subject: [PATCH 24/41] [texts] Capitalise NDK to be consistent --- pythonforandroid/recommendations.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pythonforandroid/recommendations.py b/pythonforandroid/recommendations.py index 09ebb587d3..16a0016586 100644 --- a/pythonforandroid/recommendations.py +++ b/pythonforandroid/recommendations.py @@ -22,7 +22,7 @@ def check_ndk_version(ndk_dir): specified via attribute `MIN_NDK_VERSION`. .. versionchanged:: 2019.06.06.1.dev0 - Added the ability to get android's ndk `letter version` and also + Added the ability to get android's NDK `letter version` and also rewrote to raise an exception in case that an NDK version lower than the minimum supported is detected. """ @@ -30,11 +30,11 @@ def check_ndk_version(ndk_dir): if version is None: warning( - 'Unable to read the ndk version, assuming that you are using an' - ' NDK greater than {min_supported} (the minimum ndk required to' + 'Unable to read the NDK version, assuming that you are using an' + ' NDK greater than {min_supported} (the minimum NDK required to' ' use p4a successfully).\n' 'Note: If you got build errors, consider to download the' - ' recommended ndk version which is {rec_version} and try' + ' recommended NDK version which is {rec_version} and try' ' it again (after removing all the files generated with this' ' build). To download the android NDK visit the following page: ' '{ndk_url}'.format( @@ -46,7 +46,7 @@ def check_ndk_version(ndk_dir): return # create a dictionary which will describe the relationship of the android's - # ndk minor version with the `human readable` letter version, egs: + # NDK minor version with the `human readable` letter version, egs: # Pkg.Revision = 17.1.4828580 => ndk-17b # Pkg.Revision = 17.2.4988734 => ndk-17c # Pkg.Revision = 19.0.5232133 => ndk-19 (No letter) @@ -70,7 +70,7 @@ def check_ndk_version(ndk_dir): user_version=string_version, min_supported=MIN_NDK_VERSION ), instructions=( - 'Please, go to the android ndk page ({ndk_url}) and download a' + 'Please, go to the android NDK page ({ndk_url}) and download a' ' supported version.\n*** The currently recommended NDK' ' version is {rec_version} ***'.format( ndk_url=NDK_DOWNLOAD_URL, From 6a8b3cb31ff3eb891b295c31ce86b61bed52e493 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 19 Jul 2019 20:28:13 +0200 Subject: [PATCH 25/41] [texts] Rewrite log messages to be more direct --- pythonforandroid/recommendations.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/pythonforandroid/recommendations.py b/pythonforandroid/recommendations.py index 16a0016586..ba98b6cbfb 100644 --- a/pythonforandroid/recommendations.py +++ b/pythonforandroid/recommendations.py @@ -30,14 +30,13 @@ def check_ndk_version(ndk_dir): if version is None: warning( - 'Unable to read the NDK version, assuming that you are using an' - ' NDK greater than {min_supported} (the minimum NDK required to' - ' use p4a successfully).\n' - 'Note: If you got build errors, consider to download the' - ' recommended NDK version which is {rec_version} and try' - ' it again (after removing all the files generated with this' - ' build). To download the android NDK visit the following page: ' - '{ndk_url}'.format( + 'Unable to read the NDK version from the given directory ' + '{}'.format(ndk_dir) + ) + warning( + "Make sure your NDK version is greater than {min_supported}. " + "If you get build errors, download the recommended NDK " + "{rec_version} from {ndk_url}".format( min_supported=MIN_NDK_VERSION, rec_version=RECOMMENDED_NDK_VERSION, ndk_url=NDK_DOWNLOAD_URL, @@ -65,9 +64,9 @@ def check_ndk_version(ndk_dir): if major_version < MIN_NDK_VERSION: raise BuildInterruptingException( - 'Unsupported NDK version detected {user_version}\n' - '* Note: Minimum supported NDK version is {min_supported}'.format( - user_version=string_version, min_supported=MIN_NDK_VERSION + 'The minimum supported NDK version is {min_supported}. You can ' + 'download it from {ndk_url}'.format( + min_supported=MIN_NDK_VERSION, ndk_url=NDK_DOWNLOAD_URL, ), instructions=( 'Please, go to the android NDK page ({ndk_url}) and download a' From 6688562a7eb455512007919e5f67b41741d99efd Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 19 Jul 2019 22:58:00 +0200 Subject: [PATCH 26/41] [recommendations] Refactor important log messages So it will be easier for us to maintain the tests for recommendations module --- pythonforandroid/recommendations.py | 74 ++++++++++++++++--------- tests/test_recommendations.py | 83 +++++++++++++++++------------ 2 files changed, 98 insertions(+), 59 deletions(-) diff --git a/pythonforandroid/recommendations.py b/pythonforandroid/recommendations.py index ba98b6cbfb..98e0a33e67 100644 --- a/pythonforandroid/recommendations.py +++ b/pythonforandroid/recommendations.py @@ -9,9 +9,38 @@ MIN_NDK_VERSION = 17 MAX_NDK_VERSION = 17 -RECOMMENDED_NDK_VERSION = '17c' +RECOMMENDED_NDK_VERSION = "17c" +NDK_DOWNLOAD_URL = "https://developer.android.com/ndk/downloads/" + +# Important log messages NEW_NDK_MESSAGE = 'Newer NDKs may not be fully supported by p4a.' -NDK_DOWNLOAD_URL = 'https://developer.android.com/ndk/downloads/' +UNKNOWN_NDK_MESSAGE = ( + 'Could not determine NDK version, no source.properties in the NDK dir' +) +PARSE_ERROR_NDK_MESSAGE = ( + 'Could not parse $NDK_DIR/source.properties, not checking NDK version' +) +READ_ERROR_NDK_MESSAGE = ( + 'Unable to read the NDK version from the given directory {ndk_dir}' +) +ENSURE_RIGHT_NDK_MESSAGE = ( + 'Make sure your NDK version is greater than {min_supported}. If you get ' + 'build errors, download the recommended NDK {rec_version} from {ndk_url}' +) +NDK_LOWER_THAN_SUPPORTED_MESSAGE = ( + 'The minimum supported NDK version is {min_supported}. ' + 'You can download it from {ndk_url}' +) +UNSUPPORTED_NDK_API_FOR_ARMEABI_MESSAGE = ( + 'Asked to build for armeabi architecture with API ' + '{req_ndk_api}, but API {max_ndk_api} or greater does not support armeabi' +) +CURRENT_NDK_VERSION_MESSAGE = ( + 'Found NDK version {ndk_version}' +) +RECOMMENDED_NDK_VERSION_MESSAGE = ( + 'Maximum recommended NDK version is {recommended_ndk_version}' +) def check_ndk_version(ndk_dir): @@ -29,14 +58,9 @@ def check_ndk_version(ndk_dir): version = read_ndk_version(ndk_dir) if version is None: + warning(READ_ERROR_NDK_MESSAGE.format(ndk_dir=ndk_dir)) warning( - 'Unable to read the NDK version from the given directory ' - '{}'.format(ndk_dir) - ) - warning( - "Make sure your NDK version is greater than {min_supported}. " - "If you get build errors, download the recommended NDK " - "{rec_version} from {ndk_url}".format( + ENSURE_RIGHT_NDK_MESSAGE.format( min_supported=MIN_NDK_VERSION, rec_version=RECOMMENDED_NDK_VERSION, ndk_url=NDK_DOWNLOAD_URL, @@ -60,13 +84,12 @@ def check_ndk_version(ndk_dir): major_version=major_version, letter_version=letter_version ) - info('Found NDK version {}'.format(string_version)) + info(CURRENT_NDK_VERSION_MESSAGE.format(ndk_version=string_version)) if major_version < MIN_NDK_VERSION: raise BuildInterruptingException( - 'The minimum supported NDK version is {min_supported}. You can ' - 'download it from {ndk_url}'.format( - min_supported=MIN_NDK_VERSION, ndk_url=NDK_DOWNLOAD_URL, + NDK_LOWER_THAN_SUPPORTED_MESSAGE.format( + min_supported=MIN_NDK_VERSION, ndk_url=NDK_DOWNLOAD_URL ), instructions=( 'Please, go to the android NDK page ({ndk_url}) and download a' @@ -79,8 +102,8 @@ def check_ndk_version(ndk_dir): ) elif major_version > MAX_NDK_VERSION: warning( - 'Maximum recommended NDK version is {}'.format( - RECOMMENDED_NDK_VERSION + RECOMMENDED_NDK_VERSION_MESSAGE.format( + recommended_ndk_version=RECOMMENDED_NDK_VERSION ) ) warning(NEW_NDK_MESSAGE) @@ -92,16 +115,14 @@ def read_ndk_version(ndk_dir): with open(join(ndk_dir, 'source.properties')) as fileh: ndk_data = fileh.read() except IOError: - info('Could not determine NDK version, no source.properties ' - 'in the NDK dir') + info(UNKNOWN_NDK_MESSAGE) return for line in ndk_data.split('\n'): if line.startswith('Pkg.Revision'): break else: - info('Could not parse $NDK_DIR/source.properties, not checking ' - 'NDK version') + info(PARSE_ERROR_NDK_MESSAGE) return # Line should have the form "Pkg.Revision = ..." @@ -130,9 +151,9 @@ def check_target_api(api, arch): if api >= ARMEABI_MAX_TARGET_API and arch == 'armeabi': raise BuildInterruptingException( - 'Asked to build for armeabi architecture with API ' - '{}, but API {} or greater does not support armeabi'.format( - api, ARMEABI_MAX_TARGET_API), + UNSUPPORTED_NDK_API_FOR_ARMEABI_MESSAGE.format( + req_ndk_api=api, max_ndk_api=ARMEABI_MAX_TARGET_API + ), instructions='You probably want to build with --arch=armeabi-v7a instead') if api < MIN_TARGET_API: @@ -143,14 +164,19 @@ def check_target_api(api, arch): MIN_NDK_API = 21 RECOMMENDED_NDK_API = 21 OLD_NDK_API_MESSAGE = ('NDK API less than {} is not supported'.format(MIN_NDK_API)) +TARGET_NDK_API_GREATER_THAN_TARGET_API_MESSAGE = ( + 'Target NDK API is {ndk_api}, ' + 'higher than the target Android API {android_api}.' +) def check_ndk_api(ndk_api, android_api): """Warn if the user's NDK is too high or low.""" if ndk_api > android_api: raise BuildInterruptingException( - 'Target NDK API is {}, higher than the target Android API {}.'.format( - ndk_api, android_api), + TARGET_NDK_API_GREATER_THAN_TARGET_API_MESSAGE.format( + ndk_api=ndk_api, android_api=android_api + ), instructions=('The NDK API is a minimum supported API number and must be lower ' 'than the target Android API')) diff --git a/tests/test_recommendations.py b/tests/test_recommendations.py index 9567b12848..2f3cc18db2 100644 --- a/tests/test_recommendations.py +++ b/tests/test_recommendations.py @@ -21,8 +21,21 @@ NDK_DOWNLOAD_URL, ARMEABI_MAX_TARGET_API, MIN_TARGET_API, + UNKNOWN_NDK_MESSAGE, + PARSE_ERROR_NDK_MESSAGE, + READ_ERROR_NDK_MESSAGE, + ENSURE_RIGHT_NDK_MESSAGE, + NDK_LOWER_THAN_SUPPORTED_MESSAGE, + UNSUPPORTED_NDK_API_FOR_ARMEABI_MESSAGE, + CURRENT_NDK_VERSION_MESSAGE, + RECOMMENDED_NDK_VERSION_MESSAGE, + TARGET_NDK_API_GREATER_THAN_TARGET_API_MESSAGE, + OLD_NDK_API_MESSAGE, + NEW_NDK_MESSAGE, + OLD_API_MESSAGE, ) from pythonforandroid.util import BuildInterruptingException + running_in_py2 = int(py_version[0]) < 3 @@ -45,15 +58,17 @@ def test_check_ndk_version_greater_than_recommended(self, mock_read_ndk): self.assertEqual( cm.output, [ - "INFO:p4a:[INFO]: Found NDK version {ndk_current}".format( - ndk_current=MAX_NDK_VERSION + 1 + "INFO:p4a:[INFO]: {}".format( + CURRENT_NDK_VERSION_MESSAGE.format( + ndk_version=MAX_NDK_VERSION + 1 + ) ), - "WARNING:p4a:[WARNING]:" - " Maximum recommended NDK version is {ndk_recommended}".format( - ndk_recommended=RECOMMENDED_NDK_VERSION + "WARNING:p4a:[WARNING]: {}".format( + RECOMMENDED_NDK_VERSION_MESSAGE.format( + recommended_ndk_version=RECOMMENDED_NDK_VERSION + ) ), - "WARNING:p4a:[WARNING]:" - " Newer NDKs may not be fully supported by p4a.", + "WARNING:p4a:[WARNING]: {}".format(NEW_NDK_MESSAGE), ], ) @@ -64,9 +79,8 @@ def test_check_ndk_version_lower_than_recommended(self, mock_read_ndk): check_ndk_version(self.ndk_dir) self.assertEqual( e.exception.args[0], - "Unsupported NDK version detected {ndk_current}" - "\n* Note: Minimum supported NDK version is {ndk_min}".format( - ndk_current=MIN_NDK_VERSION - 1, ndk_min=MIN_NDK_VERSION + NDK_LOWER_THAN_SUPPORTED_MESSAGE.format( + min_supported=MIN_NDK_VERSION, ndk_url=NDK_DOWNLOAD_URL ), ) mock_read_ndk.assert_called_once_with(self.ndk_dir) @@ -83,16 +97,17 @@ def test_check_ndk_version_error(self): self.assertEqual( cm.output, [ - "INFO:p4a:[INFO]: Could not determine NDK version, " - "no source.properties in the NDK dir", - "WARNING:p4a:[WARNING]: Unable to read the ndk version, " - "assuming that you are using an NDK greater than 17 (the " - "minimum ndk required to use p4a successfully).\n" - "Note: If you got build errors, consider to download the " - "recommended ndk version which is 17c and try it again (after " - "removing all the files generated with this build). To " - "download the android NDK visit the following " - "page: {download_url}".format(download_url=NDK_DOWNLOAD_URL), + "INFO:p4a:[INFO]: {}".format(UNKNOWN_NDK_MESSAGE), + "WARNING:p4a:[WARNING]: {}".format( + READ_ERROR_NDK_MESSAGE.format(ndk_dir=self.ndk_dir) + ), + "WARNING:p4a:[WARNING]: {}".format( + ENSURE_RIGHT_NDK_MESSAGE.format( + min_supported=MIN_NDK_VERSION, + rec_version=RECOMMENDED_NDK_VERSION, + ndk_url=NDK_DOWNLOAD_URL, + ) + ), ], ) @@ -119,10 +134,7 @@ def test_read_ndk_version_error(self, mock_open_src_prop): version = read_ndk_version(self.ndk_dir) self.assertEqual( cm.output, - [ - "INFO:p4a:[INFO]: Could not parse " - "$NDK_DIR/source.properties, not checking NDK version" - ], + ["INFO:p4a:[INFO]: {}".format(PARSE_ERROR_NDK_MESSAGE)], ) mock_open_src_prop.assert_called_once_with( join(self.ndk_dir, "source.properties") @@ -135,10 +147,9 @@ def test_check_target_api_error_arch_armeabi(self): check_target_api(RECOMMENDED_TARGET_API, "armeabi") self.assertEqual( e.exception.args[0], - "Asked to build for armeabi architecture with API {ndk_api}, but " - "API {max_target_api} or greater does not support armeabi".format( - ndk_api=RECOMMENDED_TARGET_API, - max_target_api=ARMEABI_MAX_TARGET_API, + UNSUPPORTED_NDK_API_FOR_ARMEABI_MESSAGE.format( + req_ndk_api=RECOMMENDED_TARGET_API, + max_ndk_api=ARMEABI_MAX_TARGET_API, ), ) @@ -151,10 +162,9 @@ def test_check_target_api_warning_target_api(self): cm.output, [ "WARNING:p4a:[WARNING]: Target API 25 < 26", - "WARNING:p4a:[WARNING]: Target APIs lower than 26 are no " - "longer supported on Google Play, and are not recommended. " - "Note that the Target API can be higher than your device " - "Android version, and should usually be as high as possible.", + "WARNING:p4a:[WARNING]: {old_api_msg}".format( + old_api_msg=OLD_API_MESSAGE + ), ], ) @@ -169,8 +179,7 @@ def test_check_ndk_api_error_android_api(self): check_ndk_api(ndk_api, android_api) self.assertEqual( e.exception.args[0], - "Target NDK API is {ndk_api}, higher than the target Android " - "API {android_api}.".format( + TARGET_NDK_API_GREATER_THAN_TARGET_API_MESSAGE.format( ndk_api=ndk_api, android_api=android_api ), ) @@ -187,5 +196,9 @@ def test_check_ndk_api_warning_old_ndk(self): check_ndk_api(ndk_api, android_api) self.assertEqual( cm.output, - ["WARNING:p4a:[WARNING]: NDK API less than 21 is not supported"], + [ + "WARNING:p4a:[WARNING]: {}".format( + OLD_NDK_API_MESSAGE.format(MIN_NDK_API) + ) + ], ) From bf2e7238f2f3969e8f1d139c04c93e24b11adb4c Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Thu, 25 Jul 2019 19:30:21 +0200 Subject: [PATCH 27/41] libzmq: fix compilation under arch-arm64 --- pythonforandroid/recipes/libzmq/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pythonforandroid/recipes/libzmq/__init__.py b/pythonforandroid/recipes/libzmq/__init__.py index b7b33aa140..7bf6c2b762 100644 --- a/pythonforandroid/recipes/libzmq/__init__.py +++ b/pythonforandroid/recipes/libzmq/__init__.py @@ -35,6 +35,7 @@ def build_arch(self, arch): '--without-documentation', '--prefix={}'.format(prefix), '--with-libsodium=no', + '--disable-libunwind', _env=env) shprint(sh.make, _env=env) shprint(sh.make, 'install', _env=env) @@ -72,8 +73,8 @@ def get_recipe_env(self, arch): env['CXXFLAGS'] += ' -lgnustl_shared' env['LDFLAGS'] += ' -L{}/sources/cxx-stl/gnu-libstdc++/{}/libs/{}'.format( self.ctx.ndk_dir, self.ctx.toolchain_version, arch) - env['CXXFLAGS'] += ' --sysroot={}/platforms/android-{}/arch-arm'.format( - self.ctx.ndk_dir, self.ctx.ndk_api) + env['CXXFLAGS'] += ' --sysroot={}/platforms/android-{}/{}'.format( + self.ctx.ndk_dir, self.ctx.ndk_api, arch.platform_dir) return env From 312b5228bdaed16d6a8cb58c87195c5bdd507ec2 Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Tue, 23 Jul 2019 23:49:34 +0200 Subject: [PATCH 28/41] Basic toolchain.py unit tests First simple set of tests for toolchain.py Also refactors `Context.prepare_build_environment()` slightly. Splits concerns to improve readability and ease unit testing. --- pythonforandroid/build.py | 114 +++++++++++++++++++++++--------------- tests/test_toolchain.py | 89 +++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 46 deletions(-) create mode 100644 tests/test_toolchain.py diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index f0fbc86e9d..a5845358bf 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -27,6 +27,66 @@ RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API) +def get_cython_path(): + for cython_fn in ("cython", "cython3", "cython2", "cython-2.7"): + cython = sh.which(cython_fn) + if cython: + return cython + raise BuildInterruptingException('No cython binary found.') + + +def get_ndk_platform_dir(ndk_dir, ndk_api, arch): + ndk_platform_dir_exists = True + platform_dir = arch.platform_dir + ndk_platform = join( + ndk_dir, + 'platforms', + 'android-{}'.format(ndk_api), + platform_dir) + if not exists(ndk_platform): + warning("ndk_platform doesn't exist: {}".format(ndk_platform)) + ndk_platform_dir_exists = False + return ndk_platform, ndk_platform_dir_exists + + +def get_toolchain_versions(ndk_dir, arch): + toolchain_versions = [] + toolchain_path_exists = True + toolchain_prefix = arch.toolchain_prefix + toolchain_path = join(ndk_dir, 'toolchains') + if isdir(toolchain_path): + toolchain_contents = glob.glob('{}/{}-*'.format(toolchain_path, + toolchain_prefix)) + toolchain_versions = [split(path)[-1][len(toolchain_prefix) + 1:] + for path in toolchain_contents] + else: + warning('Could not find toolchain subdirectory!') + toolchain_path_exists = False + return toolchain_versions, toolchain_path_exists + + +def get_targets(sdk_dir): + if exists(join(sdk_dir, 'tools', 'bin', 'avdmanager')): + avdmanager = sh.Command(join(sdk_dir, 'tools', 'bin', 'avdmanager')) + targets = avdmanager('list', 'target').stdout.decode('utf-8').split('\n') + elif exists(join(sdk_dir, 'tools', 'android')): + android = sh.Command(join(sdk_dir, 'tools', 'android')) + targets = android('list').stdout.decode('utf-8').split('\n') + else: + raise BuildInterruptingException( + 'Could not find `android` or `sdkmanager` binaries in Android SDK', + instructions='Make sure the path to the Android SDK is correct') + return targets + + +def get_available_apis(sdk_dir): + targets = get_targets(sdk_dir) + apis = [s for s in targets if re.match(r'^ *API level: ', s)] + apis = [re.findall(r'[0-9]+', s) for s in apis] + apis = [int(s[0]) for s in apis if s] + return apis + + class Context(object): '''A build context. If anything will be built, an instance this class will be instantiated and used to hold all the build state.''' @@ -238,20 +298,7 @@ def prepare_build_environment(self, self.android_api = android_api check_target_api(android_api, self.archs[0].arch) - - if exists(join(sdk_dir, 'tools', 'bin', 'avdmanager')): - avdmanager = sh.Command(join(sdk_dir, 'tools', 'bin', 'avdmanager')) - targets = avdmanager('list', 'target').stdout.decode('utf-8').split('\n') - elif exists(join(sdk_dir, 'tools', 'android')): - android = sh.Command(join(sdk_dir, 'tools', 'android')) - targets = android('list').stdout.decode('utf-8').split('\n') - else: - raise BuildInterruptingException( - 'Could not find `android` or `sdkmanager` binaries in Android SDK', - instructions='Make sure the path to the Android SDK is correct') - apis = [s for s in targets if re.match(r'^ *API level: ', s)] - apis = [re.findall(r'[0-9]+', s) for s in apis] - apis = [int(s[0]) for s in apis if s] + apis = get_available_apis(self.sdk_dir) info('Available Android APIs are ({})'.format( ', '.join(map(str, apis)))) if android_api in apis: @@ -327,46 +374,21 @@ def prepare_build_environment(self, if not self.ccache: info('ccache is missing, the build will not be optimized in the ' 'future.') - for cython_fn in ("cython", "cython3", "cython2", "cython-2.7"): - cython = sh.which(cython_fn) - if cython: - self.cython = cython - break - else: - raise BuildInterruptingException('No cython binary found.') - if not self.cython: - ok = False - warning("Missing requirement: cython is not installed") + self.cython = get_cython_path() # This would need to be changed if supporting multiarch APKs arch = self.archs[0] - platform_dir = arch.platform_dir toolchain_prefix = arch.toolchain_prefix - toolchain_version = None - self.ndk_platform = join( - self.ndk_dir, - 'platforms', - 'android-{}'.format(self.ndk_api), - platform_dir) - if not exists(self.ndk_platform): - warning('ndk_platform doesn\'t exist: {}'.format( - self.ndk_platform)) - ok = False + self.ndk_platform, ndk_platform_dir_exists = get_ndk_platform_dir( + self.ndk_dir, self.ndk_api, arch) + ok = ok and ndk_platform_dir_exists py_platform = sys.platform if py_platform in ['linux2', 'linux3']: py_platform = 'linux' - - toolchain_versions = [] - toolchain_path = join(self.ndk_dir, 'toolchains') - if isdir(toolchain_path): - toolchain_contents = glob.glob('{}/{}-*'.format(toolchain_path, - toolchain_prefix)) - toolchain_versions = [split(path)[-1][len(toolchain_prefix) + 1:] - for path in toolchain_contents] - else: - warning('Could not find toolchain subdirectory!') - ok = False + toolchain_versions, toolchain_path_exists = get_toolchain_versions( + self.ndk_dir, arch) + ok = ok and toolchain_path_exists toolchain_versions.sort() toolchain_versions_gcc = [] diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py new file mode 100644 index 0000000000..d4fa91f160 --- /dev/null +++ b/tests/test_toolchain.py @@ -0,0 +1,89 @@ +import sys +import pytest +import mock +from pythonforandroid.toolchain import ToolchainCL +from pythonforandroid.util import BuildInterruptingException + + +def patch_sys_argv(argv): + return mock.patch('sys.argv', argv) + + +def patch_argparse_print_help(): + return mock.patch('argparse.ArgumentParser.print_help') + + +def raises_system_exit(): + return pytest.raises(SystemExit) + + +class TestToolchainCL: + + def test_help(self): + """ + Calling with `--help` should print help and exit 0. + """ + argv = ['toolchain.py', '--help', '--storage-dir=/tmp'] + with patch_sys_argv(argv), raises_system_exit( + ) as ex_info, patch_argparse_print_help() as m_print_help: + ToolchainCL() + assert ex_info.value.code == 0 + assert m_print_help.call_args_list == [mock.call()] + + @pytest.mark.skipif(sys.version_info < (3, 0), reason="requires python3") + def test_unknown(self): + """ + Calling with unknown args should print help and exit 1. + """ + argv = ['toolchain.py', '--unknown'] + with patch_sys_argv(argv), raises_system_exit( + ) as ex_info, patch_argparse_print_help() as m_print_help: + ToolchainCL() + assert ex_info.value.code == 1 + assert m_print_help.call_args_list == [mock.call()] + + def test_create(self): + """ + Basic `create` distribution test. + """ + argv = [ + 'toolchain.py', + 'create', + '--sdk-dir=/tmp/android-sdk', + '--ndk-dir=/tmp/android-ndk', + '--dist-name=test_toolchain', + ] + with patch_sys_argv(argv), mock.patch( + 'pythonforandroid.build.get_available_apis' + ) as m_get_available_apis, mock.patch( + 'pythonforandroid.build.get_toolchain_versions' + ) as m_get_toolchain_versions, mock.patch( + 'pythonforandroid.build.get_ndk_platform_dir' + ) as m_get_ndk_platform_dir, mock.patch( + 'pythonforandroid.build.get_cython_path' + ) as m_get_cython_path, mock.patch( + 'pythonforandroid.toolchain.build_dist_from_args' + ) as m_build_dist_from_args: + m_get_available_apis.return_value = [27] + m_get_toolchain_versions.return_value = (['4.9'], True) + m_get_ndk_platform_dir.return_value = ( + '/tmp/android-ndk/platforms/android-21/arch-arm', True) + ToolchainCL() + assert m_get_available_apis.call_args_list == [ + mock.call('/tmp/android-sdk')] + assert m_get_toolchain_versions.call_args_list == [ + mock.call('/tmp/android-ndk', mock.ANY)] + assert m_get_cython_path.call_args_list == [mock.call()] + assert m_build_dist_from_args.call_count == 1 + + def test_create_no_sdk_dir(self): + """ + The `--sdk-dir` is mandatory to `create` a distribution. + """ + argv = ['toolchain.py', 'create'] + with mock.patch('sys.argv', argv), pytest.raises( + BuildInterruptingException + ) as ex_info: + ToolchainCL() + assert ex_info.value.message == ( + 'Android SDK dir was not specified, exiting.') From 47b472b11dea4bf5667528faa376ee7727d3a9b2 Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Sat, 27 Jul 2019 02:01:42 +0200 Subject: [PATCH 29/41] Increases toolchain.py test coverage Increases `test_create()` coverage demonstrating crash referenced in: Adds `test_recipes()` checking if it prints out without crashing. --- pythonforandroid/toolchain.py | 9 ++++++ tests/test_toolchain.py | 57 ++++++++++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index 6cfb300763..d7883a2690 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -739,6 +739,15 @@ def _read_configuration(): sys.argv.append(arg) def recipes(self, args): + """ + Prints recipes basic info, e.g. + ``` + python3 3.7.1 + depends: ['hostpython3', 'sqlite3', 'openssl', 'libffi'] + conflicts: ['python2'] + optional depends: ['sqlite3', 'libffi', 'openssl'] + ``` + """ ctx = self.ctx if args.compact: print(" ".join(set(Recipe.list_recipes(ctx)))) diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py index d4fa91f160..a4b68c86ee 100644 --- a/tests/test_toolchain.py +++ b/tests/test_toolchain.py @@ -1,6 +1,8 @@ +import io import sys import pytest import mock +from pythonforandroid.recipe import Recipe from pythonforandroid.toolchain import ToolchainCL from pythonforandroid.util import BuildInterruptingException @@ -13,6 +15,10 @@ def patch_argparse_print_help(): return mock.patch('argparse.ArgumentParser.print_help') +def patch_sys_stdout(): + return mock.patch('sys.stdout', new_callable=io.StringIO) + + def raises_system_exit(): return pytest.raises(SystemExit) @@ -51,6 +57,8 @@ def test_create(self): 'create', '--sdk-dir=/tmp/android-sdk', '--ndk-dir=/tmp/android-ndk', + '--bootstrap=service_only', + '--requirements=python3', '--dist-name=test_toolchain', ] with patch_sys_argv(argv), mock.patch( @@ -62,8 +70,11 @@ def test_create(self): ) as m_get_ndk_platform_dir, mock.patch( 'pythonforandroid.build.get_cython_path' ) as m_get_cython_path, mock.patch( - 'pythonforandroid.toolchain.build_dist_from_args' - ) as m_build_dist_from_args: + 'pythonforandroid.toolchain.build_recipes' + ) as m_build_recipes, mock.patch( + 'pythonforandroid.bootstraps.service_only.' + 'ServiceOnlyBootstrap.run_distribute' + ) as m_run_distribute: m_get_available_apis.return_value = [27] m_get_toolchain_versions.return_value = (['4.9'], True) m_get_ndk_platform_dir.return_value = ( @@ -74,16 +85,54 @@ def test_create(self): assert m_get_toolchain_versions.call_args_list == [ mock.call('/tmp/android-ndk', mock.ANY)] assert m_get_cython_path.call_args_list == [mock.call()] - assert m_build_dist_from_args.call_count == 1 + build_order = [ + 'hostpython3', 'libffi', 'openssl', 'sqlite3', 'python3', + 'genericndkbuild', 'setuptools', 'six', 'pyjnius', 'android', + ] + python_modules = [] + context = mock.ANY + project_dir = None + assert m_build_recipes.call_args_list == [ + mock.call( + build_order, + python_modules, + context, + project_dir, + ignore_project_setup_py=False + ) + ] + assert m_run_distribute.call_args_list == [mock.call()] def test_create_no_sdk_dir(self): """ The `--sdk-dir` is mandatory to `create` a distribution. """ argv = ['toolchain.py', 'create'] - with mock.patch('sys.argv', argv), pytest.raises( + with patch_sys_argv(argv), pytest.raises( BuildInterruptingException ) as ex_info: ToolchainCL() assert ex_info.value.message == ( 'Android SDK dir was not specified, exiting.') + + @pytest.mark.skipif(sys.version_info < (3, 0), reason="requires python3") + def test_recipes(self): + """ + Checks the `recipes` command prints out recipes information without crashing. + """ + argv = ['toolchain.py', 'recipes'] + with patch_sys_argv(argv), patch_sys_stdout() as m_stdout: + ToolchainCL() + # check if we have common patterns in the output + expected_strings = ( + 'conflicts:', + 'depends:', + 'kivy', + 'optional depends:', + 'python3', + 'sdl2', + ) + for expected_string in expected_strings: + assert expected_string in m_stdout.getvalue() + # deletes static attribute to not mess with other tests + del Recipe.recipes From 901f9e4edb8c6a4f965e49c5bcc90f1e98cd4c73 Mon Sep 17 00:00:00 2001 From: Jonas Thiem Date: Thu, 25 Jul 2019 20:19:07 +0200 Subject: [PATCH 30/41] Add a document describing how p4a interacts with pip & python packages --- doc/source/contribute.rst | 151 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/doc/source/contribute.rst b/doc/source/contribute.rst index 2fde4daf17..1de0883c4c 100644 --- a/doc/source/contribute.rst +++ b/doc/source/contribute.rst @@ -79,3 +79,154 @@ Release checklist - [ ] `armeabi-v7a` - [ ] `arm64-v8a` - [ ] Check that the version number is correct + + + +How python-for-android uses `pip` +--------------------------------- + +*Last update: July 2019* + +This section is meant to provide a quick summary how +p4a (=python-for-android) uses pip and python packages in +its build process. +**It is written for a python +packagers point of view, not for regular end users or +contributors,** to assist with making pip developers and +other packaging experts aware of p4a's packaging needs. + +Please note this section just attempts to neutrally list the +current mechanisms, so some of this isn't necessarily meant +to stay but just how things work inside p4a in +this very moment. + + +Basic concepts +~~~~~~~~~~~~~~ + +*(This part repeats other parts of the docs, for the sake of +making this a more independent read)* + +p4a builds & packages a python application for use on Android. +It does this by providing a Java wrapper, and for graphical applications +an SDL2-based wrapper which can be used with the kivy UI toolkit if +desired (or alternatively just plain PySDL2). Any such python application +will of course have further library dependencies to do its work. + +p4a supports two types of package dependencies for a project: + +**Recipe:** install script in custom p4a format. Can either install +C/C++ or other things that cannot be pulled in via pip, or things +that can be installed via pip but break on android by default. +These are maintained primarily inside the p4a source tree by p4a +contributors and interested folks. + +**Python package:** any random pip python package can be directly +installed if it doesn't need adjustments to work for Android. + +p4a will map any dependency to an internal recipe if present, and +otherwise use pip to obtain it regularly from whatever external source. + + +Install process regarding packages +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The install/build process of a p4a project, as triggered by the +`p4a apk` command, roughly works as follows in regards to python +packages: + +1. The user has specified a project folder to install. This is either + just a folder with python scripts and a `main.py`, or it may + also have a `pyproject.toml` for a more standardized install. + +2. Dependencies are collected: they can be either specified via + ``--requirements`` as a list of names or pip-style URLs, or p4a + can optionally scan them from a project folder via the + pep517 library (if there is a `pyproject.toml` or `setup.py`). + +3. The collected dependencies are mapped to p4a's recipes if any are + available for them, otherwise they're kept around as external + regular package references. + +4. All the dependencies mapped to recipes are built via p4a's internal + mechanisms to build these recipes. (This may or may not indirectly + use pip, depending on whether the recipe wraps a python package + or not and uses pip to install or not.) + +5. **If the user has specified to install the project in standardized + ways,** then the `setup.py`/whatever build system + of the project will be run. This happens with cross compilation set up + (`CC`/`CFLAGS`/... set to use the + proper toolchain) and a custom site-packages location. + The actual comand is a simple `pip install .` in the project folder + with some extra options: e.g. all dependencies that were already + installed by recipes will be pinned with a `-c` constraints file + to make sure pip won't install them, and build isolation will be + disabled via ``--no-build-isolation`` so pip doesn't reinstall + recipe-packages on its own. + + **If the user has not specified to use standardized build approaches**, + p4a will simply install all the remaining dependencies that weren't + mapped to recipes directly and just plain copy in the user project + without installing. Any `setup.py` or `pyproject.toml` of the user + project will then be ignored in this step. + +6. Google's gradle is invoked to package it all up into an `.apk`. + + +Overall process / package relevant notes for p4a +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Here are some common things worth knowing about python-for-android's +dealing with python packages: + +- Packages will work fine without a recipe if they would also build + on Linux ARM, don't use any API not available in the NDK if they + use native code, and don't use any weird compiler flags the toolchain + doesn't like if they use native code. The package also needs to + work with cross compilation. + +- There is currently no easy way for a package to know it is being + cross-compiled (at least that we know of) other than examining the + `CC` compiler that was set, or that it is being cross-compiled for + Android specifically. If that breaks a package it currently needs + to be worked around with a recipe. + +- If a package does **not** work, p4a developers will often create a + recipe instead of getting upstream to fix it because p4a simply + is too niche. + +- Most packages without native code will just work out of the box. + Many with native code tend not to, especially if complex, e.g. numpy. + +- Anything mapped to a p4a recipe cannot be just reinstalled by pip, + specifically also not inside build isolation as a dependency. + (It *may* work if the patches of the recipe are just relevant + to fix runtime issues.) + Therefore as of now, the best way to deal with this limitation seems + to be to keep build isolation always off. + + +Ideas for the future regarding packaging +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- We in overall prefer to use the recipe mechanism less if we can. + In overall the recipes are just a collection of workarounds. + It may look quite hacky from the outside, since p4a + version pins recipe-wrapped packages usually to make the patches reliably + apply. This creates work for the recipes to be kept up-to-date, and + obviously this approach doesn't scale too well. However, it has ended + up as a quite practical interims solution until better ways are found. + +- Obviously, it would be nice if packages could know they are being + cross-compiled, and for Android specifically. We aren't currently aware + of a good mechanism for that. + +- If pip could actually run the recipes (instead of p4a wrapping pip and + doing so) then this might even allow build isolation to work - but + this might be too complex to get working. It might be more practical + to just gradually reduce the reliance on recipes instead and make + more packages work out of the box. This has been done e.g. with + improvements to the cross-compile environment being set up automatically, + and we're open for any ideas on how to improve this. + From 55e3cddb319cceda48277d532c0999a8857fd0d4 Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Sun, 28 Jul 2019 10:23:14 +0200 Subject: [PATCH 31/41] Bumps to Kivy==1.11.1 --- pythonforandroid/recipes/kivy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonforandroid/recipes/kivy/__init__.py b/pythonforandroid/recipes/kivy/__init__.py index 689d5da646..baf3e2ae6c 100644 --- a/pythonforandroid/recipes/kivy/__init__.py +++ b/pythonforandroid/recipes/kivy/__init__.py @@ -6,7 +6,7 @@ class KivyRecipe(CythonRecipe): - version = '1.11.0' + version = '1.11.1' url = 'https://github.com/kivy/kivy/archive/{version}.zip' name = 'kivy' From db6e7351b1d1ed6b67d02f6cf10f5a8615a980df Mon Sep 17 00:00:00 2001 From: Jonas Thiem Date: Tue, 25 Jun 2019 14:06:19 +0200 Subject: [PATCH 32/41] Fix foreground notification being mandatory and more. Details: - Adds start_service_not_as_foreground() to avoid always displaying an associated notification without this being configurable at runtime. It only is configurable right now if setting the foreground property via --service, which cannot be done when using the simpler service/main.py entrypoint - Fixes service_only service code template not rendering .foreground correctly since it added an outdated, useless function name override - Fixes notification channel name which is visible to end user hardcoding "python" in its name which is not ideal for end user naming (since the average user might not even know what python is) - Fixes override of doStartForeground() leading to quite some error-prone code duplication --- .../java/org/kivy/android/PythonService.java | 21 +++++---- .../common/build/templates/Service.tmpl.java | 46 ++----------------- .../java/org/kivy/android/PythonActivity.java | 31 ++++++++++++- .../java/org/kivy/android/PythonActivity.java | 31 ++++++++++++- .../build/templates/Service.tmpl.java | 10 ++-- .../java/org/kivy/android/PythonActivity.java | 31 ++++++++++++- .../recipes/android/src/android/_android.pyx | 34 +++++++++----- .../android/src/android/_android_jni.c | 28 ----------- tests/test_distribution.py | 5 +- 9 files changed, 134 insertions(+), 103 deletions(-) diff --git a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonService.java b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonService.java index 7456c617b3..6d951e8525 100644 --- a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonService.java +++ b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonService.java @@ -35,8 +35,11 @@ public class PythonService extends Service implements Runnable { private String pythonHome; private String pythonPath; private String serviceEntrypoint; + private boolean serviceStartAsForeground; // Argument to pass to Python code, private String pythonServiceArgument; + + public static PythonService mService = null; private Intent startIntent = null; @@ -46,10 +49,6 @@ public void setAutoRestartService(boolean restart) { autoRestartService = restart; } - public boolean canDisplayNotification() { - return true; - } - public int startType() { return START_NOT_STICKY; } @@ -79,18 +78,24 @@ public int onStartCommand(Intent intent, int flags, int startId) { pythonName = extras.getString("pythonName"); pythonHome = extras.getString("pythonHome"); pythonPath = extras.getString("pythonPath"); + serviceStartAsForeground = ( + extras.getString("serviceStartAsForeground") == "true" + ); pythonServiceArgument = extras.getString("pythonServiceArgument"); - pythonThread = new Thread(this); pythonThread.start(); - if (canDisplayNotification()) { + if (serviceStartAsForeground) { doStartForeground(extras); } return startType(); } + protected int getServiceId() { + return 1; + } + protected void doStartForeground(Bundle extras) { String serviceTitle = extras.getString("serviceTitle"); String serviceDescription = extras.getString("serviceDescription"); @@ -116,7 +121,7 @@ protected void doStartForeground(Bundle extras) { // for android 8+ we need to create our own channel // https://stackoverflow.com/questions/47531742/startforeground-fail-after-upgrade-to-android-8-1 String NOTIFICATION_CHANNEL_ID = "org.kivy.p4a"; //TODO: make this configurable - String channelName = "PythonSerice"; //TODO: make this configurable + String channelName = "Background Service"; //TODO: make this configurable NotificationChannel chan = new NotificationChannel(NOTIFICATION_CHANNEL_ID, channelName, NotificationManager.IMPORTANCE_NONE); @@ -132,7 +137,7 @@ protected void doStartForeground(Bundle extras) { builder.setSmallIcon(context.getApplicationInfo().icon); notification = builder.build(); } - startForeground(1, notification); + startForeground(getServiceId(), notification); } @Override diff --git a/pythonforandroid/bootstraps/common/build/templates/Service.tmpl.java b/pythonforandroid/bootstraps/common/build/templates/Service.tmpl.java index 3ed10c2690..50c08f08a1 100644 --- a/pythonforandroid/bootstraps/common/build/templates/Service.tmpl.java +++ b/pythonforandroid/bootstraps/common/build/templates/Service.tmpl.java @@ -1,15 +1,8 @@ package {{ args.package }}; -import android.os.Build; -import java.lang.reflect.Method; -import java.lang.reflect.InvocationTargetException; import android.content.Intent; import android.content.Context; -import android.app.Notification; -import android.app.PendingIntent; -import android.os.Bundle; import org.kivy.android.PythonService; -import org.kivy.android.PythonActivity; public class Service{{ name|capitalize }} extends PythonService { @@ -20,41 +13,9 @@ public int startType() { } {% endif %} - {% if not foreground %} @Override - public boolean canDisplayNotification() { - return false; - } - {% endif %} - - @Override - protected void doStartForeground(Bundle extras) { - Notification notification; - Context context = getApplicationContext(); - Intent contextIntent = new Intent(context, PythonActivity.class); - PendingIntent pIntent = PendingIntent.getActivity(context, 0, contextIntent, - PendingIntent.FLAG_UPDATE_CURRENT); - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.HONEYCOMB) { - notification = new Notification( - context.getApplicationInfo().icon, "{{ args.name }}", System.currentTimeMillis()); - try { - // prevent using NotificationCompat, this saves 100kb on apk - Method func = notification.getClass().getMethod( - "setLatestEventInfo", Context.class, CharSequence.class, - CharSequence.class, PendingIntent.class); - func.invoke(notification, context, "{{ args.name }}", "{{ name| capitalize }}", pIntent); - } catch (NoSuchMethodException | IllegalAccessException | - IllegalArgumentException | InvocationTargetException e) { - } - } else { - Notification.Builder builder = new Notification.Builder(context); - builder.setContentTitle("{{ args.name }}"); - builder.setContentText("{{ name| capitalize }}"); - builder.setContentIntent(pIntent); - builder.setSmallIcon(context.getApplicationInfo().icon); - notification = builder.build(); - } - startForeground({{ service_id }}, notification); + protected int getServiceId() { + return {{ service_id }}; } static public void start(Context ctx, String pythonServiceArgument) { @@ -62,8 +23,11 @@ static public void start(Context ctx, String pythonServiceArgument) { String argument = ctx.getFilesDir().getAbsolutePath() + "/app"; intent.putExtra("androidPrivate", ctx.getFilesDir().getAbsolutePath()); intent.putExtra("androidArgument", argument); + intent.putExtra("serviceTitle", "{{ args.name }}"); + intent.putExtra("serviceDescription", "{{ name|capitalize }}"); intent.putExtra("serviceEntrypoint", "{{ entrypoint }}"); intent.putExtra("pythonName", "{{ name }}"); + intent.putExtra("serviceStartAsForeground", "{{ foreground|lower }}"); intent.putExtra("pythonHome", argument); intent.putExtra("pythonPath", argument + ":" + argument + "/lib"); intent.putExtra("pythonServiceArgument", pythonServiceArgument); diff --git a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java index 425923433f..406f3acb62 100644 --- a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java @@ -365,8 +365,32 @@ protected void onActivityResult(int requestCode, int resultCode, Intent intent) } } - public static void start_service(String serviceTitle, String serviceDescription, - String pythonServiceArgument) { + public static void start_service( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument + ) { + _do_start_service( + serviceTitle, serviceDescription, pythonServiceArgument, true + ); + } + + public static void start_service_not_as_foreground( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument + ) { + _do_start_service( + serviceTitle, serviceDescription, pythonServiceArgument, false + ); + } + + public static void _do_start_service( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument, + boolean showForegroundNotification + ) { Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class); String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath(); String filesDirectory = argument; @@ -378,6 +402,9 @@ public static void start_service(String serviceTitle, String serviceDescription, serviceIntent.putExtra("pythonName", "python"); serviceIntent.putExtra("pythonHome", app_root_dir); serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib"); + serviceIntent.putExtra("serviceStartAsForeground", + (showForegroundNotification ? "true" : "false") + ); serviceIntent.putExtra("serviceTitle", serviceTitle); serviceIntent.putExtra("serviceDescription", serviceDescription); serviceIntent.putExtra("pythonServiceArgument", pythonServiceArgument); diff --git a/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java index 8e45967616..85fc88150d 100644 --- a/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java @@ -380,8 +380,32 @@ protected void onActivityResult(int requestCode, int resultCode, Intent intent) } } - public static void start_service(String serviceTitle, String serviceDescription, - String pythonServiceArgument) { + public static void start_service( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument + ) { + _do_start_service( + serviceTitle, serviceDescription, pythonServiceArgument, true + ); + } + + public static void start_service_not_as_foreground( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument + ) { + _do_start_service( + serviceTitle, serviceDescription, pythonServiceArgument, false + ); + } + + public static void _do_start_service( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument, + boolean showForegroundNotification + ) { Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class); String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath(); String filesDirectory = argument; @@ -393,6 +417,9 @@ public static void start_service(String serviceTitle, String serviceDescription, serviceIntent.putExtra("pythonName", "python"); serviceIntent.putExtra("pythonHome", app_root_dir); serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib"); + serviceIntent.putExtra("serviceStartAsForeground", + (showForegroundNotification ? "true" : "false") + ); serviceIntent.putExtra("serviceTitle", serviceTitle); serviceIntent.putExtra("serviceDescription", serviceDescription); serviceIntent.putExtra("pythonServiceArgument", pythonServiceArgument); diff --git a/pythonforandroid/bootstraps/service_only/build/templates/Service.tmpl.java b/pythonforandroid/bootstraps/service_only/build/templates/Service.tmpl.java index ecbf3fe961..598549d345 100644 --- a/pythonforandroid/bootstraps/service_only/build/templates/Service.tmpl.java +++ b/pythonforandroid/bootstraps/service_only/build/templates/Service.tmpl.java @@ -22,15 +22,10 @@ public int startType() { } {% endif %} - {% if foreground %} - /** - * {@inheritDoc} - */ @Override - public boolean getStartForeground() { - return true; + protected int getServiceId() { + return {{ service_id }}; } - {% endif %} public static void start(Context ctx, String pythonServiceArgument) { String argument = ctx.getFilesDir().getAbsolutePath() + "/app"; @@ -41,6 +36,7 @@ public static void start(Context ctx, String pythonServiceArgument) { intent.putExtra("serviceTitle", "{{ name|capitalize }}"); intent.putExtra("serviceDescription", ""); intent.putExtra("pythonName", "{{ name }}"); + intent.putExtra("serviceStartAsForeground", "{{ foreground|lower }}"); intent.putExtra("pythonHome", argument); intent.putExtra("androidUnpack", argument); intent.putExtra("pythonPath", argument + ":" + argument + "/lib"); diff --git a/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java index effa54cf36..2bd908ce92 100644 --- a/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java @@ -437,8 +437,32 @@ protected void onActivityResult(int requestCode, int resultCode, Intent intent) } } - public static void start_service(String serviceTitle, String serviceDescription, - String pythonServiceArgument) { + public static void start_service( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument + ) { + _do_start_service( + serviceTitle, serviceDescription, pythonServiceArgument, true + ); + } + + public static void start_service_not_as_foreground( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument + ) { + _do_start_service( + serviceTitle, serviceDescription, pythonServiceArgument, false + ); + } + + public static void _do_start_service( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument, + boolean showForegroundNotification + ) { Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class); String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath(); String filesDirectory = argument; @@ -450,6 +474,9 @@ public static void start_service(String serviceTitle, String serviceDescription, serviceIntent.putExtra("pythonName", "python"); serviceIntent.putExtra("pythonHome", app_root_dir); serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib"); + serviceIntent.putExtra("serviceStartAsForeground", + (showForegroundNotification ? "true" : "false") + ); serviceIntent.putExtra("serviceTitle", serviceTitle); serviceIntent.putExtra("serviceDescription", serviceDescription); serviceIntent.putExtra("pythonServiceArgument", pythonServiceArgument); diff --git a/pythonforandroid/recipes/android/src/android/_android.pyx b/pythonforandroid/recipes/android/src/android/_android.pyx index aeaaf2310a..bdca2df454 100644 --- a/pythonforandroid/recipes/android/src/android/_android.pyx +++ b/pythonforandroid/recipes/android/src/android/_android.pyx @@ -280,17 +280,29 @@ class AndroidBrowser(object): import webbrowser webbrowser.register('android', AndroidBrowser) -cdef extern void android_start_service(char *, char *, char *) -def start_service(title=None, description=None, arg=None): - cdef char *j_title = NULL - cdef char *j_description = NULL - if title is not None: - j_title = title - if description is not None: - j_description = description - if arg is not None: - j_arg = arg - android_start_service(j_title, j_description, j_arg) + +def start_service(title="Background Service", + description="", arg="", + as_foreground=True): + # Legacy None value support (for old function signature style): + if title is None: + title = "Background Service" + if description is None: + description = "" + if arg is None: + arg = "" + + # Start service: + mActivity = autoclass('org.kivy.android.PythonActivity').mActivity + if as_foreground: + mActivity.start_service( + title, description, arg + ) + else: + mActivity.start_service_not_as_foreground( + title, description, arg + ) + cdef extern void android_stop_service() def stop_service(): diff --git a/pythonforandroid/recipes/android/src/android/_android_jni.c b/pythonforandroid/recipes/android/src/android/_android_jni.c index 9fea723ed8..cf1b1bf500 100644 --- a/pythonforandroid/recipes/android/src/android/_android_jni.c +++ b/pythonforandroid/recipes/android/src/android/_android_jni.c @@ -201,34 +201,6 @@ void android_get_buildinfo() { } } -void android_start_service(char *title, char *description, char *arg) { - static JNIEnv *env = NULL; - static jclass *cls = NULL; - static jmethodID mid = NULL; - - if (env == NULL) { - env = SDL_ANDROID_GetJNIEnv(); - aassert(env); - cls = (*env)->FindClass(env, JNI_NAMESPACE "/PythonActivity"); - aassert(cls); - mid = (*env)->GetStaticMethodID(env, cls, "start_service", - "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V"); - aassert(mid); - } - - jstring j_title = NULL; - jstring j_description = NULL; - jstring j_arg = NULL; - if ( title != 0 ) - j_title = (*env)->NewStringUTF(env, title); - if ( description != 0 ) - j_description = (*env)->NewStringUTF(env, description); - if ( arg != 0 ) - j_arg = (*env)->NewStringUTF(env, arg); - - (*env)->CallStaticVoidMethod(env, cls, mid, j_title, j_description, j_arg); -} - void android_stop_service() { static JNIEnv *env = NULL; static jclass *cls = NULL; diff --git a/tests/test_distribution.py b/tests/test_distribution.py index 1716060513..dece511074 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -84,11 +84,12 @@ def test_folder_exist(self, mock_exists): :meth:`~pythonforandroid.distribution.Distribution.folder_exist` is called once with the proper arguments.""" + mock_exists.return_value = False self.setUp_distribution_with_bootstrap( - Bootstrap().get_bootstrap("sdl2", self.ctx) + Bootstrap.get_bootstrap("sdl2", self.ctx) ) self.ctx.bootstrap.distribution.folder_exists() - mock_exists.assert_called_once_with( + mock_exists.assert_called_with( self.ctx.bootstrap.distribution.dist_dir ) From 7d5fdee64b1280469edc2020b3084ed8f9421162 Mon Sep 17 00:00:00 2001 From: Jonas Thiem Date: Thu, 17 Jan 2019 22:27:04 +0100 Subject: [PATCH 33/41] Add functions for obtaining the default storage paths --- doc/source/apis.rst | 54 +++++++++ .../recipes/android/src/android/storage.py | 103 ++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 pythonforandroid/recipes/android/src/android/storage.py diff --git a/doc/source/apis.rst b/doc/source/apis.rst index 7c3c307f4e..beae347625 100644 --- a/doc/source/apis.rst +++ b/doc/source/apis.rst @@ -5,6 +5,60 @@ Working on Android This page gives details on accessing Android APIs and managing other interactions on Android. +Storage paths +------------- + +If you want to store and retrieve data, you shouldn't just save to +the current directory, and not hardcode `/sdcard/` or some other +path either - it might differ per device. + +Instead, the `android` module which you can add to your `--requirements` +allows you to query the most commonly required paths:: + + from android.storage import app_storage_path + settings_path = app_storage_path() + + from android.storage import primary_external_storage_path + primary_ext_storage = primary_external_storage_path() + + from android.storage import secondary_external_storage_path + secondary_ext_storage = secondary_external_storage_path() + +`app_storage_path()` gives you Android's so-called "internal storage" +which is specific to your app and cannot seen by others or the user. +It compares best to the AppData directory on Windows. + +`primary_external_storage_path()` returns Android's so-called +"primary external storage", often found at `/sdcard/` and potentially +accessible to any other app. +It compares best to the Documents directory on Windows. +Requires `Permission.WRITE_EXTERNAL_STORAGE` to read and write to. + +`secondary_external_storage_path()` returns Android's so-called +"secondary external storage", often found at `/storage/External_SD/`. +It compares best to an external disk plugged to a Desktop PC, and can +after a device restart become inaccessible if removed. +Requires `Permission.WRITE_EXTERNAL_STORAGE` to read and write to. + +.. warning:: + Even if `secondary_external_storage_path` returns a path + the external sd card may still not be present. + Only non-empty contents or a successful write indicate that it is. + +Read more on all the different storage types and what to use them for +in the Android documentation: + +https://developer.android.com/training/data-storage/files + +A note on permissions +~~~~~~~~~~~~~~~~~~~~~ + +Only the internal storage is always accessible with no additional +permissions. For both primary and secondary external storage, you need +to obtain `Permission.WRITE_EXTERNAL_STORAGE` **and the user may deny it.** +Also, if you get it, both forms of external storage may only allow +your app to write to the common pre-existing folders like "Music", +"Documents", and so on. (see the Android Docs linked above for details) Runtime permissions ------------------- diff --git a/pythonforandroid/recipes/android/src/android/storage.py b/pythonforandroid/recipes/android/src/android/storage.py new file mode 100644 index 0000000000..d1b5cde472 --- /dev/null +++ b/pythonforandroid/recipes/android/src/android/storage.py @@ -0,0 +1,103 @@ + +from jnius import autoclass, cast +import os + + +Environment = autoclass('android.os.Environment') + + +def _android_has_is_removable_func(): + VERSION = autoclass('android.os.Build$VERSION') + return (VERSION.SDK_INT >= 24) + + +def _get_sdcard_path(): + """ Internal function to return getExternalStorageDirectory() + path. This is internal because it may either return the internal, + or an external sd card, depending on the device. + Use primary_external_storage_path() + or secondary_external_storage_path() instead which try to + distinguish this properly. + """ + return ( + Environment.getExternalStorageDirectory().getAbsolutePath() + ) + + +def app_storage_path(): + """ Locate the built-in device storage used for this app only. + + This storage is APP-SPECIFIC, and not visible to other apps. + It will be wiped when your app is uninstalled. + + Returns directory path to storage. + """ + PythonActivity = autoclass('org.kivy.android.PythonActivity') + currentActivity = cast('android.app.Activity', + PythonActivity.mActivity) + context = cast('android.content.ContextWrapper', + currentActivity.getApplicationContext()) + file_p = cast('java.io.File', context.getFilesDir()) + return os.path.normpath(os.path.abspath( + file_p.getAbsolutePath().replace("/", os.path.sep))) + + +def primary_external_storage_path(): + """ Locate the built-in device storage that user can see via file browser. + Often found at: /sdcard/ + + This is storage is SHARED, and visible to other apps and the user. + It will remain untouched when your app is uninstalled. + + Returns directory path to storage. + + WARNING: You need storage permissions to access this storage. + """ + if _android_has_is_removable_func(): + sdpath = _get_sdcard_path() + # Apparently this can both return primary (built-in) or + # secondary (removable) external storage depending on the device, + # therefore check that we got what we wanted: + if not Environment.isExternalStorageRemovable(sdpath): + return sdpath + if "EXTERNAL_STORAGE" in os.environ: + return os.environ["EXTERNAL_STORAGE"] + raise RuntimeError( + "unexpectedly failed to determine " + + "primary external storage path" + ) + + +def secondary_external_storage_path(): + """ Locate the external SD Card storage, which may not be present. + Often found at: /sdcard/External_SD/ + + This storage is SHARED, visible to other apps, and may not be + be available if the user didn't put in an external SD card. + It will remain untouched when your app is uninstalled. + + Returns None if not found, otherwise path to storage. + + WARNING: You need storage permissions to access this storage. + If it is not writable and presents as empty even with + permissions, then the external sd card may not be present. + """ + if _android_has_is_removable_func: + # See if getExternalStorageDirectory() returns secondary ext storage: + sdpath = _get_sdcard_path() + # Apparently this can both return primary (built-in) or + # secondary (removable) external storage depending on the device, + # therefore check that we got what we wanted: + if Environment.isExternalStorageRemovable(sdpath): + if os.path.exists(sdpath): + return sdpath + + # See if we can take a guess based on environment variables: + p = None + if "SECONDARY_STORAGE" in os.environ: + p = os.environ["SECONDARY_STORAGE"] + elif "EXTERNAL_SDCARD_STORAGE" in os.environ: + p = os.environ["EXTERNAL_SDCARD_STORAGE"] + if os.path.exists(p): + return p + return None From ea38331d25452a5eafe003bd0105f75098a5f266 Mon Sep 17 00:00:00 2001 From: Alexander Taylor Date: Sun, 28 Jul 2019 17:10:59 +0100 Subject: [PATCH 34/41] Added setuptools to Kivy recipe requirements --- pythonforandroid/recipes/kivy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonforandroid/recipes/kivy/__init__.py b/pythonforandroid/recipes/kivy/__init__.py index 689d5da646..1980f709ff 100644 --- a/pythonforandroid/recipes/kivy/__init__.py +++ b/pythonforandroid/recipes/kivy/__init__.py @@ -10,7 +10,7 @@ class KivyRecipe(CythonRecipe): url = 'https://github.com/kivy/kivy/archive/{version}.zip' name = 'kivy' - depends = ['sdl2', 'pyjnius'] + depends = ['sdl2', 'pyjnius', 'setuptools'] def cythonize_build(self, env, build_dir='.'): super(KivyRecipe, self).cythonize_build(env, build_dir=build_dir) From c63588dd1ec08870b59be9cf6d7bec70e17b9af9 Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Sun, 28 Jul 2019 23:08:29 +0200 Subject: [PATCH 35/41] Update storage.py fixes runtime errors 1) fixes isExternalStorageRemovable is expecting a file object, error was: ``` 07-28 16:36:46.845 2152 2784 I python : File "/home/andre/workspace/EtherollApp/.buildozer/android/platform/build/build/python-installs/etheroll/android/storage.py", line 61, in primary_external_storage_path 07-28 16:36:46.847 2152 2784 I python : File "jnius/jnius_export_class.pxi", line 1034, in jnius.jnius.JavaMultipleMethod.__call__ 07-28 16:36:46.848 2152 2784 I python : jnius.jnius.JavaException: No methods matching your arguments, available: ['()Z', '(Ljava/io/File;)Z'] ``` 2) fixes `os.path.exists()` doesn't accept `None`, error was: 07-28 22:31:25.877 31203 31313 I python : File "/home/andre/workspace/EtherollApp/.buildozer/android/platform/build/build/python-installs/etheroll/android/storage.py", line 101, in secondary_external_storage_path 07-28 22:31:25.878 31203 31313 I python : File "/home/andre/workspace/EtherollApp/.buildozer/android/platform/build/build/other_builds/python3-libffi-openssl-sqlite3/armeabi-v7a__ndk_target_21/python3/Lib/genericpath.py", line 19, in exists 07-28 22:31:25.879 31203 31313 I python : TypeError: stat: path should be string, bytes, os.PathLike or integer, not NoneType 3) fixes on services `PythonActivity.mActivity` is None, refs https://github.com/kivy/kivy/pull/6388 --- .../recipes/android/src/android/storage.py | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/pythonforandroid/recipes/android/src/android/storage.py b/pythonforandroid/recipes/android/src/android/storage.py index d1b5cde472..f4d01403bc 100644 --- a/pythonforandroid/recipes/android/src/android/storage.py +++ b/pythonforandroid/recipes/android/src/android/storage.py @@ -1,9 +1,9 @@ - from jnius import autoclass, cast import os Environment = autoclass('android.os.Environment') +File = autoclass('java.io.File') def _android_has_is_removable_func(): @@ -24,6 +24,19 @@ def _get_sdcard_path(): ) +def _get_activity(): + """ + Retrieves the activity from `PythonActivity` fallback to `PythonService`. + """ + PythonActivity = autoclass('org.kivy.android.PythonActivity') + activity = PythonActivity.mActivity + if activity is None: + # assume we're running from the background service + PythonService = autoclass('org.kivy.android.PythonService') + activity = PythonService.mService + return activity + + def app_storage_path(): """ Locate the built-in device storage used for this app only. @@ -32,9 +45,8 @@ def app_storage_path(): Returns directory path to storage. """ - PythonActivity = autoclass('org.kivy.android.PythonActivity') - currentActivity = cast('android.app.Activity', - PythonActivity.mActivity) + activity = _get_activity() + currentActivity = cast('android.app.Activity', activity) context = cast('android.content.ContextWrapper', currentActivity.getApplicationContext()) file_p = cast('java.io.File', context.getFilesDir()) @@ -58,7 +70,7 @@ def primary_external_storage_path(): # Apparently this can both return primary (built-in) or # secondary (removable) external storage depending on the device, # therefore check that we got what we wanted: - if not Environment.isExternalStorageRemovable(sdpath): + if not Environment.isExternalStorageRemovable(File(sdpath)): return sdpath if "EXTERNAL_STORAGE" in os.environ: return os.environ["EXTERNAL_STORAGE"] @@ -88,7 +100,7 @@ def secondary_external_storage_path(): # Apparently this can both return primary (built-in) or # secondary (removable) external storage depending on the device, # therefore check that we got what we wanted: - if Environment.isExternalStorageRemovable(sdpath): + if Environment.isExternalStorageRemovable(File(sdpath)): if os.path.exists(sdpath): return sdpath @@ -98,6 +110,6 @@ def secondary_external_storage_path(): p = os.environ["SECONDARY_STORAGE"] elif "EXTERNAL_SDCARD_STORAGE" in os.environ: p = os.environ["EXTERNAL_SDCARD_STORAGE"] - if os.path.exists(p): + if p is not None and os.path.exists(p): return p return None From 76b947ceee581f7247ad4edfc46e22d7d5442672 Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Mon, 29 Jul 2019 09:01:55 +0100 Subject: [PATCH 36/41] fix: Call request_permissions callback immediately for SDK_INT < 23 --- .../android/src/android/permissions.py | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/pythonforandroid/recipes/android/src/android/permissions.py b/pythonforandroid/recipes/android/src/android/permissions.py index 776fbd8981..963f48454f 100644 --- a/pythonforandroid/recipes/android/src/android/permissions.py +++ b/pythonforandroid/recipes/android/src/android/permissions.py @@ -457,11 +457,8 @@ class _RequestPermissionsManager: The callback supplied must accept two arguments: 'permissions' and 'grantResults' (as supplied to onPermissionsCallbackResult). - Note that calling request_permission on SDK_INT < 23 will return - immediately (as run-time permissions are not required), and so the callback - will never happen. Therefore, request_permissions should only be called - with a callback if calling check_permission has indicated that it is - necessary. + Note that for SDK_INT < 23, run-time permissions are not required, and so + the callback will be called immediately. The attribute '_java_callback' is initially None, but is set when the first permissions request is made. It is set to an instance of @@ -475,6 +472,7 @@ class _RequestPermissionsManager: the Java call, and used to identify (via the _callbacks dictionary) the matching call. """ + _SDK_INT = None _java_callback = None _callbacks = {1: None} _callback_id = 1 @@ -497,6 +495,16 @@ def request_permissions(cls, permissions, callback=None): with the matching requestCode is received, callback will be called with arguments of 'permissions' and 'grant_results'. """ + if not cls._SDK_INT: + # Get the Android build version and store it + VERSION = autoclass('android.os.Build$VERSION') + cls.SDK_INT = VERSION.SDK_INT + if cls.SDK_INT < 23: + # No run-time permissions needed, return immediately. + if callback: + callback(permissions, [True for x in permissions]) + return + # Request permissions with cls._lock: if not cls._java_callback: cls.register_callback() @@ -555,11 +563,9 @@ def request_permissions(permissions, callback=None): list of permissions, without permissions being granted; the App should check that each permission requested has been granted. - Also note that calling request_permission on SDK_INT < 23 will return - immediately (as run-time permissions are not required), and so the callback - will never happen. Therefore, request_permissions should only be called - with a callback if calling check_permission has indicated that it is - necessary. + Also note that when calling request_permission on SDK_INT < 23, the + callback will be returned immediately as requesting permissions is not + required. """ _RequestPermissionsManager.request_permissions(permissions, callback) From 721e7c57b29c2481e844dee3f314b1b96ee7ae38 Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Tue, 30 Jul 2019 23:36:45 +0200 Subject: [PATCH 37/41] Unit tests Recipe download feature Currently unit tests the following: - `Recipe.download_if_necessary()` - `Recipe.download()` Next up is `Recipe.download_file()`. This is more difficult and will be addressed in subsequent pull request. --- pythonforandroid/recipe.py | 4 +-- tests/test_recipe.py | 64 ++++++++++++++++++++++++++++++++++++++ tox.ini | 1 + 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 4de888306a..48efa8de1d 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -156,10 +156,10 @@ def report_hook(index, blksize, size): while True: try: urlretrieve(url, target, report_hook) - except OSError as e: + except OSError: attempts += 1 if attempts >= 5: - raise e + raise stdout.write('Download failed retrying in a second...') time.sleep(1) continue diff --git a/tests/test_recipe.py b/tests/test_recipe.py index 8946c92247..eefddee8aa 100644 --- a/tests/test_recipe.py +++ b/tests/test_recipe.py @@ -2,10 +2,28 @@ import types import unittest import warnings +import mock +from backports import tempfile from pythonforandroid.build import Context from pythonforandroid.recipe import Recipe, import_recipe +def patch_logger(level): + return mock.patch('pythonforandroid.recipe.{}'.format(level)) + + +def patch_logger_info(): + return patch_logger('info') + + +def patch_logger_debug(): + return patch_logger('debug') + + +class DummyRecipe(Recipe): + pass + + class TestRecipe(unittest.TestCase): def test_recipe_dirs(self): @@ -58,3 +76,49 @@ def test_import_recipe(self): module = import_recipe(name, pathname) assert module is not None assert recorded_warnings == [] + + def test_download_if_necessary(self): + """ + Download should happen via `Recipe.download()` only if the recipe + specific environment variable is not set. + """ + # download should happen as the environment variable is not set + recipe = DummyRecipe() + with mock.patch.object(Recipe, 'download') as m_download: + recipe.download_if_necessary() + assert m_download.call_args_list == [mock.call()] + # after setting it the download should be skipped + env_var = 'P4A_test_recipe_DIR' + env_dict = {env_var: '1'} + with mock.patch.object(Recipe, 'download') as m_download, mock.patch.dict(os.environ, env_dict): + recipe.download_if_necessary() + assert m_download.call_args_list == [] + + def test_download(self): + """ + Verifies the actual download gets triggered when the URL is set. + """ + # test with no URL set + recipe = DummyRecipe() + with patch_logger_info() as m_info: + recipe.download() + assert m_info.call_args_list == [ + mock.call('Skipping test_recipe download as no URL is set')] + # when the URL is set `Recipe.download_file()` should be called + filename = 'Python-3.7.4.tgz' + url = 'https://www.python.org/ftp/python/3.7.4/{}'.format(filename) + recipe._url = url + recipe.ctx = Context() + with ( + patch_logger_debug()) as m_debug, ( + mock.patch.object(Recipe, 'download_file')) as m_download_file, ( + mock.patch('pythonforandroid.recipe.sh.touch')) as m_touch, ( + tempfile.TemporaryDirectory()) as temp_dir: + recipe.ctx.setup_dirs(temp_dir) + recipe.download() + assert m_download_file.call_args_list == [mock.call(url, filename)] + assert m_debug.call_args_list == [ + mock.call( + 'Downloading test_recipe from ' + 'https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz')] + assert m_touch.call_count == 1 diff --git a/tox.ini b/tox.ini index 63393eab3f..bd1ae28df4 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ deps = pytest virtualenv py3: coveralls + backports.tempfile # makes it possible to override pytest args, e.g. # tox -- tests/test_graph.py commands = pytest {posargs:tests/} From 542684ed20174e458ccc35fbc07cecfa9bc8d2ff Mon Sep 17 00:00:00 2001 From: Jonas Thiem Date: Sun, 28 Jul 2019 17:29:00 +0200 Subject: [PATCH 38/41] Call Cython via `python -m Cython` rather than system-wide binary - call Cython via `python -m Cython` to avoid picking one not matching the current python version (which can happen if just calling `Cython`) - make sure Cython is present in Dockerfile.py3 and Dockerfile.py2 - make sure python 2 is available in Dockerfile.py3 - this should fix #1885 --- Dockerfile.py2 | 3 +-- Dockerfile.py3 | 7 ++++--- pythonforandroid/build.py | 18 ++++++++---------- pythonforandroid/recipe.py | 7 +++++-- tests/test_toolchain.py | 3 --- 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/Dockerfile.py2 b/Dockerfile.py2 index 5b9f2e2744..5c9202c6df 100644 --- a/Dockerfile.py2 +++ b/Dockerfile.py2 @@ -122,8 +122,6 @@ RUN usermod -append --groups sudo ${USER} RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers -RUN pip install --upgrade cython==0.28.6 - WORKDIR ${WORK_DIR} COPY --chown=user:user . ${WORK_DIR} RUN chown --recursive ${USER} ${ANDROID_SDK_HOME} @@ -132,4 +130,5 @@ USER ${USER} # install python-for-android from current branch RUN virtualenv --python=python venv \ && . venv/bin/activate \ + && pip install --upgrade cython==0.28.6 \ && pip install -e . diff --git a/Dockerfile.py3 b/Dockerfile.py3 index e441417112..5a3b40a878 100644 --- a/Dockerfile.py3 +++ b/Dockerfile.py3 @@ -94,7 +94,7 @@ ENV WORK_DIR="${HOME_DIR}" \ # install system dependencies RUN ${RETRY} apt -y install -qq --no-install-recommends \ python3 virtualenv python3-pip python3-venv \ - wget lbzip2 patch sudo \ + wget lbzip2 patch sudo python python-pip \ && apt -y autoremove # build dependencies @@ -122,8 +122,8 @@ RUN useradd --create-home --shell /bin/bash ${USER} RUN usermod -append --groups sudo ${USER} RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers - -RUN pip3 install --upgrade cython==0.28.6 +# install cython for python 2 (for python 3 it's inside the venv) +RUN pip2 install --upgrade Cython==0.28.6 WORKDIR ${WORK_DIR} COPY --chown=user:user . ${WORK_DIR} @@ -133,4 +133,5 @@ USER ${USER} # install python-for-android from current branch RUN virtualenv --python=python3 venv \ && . venv/bin/activate \ + && pip3 install --upgrade Cython==0.28.6 \ && pip3 install -e . diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index a5845358bf..16cc459b4c 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -27,14 +27,6 @@ RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API) -def get_cython_path(): - for cython_fn in ("cython", "cython3", "cython2", "cython-2.7"): - cython = sh.which(cython_fn) - if cython: - return cython - raise BuildInterruptingException('No cython binary found.') - - def get_ndk_platform_dir(ndk_dir, ndk_api, arch): ndk_platform_dir_exists = True platform_dir = arch.platform_dir @@ -111,7 +103,6 @@ class Context(object): use_setup_py = False ccache = None # whether to use ccache - cython = None # the cython interpreter name ndk_platform = None # the ndk platform directory @@ -374,7 +365,14 @@ def prepare_build_environment(self, if not self.ccache: info('ccache is missing, the build will not be optimized in the ' 'future.') - self.cython = get_cython_path() + try: + subprocess.check_output([ + "python3", "-m", "cython", "--help", + ]) + except subprocess.CalledProcessError: + warning('Cython for python3 missing. If you are building for ' + ' a python 3 target (which is the default)' + ' then THINGS WILL BREAK.') # This would need to be changed if supporting multiarch APKs arch = self.archs[0] diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 4de888306a..cc6dae6c31 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -1024,8 +1024,11 @@ def cythonize_file(self, env, build_dir, filename): del cyenv['PYTHONPATH'] if 'PYTHONNOUSERSITE' in cyenv: cyenv.pop('PYTHONNOUSERSITE') - cython_command = sh.Command(self.ctx.cython) - shprint(cython_command, filename, *self.cython_args, _env=cyenv) + python_command = sh.Command("python{}".format( + self.ctx.python_recipe.major_minor_version_string.split(".")[0] + )) + shprint(python_command, "-m", "Cython.Build.Cythonize", + filename, *self.cython_args, _env=cyenv) def cythonize_build(self, env, build_dir="."): if not self.cythonize: diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py index a4b68c86ee..11e35ff989 100644 --- a/tests/test_toolchain.py +++ b/tests/test_toolchain.py @@ -68,8 +68,6 @@ def test_create(self): ) as m_get_toolchain_versions, mock.patch( 'pythonforandroid.build.get_ndk_platform_dir' ) as m_get_ndk_platform_dir, mock.patch( - 'pythonforandroid.build.get_cython_path' - ) as m_get_cython_path, mock.patch( 'pythonforandroid.toolchain.build_recipes' ) as m_build_recipes, mock.patch( 'pythonforandroid.bootstraps.service_only.' @@ -84,7 +82,6 @@ def test_create(self): mock.call('/tmp/android-sdk')] assert m_get_toolchain_versions.call_args_list == [ mock.call('/tmp/android-ndk', mock.ANY)] - assert m_get_cython_path.call_args_list == [mock.call()] build_order = [ 'hostpython3', 'libffi', 'openssl', 'sqlite3', 'python3', 'genericndkbuild', 'setuptools', 'six', 'pyjnius', 'android', From 40b1164e744702e99bc535be4b11d01168e0d250 Mon Sep 17 00:00:00 2001 From: Alexander Taylor Date: Sun, 4 Aug 2019 17:42:41 +0100 Subject: [PATCH 39/41] Drop Python 2 support (#1918) * Added build failure if run under Python 2 * Made Python 2 check more robust * Style fixes * Changed minimum python3 minor version to 3.4 * Added a deprecation warning log to the Python 2 recipe during build * Updated tox.ini to run py2 tests only on android module * Added tests for Python version checking * Added test for Python 2 running long enough to exit nicely * Pep8 and code style improvements * Added descriptions for flake8 exceptions * Hardcoded python2 tests in tox.ini * Removed E127 and E129 global disable --- pythonforandroid/bdistapk.py | 2 +- pythonforandroid/entrypoints.py | 20 +++++++++ pythonforandroid/logger.py | 2 + pythonforandroid/recipes/python2/__init__.py | 7 ++- pythonforandroid/recommendations.py | 36 +++++++++++++++ pythonforandroid/toolchain.py | 11 ++--- pythonforandroid/util.py | 9 ++-- setup.py | 8 ++-- tests/test_androidmodule_ctypes_finder.py | 10 ++++- tests/test_entrypoints_python2.py | 38 ++++++++++++++++ tests/test_recommendations.py | 47 +++++++++++++++++--- tox.ini | 23 +++++++--- 12 files changed, 179 insertions(+), 34 deletions(-) create mode 100644 pythonforandroid/entrypoints.py create mode 100644 tests/test_entrypoints_python2.py diff --git a/pythonforandroid/bdistapk.py b/pythonforandroid/bdistapk.py index 6c156ebf39..ea1b78ca2f 100644 --- a/pythonforandroid/bdistapk.py +++ b/pythonforandroid/bdistapk.py @@ -75,7 +75,7 @@ def finalize_options(self): def run(self): self.prepare_build_dir() - from pythonforandroid.toolchain import main + from pythonforandroid.entrypoints import main sys.argv[1] = 'apk' main() diff --git a/pythonforandroid/entrypoints.py b/pythonforandroid/entrypoints.py new file mode 100644 index 0000000000..1ba6a2601f --- /dev/null +++ b/pythonforandroid/entrypoints.py @@ -0,0 +1,20 @@ +from pythonforandroid.recommendations import check_python_version +from pythonforandroid.util import BuildInterruptingException, handle_build_exception + + +def main(): + """ + Main entrypoint for running python-for-android as a script. + """ + + try: + # Check the Python version before importing anything heavier than + # the util functions. This lets us provide a nice message about + # incompatibility rather than having the interpreter crash if it + # reaches unsupported syntax from a newer Python version. + check_python_version() + + from pythonforandroid.toolchain import ToolchainCL + ToolchainCL() + except BuildInterruptingException as exc: + handle_build_exception(exc) diff --git a/pythonforandroid/logger.py b/pythonforandroid/logger.py index 4aba39fcab..77cb9da323 100644 --- a/pythonforandroid/logger.py +++ b/pythonforandroid/logger.py @@ -6,6 +6,8 @@ from math import log10 from collections import defaultdict from colorama import Style as Colo_Style, Fore as Colo_Fore + +# six import left for Python 2 compatibility during initial Python version check import six # This codecs change fixes a bug with log output, but crashes under python3 diff --git a/pythonforandroid/recipes/python2/__init__.py b/pythonforandroid/recipes/python2/__init__.py index 78e666fa2a..e99697f215 100644 --- a/pythonforandroid/recipes/python2/__init__.py +++ b/pythonforandroid/recipes/python2/__init__.py @@ -1,7 +1,7 @@ from os.path import join, exists from pythonforandroid.recipe import Recipe from pythonforandroid.python import GuestPythonRecipe -from pythonforandroid.logger import shprint +from pythonforandroid.logger import shprint, warning import sh @@ -57,6 +57,11 @@ def prebuild_arch(self, arch): self.apply_patch(join('patches', 'enable-openssl.patch'), arch.arch) shprint(sh.touch, patch_mark) + def build_arch(self, arch): + warning('DEPRECATION: Support for the Python 2 recipe will be ' + 'removed in 2020, please upgrade to Python 3.') + super().build_arch(arch) + def set_libs_flags(self, env, arch): env = super(Python2Recipe, self).set_libs_flags(env, arch) if 'libffi' in self.ctx.recipe_build_order: diff --git a/pythonforandroid/recommendations.py b/pythonforandroid/recommendations.py index 98e0a33e67..6cf18eceb9 100644 --- a/pythonforandroid/recommendations.py +++ b/pythonforandroid/recommendations.py @@ -1,7 +1,9 @@ """Simple functions for checking dependency versions.""" +import sys from distutils.version import LooseVersion from os.path import join + from pythonforandroid.logger import info, warning from pythonforandroid.util import BuildInterruptingException @@ -182,3 +184,37 @@ def check_ndk_api(ndk_api, android_api): if ndk_api < MIN_NDK_API: warning(OLD_NDK_API_MESSAGE) + + +MIN_PYTHON_MAJOR_VERSION = 3 +MIN_PYTHON_MINOR_VERSION = 4 +MIN_PYTHON_VERSION = LooseVersion('{major}.{minor}'.format(major=MIN_PYTHON_MAJOR_VERSION, + minor=MIN_PYTHON_MINOR_VERSION)) +PY2_ERROR_TEXT = ( + 'python-for-android no longer supports running under Python 2. Either upgrade to ' + 'Python {min_version} or higher (recommended), or revert to python-for-android 2019.07.08. ' + 'Note that you *can* still target Python 2 on Android by including python2 in your ' + 'requirements.').format( + min_version=MIN_PYTHON_VERSION) + +PY_VERSION_ERROR_TEXT = ( + 'Your Python version {user_major}.{user_minor} is not supported by python-for-android, ' + 'please upgrade to {min_version} or higher.' + ).format( + user_major=sys.version_info.major, + user_minor=sys.version_info.minor, + min_version=MIN_PYTHON_VERSION) + + +def check_python_version(): + # Python 2 special cased because it's a major transition. In the + # future the major or minor versions can increment more quietly. + if sys.version_info.major == 2: + raise BuildInterruptingException(PY2_ERROR_TEXT) + + if ( + sys.version_info.major < MIN_PYTHON_MAJOR_VERSION or + sys.version_info.minor < MIN_PYTHON_MINOR_VERSION + ): + + raise BuildInterruptingException(PY_VERSION_ERROR_TEXT) diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index d7883a2690..a9c2c71a61 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -12,7 +12,8 @@ from pythonforandroid.pythonpackage import get_dep_names_of_package from pythonforandroid.recommendations import ( RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API) -from pythonforandroid.util import BuildInterruptingException, handle_build_exception +from pythonforandroid.util import BuildInterruptingException +from pythonforandroid.entrypoints import main def check_python_dependencies(): @@ -568,6 +569,7 @@ def add_parser(subparsers, *args, **kwargs): args, unknown = parser.parse_known_args(sys.argv[1:]) args.unknown_args = unknown + if hasattr(args, "private") and args.private is not None: # Pass this value on to the internal bootstrap build.py: args.unknown_args += ["--private", args.private] @@ -1187,12 +1189,5 @@ def build_status(self, _args): print(recipe_str) -def main(): - try: - ToolchainCL() - except BuildInterruptingException as exc: - handle_build_exception(exc) - - if __name__ == "__main__": main() diff --git a/pythonforandroid/util.py b/pythonforandroid/util.py index ba392049b6..839858cb1e 100644 --- a/pythonforandroid/util.py +++ b/pythonforandroid/util.py @@ -3,18 +3,17 @@ from os import getcwd, chdir, makedirs, walk, uname import sh import shutil -import sys from fnmatch import fnmatch from tempfile import mkdtemp -try: + +# This Python version workaround left for compatibility during initial version check +try: # Python 3 from urllib.request import FancyURLopener -except ImportError: +except ImportError: # Python 2 from urllib import FancyURLopener from pythonforandroid.logger import (logger, Err_Fore, error, info) -IS_PY3 = sys.version_info[0] >= 3 - class WgetDownloader(FancyURLopener): version = ('Wget/1.17.1') diff --git a/setup.py b/setup.py index 64ab2a0d32..52dc745763 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ data_files = [] + # must be a single statement since buildozer is currently parsing it, refs: # https://github.com/kivy/buildozer/issues/722 install_reqs = [ @@ -94,15 +95,15 @@ def recursively_include(results, directory, patterns): install_requires=install_reqs, entry_points={ 'console_scripts': [ - 'python-for-android = pythonforandroid.toolchain:main', - 'p4a = pythonforandroid.toolchain:main', + 'python-for-android = pythonforandroid.entrypoints:main', + 'p4a = pythonforandroid.entrypoints:main', ], 'distutils.commands': [ 'apk = pythonforandroid.bdistapk:BdistAPK', ], }, classifiers = [ - 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Operating System :: Microsoft :: Windows', @@ -111,7 +112,6 @@ def recursively_include(results, directory, patterns): 'Operating System :: MacOS :: MacOS X', 'Operating System :: Android', 'Programming Language :: C', - 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', 'Topic :: Software Development', 'Topic :: Utilities', diff --git a/tests/test_androidmodule_ctypes_finder.py b/tests/test_androidmodule_ctypes_finder.py index 7d4526888d..553287d12a 100644 --- a/tests/test_androidmodule_ctypes_finder.py +++ b/tests/test_androidmodule_ctypes_finder.py @@ -1,6 +1,12 @@ -import mock -from mock import MagicMock +# This test is still expected to support Python 2, as it tests +# on-Android functionality that we still maintain +try: # Python 3+ + from unittest import mock + from unittest.mock import MagicMock +except ImportError: # Python 2 + import mock + from mock import MagicMock import os import shutil import sys diff --git a/tests/test_entrypoints_python2.py b/tests/test_entrypoints_python2.py new file mode 100644 index 0000000000..0a2f6ebb4f --- /dev/null +++ b/tests/test_entrypoints_python2.py @@ -0,0 +1,38 @@ + +# This test is a special case that we expect to run under Python 2, so +# include the necessary compatibility imports: +try: # Python 3 + from unittest import mock +except ImportError: # Python 2 + import mock + +from pythonforandroid.recommendations import PY2_ERROR_TEXT +from pythonforandroid import entrypoints + + +def test_main_python2(): + """Test that running under Python 2 leads to the build failing, while + running under a suitable version works fine. + + Note that this test must be run *using* Python 2 to truly test + that p4a can reach the Python version check before importing some + Python-3-only syntax and crashing. + """ + + # Under Python 2, we should get a normal control flow exception + # that is handled properly, not any other type of crash + handle_exception_path = 'pythonforandroid.entrypoints.handle_build_exception' + with mock.patch('sys.version_info') as fake_version_info, \ + mock.patch(handle_exception_path) as handle_build_exception: # noqa: E127 + + fake_version_info.major = 2 + fake_version_info.minor = 7 + + def check_python2_exception(exc): + """Check that the exception message is Python 2 specific.""" + assert exc.message == PY2_ERROR_TEXT + handle_build_exception.side_effect = check_python2_exception + + entrypoints.main() + + handle_build_exception.assert_called_once() diff --git a/tests/test_recommendations.py b/tests/test_recommendations.py index 2f3cc18db2..649fb3b1f9 100644 --- a/tests/test_recommendations.py +++ b/tests/test_recommendations.py @@ -2,17 +2,13 @@ from os.path import join from sys import version as py_version -try: - from unittest import mock -except ImportError: - # `Python 2` or lower than `Python 3.3` does not - # have the `unittest.mock` module built-in - import mock +from unittest import mock from pythonforandroid.recommendations import ( check_ndk_api, check_ndk_version, check_target_api, read_ndk_version, + check_python_version, MAX_NDK_VERSION, RECOMMENDED_NDK_VERSION, RECOMMENDED_TARGET_API, @@ -33,7 +29,12 @@ OLD_NDK_API_MESSAGE, NEW_NDK_MESSAGE, OLD_API_MESSAGE, + MIN_PYTHON_MAJOR_VERSION, + MIN_PYTHON_MINOR_VERSION, + PY2_ERROR_TEXT, + PY_VERSION_ERROR_TEXT, ) + from pythonforandroid.util import BuildInterruptingException running_in_py2 = int(py_version[0]) < 3 @@ -202,3 +203,37 @@ def test_check_ndk_api_warning_old_ndk(self): ) ], ) + + def test_check_python_version(self): + """With any version info lower than the minimum, we should get a + BuildInterruptingException with an appropriate message. + """ + with mock.patch('sys.version_info') as fake_version_info: + + # Major version is Python 2 => exception + fake_version_info.major = MIN_PYTHON_MAJOR_VERSION - 1 + fake_version_info.minor = MIN_PYTHON_MINOR_VERSION + with self.assertRaises(BuildInterruptingException) as context: + check_python_version() + assert context.exception.message == PY2_ERROR_TEXT + + # Major version too low => exception + # Using a float valued major version just to test the logic and avoid + # clashing with the Python 2 check + fake_version_info.major = MIN_PYTHON_MAJOR_VERSION - 0.1 + fake_version_info.minor = MIN_PYTHON_MINOR_VERSION + with self.assertRaises(BuildInterruptingException) as context: + check_python_version() + assert context.exception.message == PY_VERSION_ERROR_TEXT + + # Minor version too low => exception + fake_version_info.major = MIN_PYTHON_MAJOR_VERSION + fake_version_info.minor = MIN_PYTHON_MINOR_VERSION - 1 + with self.assertRaises(BuildInterruptingException) as context: + check_python_version() + assert context.exception.message == PY_VERSION_ERROR_TEXT + + # Version high enough => nothing interesting happens + fake_version_info.major = MIN_PYTHON_MAJOR_VERSION + fake_version_info.minor = MIN_PYTHON_MINOR_VERSION + check_python_version() diff --git a/tox.ini b/tox.ini index bd1ae28df4..2ca84ae803 100644 --- a/tox.ini +++ b/tox.ini @@ -9,13 +9,19 @@ deps = virtualenv py3: coveralls backports.tempfile -# makes it possible to override pytest args, e.g. -# tox -- tests/test_graph.py +# posargs will be replaced by the tox args, so you can override pytest +# args e.g. `tox -- tests/test_graph.py` commands = pytest {posargs:tests/} passenv = TRAVIS TRAVIS_* setenv = PYTHONPATH={toxinidir} +[testenv:py27] +# Note that the set of tests is not posargs-configurable here: we only +# check a minimal set of Python 2 tests for the remaining Python 2 +# functionality that we support +commands = pytest tests/test_androidmodule_ctypes_finder.py tests/test_entrypoints_python2.py + [testenv:py3] # for py3 env we will get code coverage commands = @@ -28,8 +34,11 @@ commands = flake8 pythonforandroid/ tests/ ci/ [flake8] ignore = - E123, E124, E126, - E226, - E402, E501, - W503, - W504 + E123, # Closing bracket does not match indentation of opening bracket's line + E124, # Closing bracket does not match visual indentation + E126, # Continuation line over-indented for hanging indent + E226, # Missing whitespace around arithmetic operator + E402, # Module level import not at top of file + E501, # Line too long (82 > 79 characters) + W503, # Line break occurred before a binary operator + W504 # Line break occurred after a binary operator From 5abb0ec9715869c4b6f25eb0654f53e722ca6759 Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Mon, 5 Aug 2019 23:27:42 +0200 Subject: [PATCH 40/41] Unit tests Recipe.download_file() partly (#1952) Follow up of #1946. Increases coverage by testing part of `Recipe.download_file()` handling https schema. Next up would be to handle more schemas in the tests. Also addressed @opacam comments from previous pull requests. --- pythonforandroid/toolchain.py | 11 +++---- tests/test_recipe.py | 62 ++++++++++++++++++++++++++++++++--- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index a9c2c71a61..cb15ca0392 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -743,12 +743,11 @@ def _read_configuration(): def recipes(self, args): """ Prints recipes basic info, e.g. - ``` - python3 3.7.1 - depends: ['hostpython3', 'sqlite3', 'openssl', 'libffi'] - conflicts: ['python2'] - optional depends: ['sqlite3', 'libffi', 'openssl'] - ``` + .. code-block:: bash + python3 3.7.1 + depends: ['hostpython3', 'sqlite3', 'openssl', 'libffi'] + conflicts: ['python2'] + optional depends: ['sqlite3', 'libffi', 'openssl'] """ ctx = self.ctx if args.compact: diff --git a/tests/test_recipe.py b/tests/test_recipe.py index eefddee8aa..9685b15d68 100644 --- a/tests/test_recipe.py +++ b/tests/test_recipe.py @@ -1,4 +1,5 @@ import os +import pytest import types import unittest import warnings @@ -20,6 +21,10 @@ def patch_logger_debug(): return patch_logger('debug') +def patch_urlretrieve(): + return mock.patch('pythonforandroid.recipe.urlretrieve') + + class DummyRecipe(Recipe): pass @@ -94,21 +99,34 @@ def test_download_if_necessary(self): recipe.download_if_necessary() assert m_download.call_args_list == [] - def test_download(self): + def test_download_url_not_set(self): """ - Verifies the actual download gets triggered when the URL is set. + Verifies that no download happens when URL is not set. """ - # test with no URL set recipe = DummyRecipe() with patch_logger_info() as m_info: recipe.download() assert m_info.call_args_list == [ mock.call('Skipping test_recipe download as no URL is set')] - # when the URL is set `Recipe.download_file()` should be called + + @staticmethod + def get_dummy_python_recipe_for_download_tests(): + """ + Helper method for creating a test recipe used in download tests. + """ + recipe = DummyRecipe() filename = 'Python-3.7.4.tgz' url = 'https://www.python.org/ftp/python/3.7.4/{}'.format(filename) recipe._url = url recipe.ctx = Context() + return recipe, filename + + def test_download_url_is_set(self): + """ + Verifies the actual download gets triggered when the URL is set. + """ + recipe, filename = self.get_dummy_python_recipe_for_download_tests() + url = recipe.url with ( patch_logger_debug()) as m_debug, ( mock.patch.object(Recipe, 'download_file')) as m_download_file, ( @@ -122,3 +140,39 @@ def test_download(self): 'Downloading test_recipe from ' 'https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz')] assert m_touch.call_count == 1 + + def test_download_file_scheme_https(self): + """ + Verifies `urlretrieve()` is being called on https downloads. + """ + recipe, filename = self.get_dummy_python_recipe_for_download_tests() + url = recipe.url + with ( + patch_urlretrieve()) as m_urlretrieve, ( + tempfile.TemporaryDirectory()) as temp_dir: + recipe.ctx.setup_dirs(temp_dir) + assert recipe.download_file(url, filename) == filename + assert m_urlretrieve.call_args_list == [ + mock.call(url, filename, mock.ANY) + ] + + def test_download_file_scheme_https_oserror(self): + """ + Checks `urlretrieve()` is being retried on `OSError`. + After a number of retries the exception is re-reaised. + """ + recipe, filename = self.get_dummy_python_recipe_for_download_tests() + url = recipe.url + with ( + patch_urlretrieve()) as m_urlretrieve, ( + mock.patch('pythonforandroid.recipe.time.sleep')) as m_sleep, ( + pytest.raises(OSError)), ( + tempfile.TemporaryDirectory()) as temp_dir: + recipe.ctx.setup_dirs(temp_dir) + m_urlretrieve.side_effect = OSError + assert recipe.download_file(url, filename) == filename + retry = 5 + expected_call_args_list = [mock.call(url, filename, mock.ANY)] * retry + assert m_urlretrieve.call_args_list == expected_call_args_list + expected_call_args_list = [mock.call(1)] * (retry - 1) + assert m_sleep.call_args_list == expected_call_args_list From b1517a3d6353c579bde7cf12543567e28f501faa Mon Sep 17 00:00:00 2001 From: Alexander Taylor Date: Fri, 9 Aug 2019 21:17:26 +0100 Subject: [PATCH 41/41] Updated version number to 2019.08.09 --- pythonforandroid/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonforandroid/__init__.py b/pythonforandroid/__init__.py index 2272e310e3..ee4d0cdbb6 100644 --- a/pythonforandroid/__init__.py +++ b/pythonforandroid/__init__.py @@ -1 +1 @@ -__version__ = '2019.07.08.1.dev0' +__version__ = '2019.08.09'