kivy · AndreMiras · Oct 26, 2019 · Oct 28, 2019
diff --git a/.env b/.env
@@ -0,0 +1,9 @@
+# used by coveralls.io, refs:
+# https://coveralls-python.readthedocs.io/en/latest/usage/tox.html#travisci
+CI
+TRAVIS
+TRAVIS_BRANCH
+TRAVIS_JOB_ID
+TRAVIS_PULL_REQUEST
+# used for running UI tests
+DISPLAY
diff --git a/.travis.yml b/.travis.yml
@@ -1,12 +1,10 @@
 sudo: required
 
-dist: xenial  # needed for more recent python 3 and python3-venv
-
 language: generic
 
 stages:
-  - lint
-  - test
+  - pre checks
+  - build testapps
 
 services:
   - docker
@@ -18,8 +16,8 @@ before_install:
 
 jobs:
   include:
-    - &linting
-      stage: lint
+    - &prechecks
+      stage: pre checks
       language: python
       python: 3.7
       before_script:
@@ -36,66 +34,43 @@ jobs:
         - pip3.7 install pyOpenSSL
         - pip3.7 install coveralls
       script:
-        # we want to fail fast on tox errors without having to `docker build` first
+        # ignores test_pythonpackage.py since it runs for too long
         - tox -- tests/ --ignore tests/test_pythonpackage.py
-        # (we ignore test_pythonpackage.py since these run way too long!!
-        #  test_pythonpackage_basic.py will still be run.)
       name: "Tox Pep8"
       env: TOXENV=pep8
-    - <<: *linting
+    - <<: *prechecks
       name: "Tox Python 2"
       env: TOXENV=py27
-    - <<: *linting
+    - <<: *prechecks
       name: "Tox Python 3 & Coverage"
       env: TOXENV=py3
       after_success:
         - coveralls
-
-    - &testing
-      stage: test
+    # the docker build stage runs almost as fast as `tox` unit tests, so we can run in parallel
+    - name: "Docker build"
       before_script:
-        # build docker image
-        - docker build --tag=p4a --file Dockerfile.py3 .
-        # Run a background process to make sure that travis will not kill our tests in
-        # case that the travis log doesn't produce any output for more than 10 minutes
-        - while sleep 540; do echo "==== Still running (travis, don't kill me) ===="; done &
+        - make docker/pull TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST TRAVIS_BRANCH=$TRAVIS_BRANCH
       script:
-        - >
-          docker run
-          -e CI
-          -e TRAVIS_JOB_ID
-          -e TRAVIS_BRANCH
-          -e ANDROID_SDK_HOME="/home/user/.android/android-sdk"
-          -e ANDROID_NDK_HOME="/home/user/.android/android-ndk"
-          p4a /bin/sh -c "$COMMAND"
-      after_script:
-        # kill the background process started before run docker
-        - kill %1
+        - make docker/build TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST TRAVIS_BRANCH=$TRAVIS_BRANCH
+      after_success:
+        - make docker/push DOCKER_USERNAME=$DOCKER_USERNAME DOCKER_PASSWORD=$DOCKER_PASSWORD
+    - &testapps
       name: Python 3 arm64-v8a
-      # overrides requirements to skip `peewee` pure python module, see:
-      # https://github.com/kivy/python-for-android/issues/1263#issuecomment-390421054
-      env:
-        COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools --arch=arm64-v8a'
-    - <<: *testing
+      stage: build testapps
+      before_script: make docker/pull TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST TRAVIS_BRANCH=$TRAVIS_BRANCH
+      script: make docker/run/make/testapps/python3/arm64-v8a TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST TRAVIS_BRANCH=$TRAVIS_BRANCH
+    - <<: *testapps
       name: Python 3 armeabi-v7a
       os: osx
       osx_image: xcode11  # since xcode1.3, python3 is the default interpreter
       before_script:
         # installs java 1.8, android's SDK/NDK and p4a
         - make -f ci/makefiles/osx.mk
         - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
-        # Run a background process (like we do with linux tests)
-        - while sleep 540; do echo "==== Still running (travis, don't kill me) ===="; done &
-      script:
-        - >
-          cd testapps && python3 setup_testapp_python3_sqlite_openssl.py apk
-          --sdk-dir $HOME/.android/android-sdk
-          --ndk-dir $HOME/.android/android-ndk
-          --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools
-          --arch=armeabi-v7a
-    - <<: *testing
+      script: make testapps/python3/armeabi-v7a PYTHON_WITH_VERSION=python3
+    - <<: *testapps
       name: Python 2 armeabi-v7a (with numpy)
-      env: COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools,numpy'
-    - <<: *testing
+      script: make docker/run/make/testapps/python2/armeabi-v7a TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST TRAVIS_BRANCH=$TRAVIS_BRANCH
+    - <<: *testapps
       name: Rebuild updated recipes
-      env: COMMAND='. venv/bin/activate && ./ci/rebuild_updated_recipes.py'
+      script: make docker/run/make/rebuild_updated_recipes TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST TRAVIS_BRANCH=$TRAVIS_BRANCH
diff --git a/Dockerfile.py3 b/Dockerfile.py3
@@ -25,9 +25,12 @@ ENV LANG="en_US.UTF-8" \
     LANGUAGE="en_US.UTF-8" \
     LC_ALL="en_US.UTF-8"
 
-RUN apt -y update -qq \
-    && apt -y install -qq --no-install-recommends curl unzip ca-certificates \
-    && apt -y autoremove
+RUN apt -y update -qq > /dev/null && apt -y install -qq --no-install-recommends \
+	ca-certificates \
+    curl \
+    && apt -y autoremove \
+    && apt -y clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # retry helper script, refs:
 # https://github.com/kivy/python-for-android/issues/1306
@@ -37,37 +40,52 @@ RUN curl https://raw.githubusercontent.com/kadwanev/retry/1.0.1/retry \
 
 ENV USER="user"
 ENV HOME_DIR="/home/${USER}"
-ENV ANDROID_HOME="${HOME_DIR}/.android"
-ENV WORK_DIR="${HOME_DIR}" \
-    PATH="${HOME_DIR}/.local/bin:${PATH}"
+ENV WORK_DIR="${HOME_DIR}/app" \
+    PATH="${HOME_DIR}/.local/bin:${PATH}" \
+    ANDROID_HOME="${HOME_DIR}/.android" \
+    JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64"
 
-# install system dependencies
-RUN ${RETRY} apt -y install -qq --no-install-recommends \
-        python3 virtualenv python3-pip python3-venv \
-        wget lbzip2 patch sudo python python-pip \
-    && apt -y autoremove
 
-# build dependencies
-# https://buildozer.readthedocs.io/en/latest/installation.html#android-on-ubuntu-16-04-64bit
+# install system dependencies
 RUN dpkg --add-architecture i386 \
-    && ${RETRY} apt -y update -qq \
+    && ${RETRY} apt -y update -qq > /dev/null \
     && ${RETRY} apt -y install -qq --no-install-recommends \
-        build-essential ccache git python3 python3-dev \
-        libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \
-        libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 \
-        zip zlib1g-dev zlib1g:i386 \
-    && apt -y autoremove
-
-# specific recipes dependencies (e.g. libffi requires autoreconf binary)
-RUN ${RETRY} apt -y install -qq --no-install-recommends \
-        libffi-dev autoconf automake cmake gettext libltdl-dev libtool pkg-config \
+    autoconf \
+    automake \
+    build-essential \
+    ccache \
+    cmake \
+    gettext \
+    git \
+    lbzip2 \
+    libffi-dev \
+    libgtk2.0-0:i386 \
+    libidn11:i386 \
+    libltdl-dev \
+    libncurses5:i386 \
+    libpangox-1.0-0:i386 \
+    libpangoxft-1.0-0:i386 \
+    libstdc++6:i386 \
+    libtool \
+    openjdk-8-jdk \
+    patch \
+    pkg-config \
+    python \
+    python-pip \
+    python3 \
+    python3-dev \
+    python3-pip \
+    python3-venv \
+    sudo \
+    unzip \
+    virtualenv \
+    wget \
+    zip \
+    zlib1g-dev \
+    zlib1g:i386 \
     && apt -y autoremove \
-    && apt -y clean
-
-# Install Java and set JAVA_HOME (to accept android's SDK licenses)
-RUN ${RETRY} apt -y install -qq --no-install-recommends openjdk-8-jdk \
-    && apt -y autoremove && apt -y clean
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+    && apt -y clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # prepare non root env
 RUN useradd --create-home --shell /bin/bash ${USER}
@@ -77,18 +95,23 @@ RUN usermod -append --groups sudo ${USER}
 RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
 
 # install cython for python 2 (for python 3 it's inside the venv)
-RUN pip2 install --upgrade Cython==0.28.6
+RUN pip2 install --upgrade Cython==0.28.6 \
+    && rm -rf ~/.cache/
 
 WORKDIR ${WORK_DIR}
-COPY --chown=user:user . ${WORK_DIR}
-RUN mkdir ${ANDROID_HOME} && chown --recursive ${USER} ${ANDROID_HOME}
+RUN mkdir ${ANDROID_HOME} && chown --recursive ${USER} ${HOME_DIR} ${ANDROID_HOME}
 USER ${USER}
 
 # Download and install android's NDK/SDK
-RUN make -f ci/makefiles/android.mk target_os=linux
+COPY ci/makefiles/android.mk /tmp/android.mk
+RUN make --file /tmp/android.mk target_os=linux \
+    && sudo rm /tmp/android.mk
 
 # install python-for-android from current branch
-RUN virtualenv --python=python3 venv \
-    && . venv/bin/activate \
-    && pip3 install --upgrade Cython==0.28.6 \
-    && pip3 install -e .
+COPY --chown=user:user Makefile README.md setup.py pythonforandroid/__init__.py ${WORK_DIR}/
+RUN mkdir pythonforandroid \
+    && mv __init__.py pythonforandroid/ \
+    && make virtualenv \
+    && rm -rf ~/.cache/
+
+COPY --chown=user:user . ${WORK_DIR}
diff --git a/Makefile b/Makefile
@@ -0,0 +1,90 @@
+VIRTUAL_ENV ?= venv
+PIP=$(VIRTUAL_ENV)/bin/pip
+TOX=`which tox`
+ACTIVATE=$(VIRTUAL_ENV)/bin/activate
+PYTHON=$(VIRTUAL_ENV)/bin/python
+FLAKE8=$(VIRTUAL_ENV)/bin/flake8
+PYTEST=$(VIRTUAL_ENV)/bin/pytest
+SOURCES=src/ tests/
+PYTHON_MAJOR_VERSION=3
+PYTHON_MINOR_VERSION=6
+PYTHON_VERSION=$(PYTHON_MAJOR_VERSION).$(PYTHON_MINOR_VERSION)
+PYTHON_MAJOR_MINOR=$(PYTHON_MAJOR_VERSION)$(PYTHON_MINOR_VERSION)
+PYTHON_WITH_VERSION=python$(PYTHON_VERSION)
+ANDROID_SDK_HOME ?= $(HOME)/.android/android-sdk
+ANDROID_NDK_HOME ?= $(HOME)/.android/android-ndk
+DOCKER_IMAGE=kivy/python-for-android
+# tag to latest only on merge to develop
+DOCKER_TAG=latest
+ifdef TRAVIS_PULL_REQUEST
+ifneq ($(TRAVIS_PULL_REQUEST), false)
+	DOCKER_TAG=pr-$(TRAVIS_PULL_REQUEST)
+else ifneq ($(TRAVIS_BRANCH), develop)
+	DOCKER_TAG=branch-$(subst /,-,$(TRAVIS_BRANCH)) # slash is an invalid docker tag character
+endif
+endif
+
+
+all: virtualenv
+
+$(VIRTUAL_ENV):
+	virtualenv --python=$(PYTHON_WITH_VERSION) $(VIRTUAL_ENV)
+	$(PIP) install Cython==0.28.6
+	$(PIP) install -e .
+
+virtualenv: $(VIRTUAL_ENV)
+
+# ignores test_pythonpackage.py since it runs for too long
+test:
+	$(TOX) -- tests/ --ignore tests/test_pythonpackage.py
+	@if test -n "$$CI"; then .tox/py$(PYTHON_MAJOR_MINOR)/bin/coveralls; fi; \
+
+rebuild_updated_recipes: virtualenv
+	$(PYTHON) ci/rebuild_updated_recipes.py
+
+testapps/python2/armeabi-v7a: virtualenv
+	. $(ACTIVATE) && cd testapps/ && \
+    python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
+    --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools,numpy
+
+testapps/python3/arm64-v8a: virtualenv
+	. $(ACTIVATE) && cd testapps/ && \
+    python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
+    --arch=arm64-v8a
+
+testapps/python3/armeabi-v7a: virtualenv
+	. $(ACTIVATE) && cd testapps/ && \
+    python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
+    --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools \
+    --arch=armeabi-v7a
+
+clean:
+	find . -type d -name "__pycache__" -exec rm -r {} +
+	find . -type d -name "*.egg-info" -exec rm -r {} +
+
+clean/all: clean
+	rm -rf $(VIRTUAL_ENV) .tox/
+
+docker/pull:
+	docker pull $(DOCKER_IMAGE):$(DOCKER_TAG) || docker pull $(DOCKER_IMAGE):latest || true
+
+docker/build:
+	docker build --cache-from=$(DOCKER_IMAGE):$(DOCKER_TAG) --tag=$(DOCKER_IMAGE):$(DOCKER_TAG) --file=Dockerfile.py3 .
+
+docker/login:
+	docker login --username $(DOCKER_USERNAME) --password $(DOCKER_PASSWORD) || true
+
+docker/push: docker/login
+	docker push $(DOCKER_IMAGE) || true
+
+docker/run/test: docker/build
+	docker run --rm --env-file=.env $(DOCKER_IMAGE) 'make test'
+
+docker/run/command: docker/build
+	docker run --rm --env-file=.env $(DOCKER_IMAGE) /bin/sh -c "$(COMMAND)"
+
+docker/run/make/%: docker/build
+	docker run --rm --env-file=.env $(DOCKER_IMAGE):$(DOCKER_TAG) make $*
+
+docker/run/shell: docker/build
+	docker run --rm --env-file=.env -it $(DOCKER_IMAGE)