From a9750156c183254cbf81c88f3f8fbac0aa736b38 Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Sun, 27 Oct 2019 00:15:16 +0200 Subject: [PATCH 1/2] Travis CI revamp part 1, refs #2008 Revamping Travis and Docker setup introducing a `Makefile`. The idea is to move the CI complexity from .travis.yml to `Makefile`. That makes a single entry point via `make` command and reproducible builds via Docker. It makes it easy to run some commands outside docker, such as: ```sh make testapps/python3/armeabi-v7a ``` Or the same command inside docker: ```sh make docker/run/make/testapps/python3/armeabi-v7a ``` This pull request also starts introducing some docker layer cache optimization as needed by #2009 to speed up docker pull/push and rebuilds from cache. It also introduces other Docker images good practices like ordering dependencies alphabetically or always enforcing `apt update` prior install, refs: https://docs.docker.com/develop/develop-images/dockerfile_best-practices/ Subsequent pull requests would simplify the process furthermore and leverage the cache to speed up builds. --- .env | 9 +++++ .travis.yml | 60 +++++++++------------------------- Dockerfile.py3 | 89 ++++++++++++++++++++++++++++++-------------------- Makefile | 78 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 80 deletions(-) create mode 100644 .env create mode 100644 Makefile diff --git a/.env b/.env new file mode 100644 index 0000000000..732d46e16b --- /dev/null +++ b/.env @@ -0,0 +1,9 @@ +# used by coveralls.io, refs: +# https://coveralls-python.readthedocs.io/en/latest/usage/tox.html#travisci +CI +TRAVIS +TRAVIS_BRANCH +TRAVIS_JOB_ID +TRAVIS_PULL_REQUEST +# used for running UI tests +DISPLAY diff --git a/.travis.yml b/.travis.yml index cc5dd0e170..30985aec6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,10 @@ sudo: required -dist: xenial # needed for more recent python 3 and python3-venv - language: generic stages: - - lint - - test + - unit tests + - build apps services: - docker @@ -18,8 +16,8 @@ before_install: jobs: include: - - &linting - stage: lint + - &unittests + stage: unit tests language: python python: 3.7 before_script: @@ -36,46 +34,26 @@ jobs: - pip3.7 install pyOpenSSL - pip3.7 install coveralls script: - # we want to fail fast on tox errors without having to `docker build` first + # ignores test_pythonpackage.py since it runs for too long - tox -- tests/ --ignore tests/test_pythonpackage.py - # (we ignore test_pythonpackage.py since these run way too long!! - # test_pythonpackage_basic.py will still be run.) name: "Tox Pep8" env: TOXENV=pep8 - - <<: *linting + - <<: *unittests name: "Tox Python 2" env: TOXENV=py27 - - <<: *linting + - <<: *unittests name: "Tox Python 3 & Coverage" env: TOXENV=py3 after_success: - coveralls - - &testing - stage: test + stage: build apps before_script: - # build docker image - - docker build --tag=p4a --file Dockerfile.py3 . - # Run a background process to make sure that travis will not kill our tests in - # case that the travis log doesn't produce any output for more than 10 minutes - - while sleep 540; do echo "==== Still running (travis, don't kill me) ===="; done & - script: - - > - docker run - -e CI - -e TRAVIS_JOB_ID - -e TRAVIS_BRANCH - -e ANDROID_SDK_HOME="/home/user/.android/android-sdk" - -e ANDROID_NDK_HOME="/home/user/.android/android-ndk" - p4a /bin/sh -c "$COMMAND" - after_script: - # kill the background process started before run docker - - kill %1 + # ideally we would have a stage for that so it's not ran multiple time + # but it seems Travis doesn't share the build cache + - make docker/build name: Python 3 arm64-v8a - # overrides requirements to skip `peewee` pure python module, see: - # https://github.com/kivy/python-for-android/issues/1263#issuecomment-390421054 - env: - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools --arch=arm64-v8a' + script: make docker/run/make/testapps/python3/arm64-v8a - <<: *testing name: Python 3 armeabi-v7a os: osx @@ -84,18 +62,10 @@ jobs: # installs java 1.8, android's SDK/NDK and p4a - make -f ci/makefiles/osx.mk - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home - # Run a background process (like we do with linux tests) - - while sleep 540; do echo "==== Still running (travis, don't kill me) ===="; done & - script: - - > - cd testapps && python3 setup_testapp_python3_sqlite_openssl.py apk - --sdk-dir $HOME/.android/android-sdk - --ndk-dir $HOME/.android/android-ndk - --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools - --arch=armeabi-v7a + script: make testapps/python3/armeabi-v7a PYTHON_WITH_VERSION=python3 - <<: *testing name: Python 2 armeabi-v7a (with numpy) - env: COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools,numpy' + script: make docker/run/make/testapps/python2/armeabi-v7a - <<: *testing name: Rebuild updated recipes - env: COMMAND='. venv/bin/activate && ./ci/rebuild_updated_recipes.py' + script: make docker/run/make/rebuild_updated_recipes diff --git a/Dockerfile.py3 b/Dockerfile.py3 index 70c9417180..c4758d89c4 100644 --- a/Dockerfile.py3 +++ b/Dockerfile.py3 @@ -25,9 +25,11 @@ ENV LANG="en_US.UTF-8" \ LANGUAGE="en_US.UTF-8" \ LC_ALL="en_US.UTF-8" -RUN apt -y update -qq \ - && apt -y install -qq --no-install-recommends curl unzip ca-certificates \ - && apt -y autoremove +RUN apt -y update -qq > /dev/null && apt -y install -qq --no-install-recommends \ + ca-certificates \ + curl \ + && apt -y autoremove \ + && apt -y clean # retry helper script, refs: # https://github.com/kivy/python-for-android/issues/1306 @@ -37,38 +39,52 @@ RUN curl https://raw.githubusercontent.com/kadwanev/retry/1.0.1/retry \ ENV USER="user" ENV HOME_DIR="/home/${USER}" -ENV ANDROID_HOME="${HOME_DIR}/.android" -ENV WORK_DIR="${HOME_DIR}" \ - PATH="${HOME_DIR}/.local/bin:${PATH}" +ENV WORK_DIR="${HOME_DIR}/app" \ + PATH="${HOME_DIR}/.local/bin:${PATH}" \ + ANDROID_HOME="${HOME_DIR}/.android" \ + JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64" -# install system dependencies -RUN ${RETRY} apt -y install -qq --no-install-recommends \ - python3 virtualenv python3-pip python3-venv \ - wget lbzip2 patch sudo python python-pip \ - && apt -y autoremove -# build dependencies -# https://buildozer.readthedocs.io/en/latest/installation.html#android-on-ubuntu-16-04-64bit +# install system dependencies RUN dpkg --add-architecture i386 \ - && ${RETRY} apt -y update -qq \ + && ${RETRY} apt -y update -qq > /dev/null \ && ${RETRY} apt -y install -qq --no-install-recommends \ - build-essential ccache git python3 python3-dev \ - libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \ - libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 \ - zip zlib1g-dev zlib1g:i386 \ - && apt -y autoremove - -# specific recipes dependencies (e.g. libffi requires autoreconf binary) -RUN ${RETRY} apt -y install -qq --no-install-recommends \ - libffi-dev autoconf automake cmake gettext libltdl-dev libtool pkg-config \ + autoconf \ + automake \ + build-essential \ + ccache \ + cmake \ + gettext \ + git \ + lbzip2 \ + libffi-dev \ + libgtk2.0-0:i386 \ + libidn11:i386 \ + libltdl-dev \ + libncurses5:i386 \ + libpangox-1.0-0:i386 \ + libpangoxft-1.0-0:i386 \ + libstdc++6:i386 \ + libtool \ + openjdk-8-jdk \ + patch \ + pkg-config \ + python \ + python-pip \ + python3 \ + python3-dev \ + python3-pip \ + python3-venv \ + sudo \ + unzip \ + virtualenv \ + wget \ + zip \ + zlib1g-dev \ + zlib1g:i386 \ && apt -y autoremove \ && apt -y clean -# Install Java and set JAVA_HOME (to accept android's SDK licenses) -RUN ${RETRY} apt -y install -qq --no-install-recommends openjdk-8-jdk \ - && apt -y autoremove && apt -y clean -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 - # prepare non root env RUN useradd --create-home --shell /bin/bash ${USER} @@ -80,15 +96,18 @@ RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers RUN pip2 install --upgrade Cython==0.28.6 WORKDIR ${WORK_DIR} -COPY --chown=user:user . ${WORK_DIR} -RUN mkdir ${ANDROID_HOME} && chown --recursive ${USER} ${ANDROID_HOME} +RUN mkdir ${ANDROID_HOME} && chown --recursive ${USER} ${HOME_DIR} ${ANDROID_HOME} USER ${USER} # Download and install android's NDK/SDK -RUN make -f ci/makefiles/android.mk target_os=linux +COPY ci/makefiles/android.mk /tmp/android.mk +RUN make --file /tmp/android.mk target_os=linux \ + && sudo rm /tmp/android.mk # install python-for-android from current branch -RUN virtualenv --python=python3 venv \ - && . venv/bin/activate \ - && pip3 install --upgrade Cython==0.28.6 \ - && pip3 install -e . +COPY --chown=user:user Makefile README.md setup.py pythonforandroid/__init__.py ${WORK_DIR}/ +RUN mkdir pythonforandroid \ + && mv __init__.py pythonforandroid/ \ + && make virtualenv + +COPY --chown=user:user . ${WORK_DIR} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..05687dffd6 --- /dev/null +++ b/Makefile @@ -0,0 +1,78 @@ +VIRTUAL_ENV ?= venv +PIP=$(VIRTUAL_ENV)/bin/pip +TOX=`which tox` +ACTIVATE=$(VIRTUAL_ENV)/bin/activate +PYTHON=$(VIRTUAL_ENV)/bin/python +FLAKE8=$(VIRTUAL_ENV)/bin/flake8 +PYTEST=$(VIRTUAL_ENV)/bin/pytest +SOURCES=src/ tests/ +PYTHON_MAJOR_VERSION=3 +PYTHON_MINOR_VERSION=6 +PYTHON_VERSION=$(PYTHON_MAJOR_VERSION).$(PYTHON_MINOR_VERSION) +PYTHON_MAJOR_MINOR=$(PYTHON_MAJOR_VERSION)$(PYTHON_MINOR_VERSION) +PYTHON_WITH_VERSION=python$(PYTHON_VERSION) +DOCKER_IMAGE=kivy/python-for-android +ANDROID_SDK_HOME ?= $(HOME)/.android/android-sdk +ANDROID_NDK_HOME ?= $(HOME)/.android/android-ndk + + +all: virtualenv + +$(VIRTUAL_ENV): + virtualenv --python=$(PYTHON_WITH_VERSION) $(VIRTUAL_ENV) + $(PIP) install Cython==0.28.6 + $(PIP) install -e . + +virtualenv: $(VIRTUAL_ENV) + +# ignores test_pythonpackage.py since it runs for too long +test: + $(TOX) -- tests/ --ignore tests/test_pythonpackage.py + @if test -n "$$CI"; then .tox/py$(PYTHON_MAJOR_MINOR)/bin/coveralls; fi; \ + +rebuild_updated_recipes: virtualenv + $(PYTHON) ci/rebuild_updated_recipes.py + +testapps/python2/armeabi-v7a: virtualenv + . $(ACTIVATE) && cd testapps/ && \ + python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools,numpy + +testapps/python3/arm64-v8a: virtualenv + . $(ACTIVATE) && cd testapps/ && \ + python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + --arch=arm64-v8a + +testapps/python3/armeabi-v7a: virtualenv + . $(ACTIVATE) && cd testapps/ && \ + python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools \ + --arch=armeabi-v7a + +clean: + find . -type d -name "__pycache__" -exec rm -r {} + + find . -type d -name "*.egg-info" -exec rm -r {} + + +clean/all: clean + rm -rf $(VIRTUAL_ENV) .tox/ + +docker/pull: + docker pull $(DOCKER_IMAGE):latest || true + +docker/build: + docker build --cache-from=$(DOCKER_IMAGE) --tag=$(DOCKER_IMAGE) --file=Dockerfile.py3 . + +docker/push: + docker push $(DOCKER_IMAGE) + +docker/run/test: docker/build + docker run --rm --env-file=.env $(DOCKER_IMAGE) 'make test' + +docker/run/command: docker/build + docker run --rm --env-file=.env $(DOCKER_IMAGE) /bin/sh -c "$(COMMAND)" + +docker/run/make/%: docker/build + docker run --rm --env-file=.env $(DOCKER_IMAGE) make $* + +docker/run/shell: docker/build + docker run --rm --env-file=.env -it $(DOCKER_IMAGE) From da99bde49a545abe99f317aa28f8463b0e19a777 Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Mon, 28 Oct 2019 15:13:44 +0100 Subject: [PATCH 2/2] Push Docker image and build from it in sub-stages The `docker build` is now running alongside the `tox` testing as part of the `pre checks` stage. It also fully leverages the docker image cache. The newly built image is pushed to be available for subsequent build stages and pull requests. Two environment variables `DOCKER_USERNAME` and `DOCKER_PASSWORD` got added to the Travis UI for making image pushing possible. Both `push` and `pull` features fail silently with `|| true` as they should be optional for a valid build. `TRAVIS_PULL_REQUEST` and `TRAVIS_BRANCH` are built-in Travis variables. Note that we're passing variables to `Makefile` explicitly on `make` call rather than using the `--environment-overrides` flag. While it's more verbose, it makes it easier to follow what's being used and when. With this change, `Python 3 arm64-v8a` the longest `testapps` build that was taking ~26 minutes to complete is now taking ~20 minutes when hitting docker layers cache (most of the time). The shorter build `Rebuild updated recipes` was taking ~9 minutes and is now down to ~3 minutes. Closes #2009 --- .travis.yml | 39 ++++++++++++++++++++++----------------- Dockerfile.py3 | 12 ++++++++---- Makefile | 24 ++++++++++++++++++------ 3 files changed, 48 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 30985aec6c..f0d9c36018 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,8 @@ sudo: required language: generic stages: - - unit tests - - build apps + - pre checks + - build testapps services: - docker @@ -16,8 +16,8 @@ before_install: jobs: include: - - &unittests - stage: unit tests + - &prechecks + stage: pre checks language: python python: 3.7 before_script: @@ -38,23 +38,28 @@ jobs: - tox -- tests/ --ignore tests/test_pythonpackage.py name: "Tox Pep8" env: TOXENV=pep8 - - <<: *unittests + - <<: *prechecks name: "Tox Python 2" env: TOXENV=py27 - - <<: *unittests + - <<: *prechecks name: "Tox Python 3 & Coverage" env: TOXENV=py3 after_success: - coveralls - - &testing - stage: build apps + # the docker build stage runs almost as fast as `tox` unit tests, so we can run in parallel + - name: "Docker build" before_script: - # ideally we would have a stage for that so it's not ran multiple time - # but it seems Travis doesn't share the build cache - - make docker/build + - make docker/pull TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST TRAVIS_BRANCH=$TRAVIS_BRANCH + script: + - make docker/build TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST TRAVIS_BRANCH=$TRAVIS_BRANCH + after_success: + - make docker/push DOCKER_USERNAME=$DOCKER_USERNAME DOCKER_PASSWORD=$DOCKER_PASSWORD + - &testapps name: Python 3 arm64-v8a - script: make docker/run/make/testapps/python3/arm64-v8a - - <<: *testing + stage: build testapps + before_script: make docker/pull TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST TRAVIS_BRANCH=$TRAVIS_BRANCH + script: make docker/run/make/testapps/python3/arm64-v8a TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST TRAVIS_BRANCH=$TRAVIS_BRANCH + - <<: *testapps name: Python 3 armeabi-v7a os: osx osx_image: xcode11 # since xcode1.3, python3 is the default interpreter @@ -63,9 +68,9 @@ jobs: - make -f ci/makefiles/osx.mk - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home script: make testapps/python3/armeabi-v7a PYTHON_WITH_VERSION=python3 - - <<: *testing + - <<: *testapps name: Python 2 armeabi-v7a (with numpy) - script: make docker/run/make/testapps/python2/armeabi-v7a - - <<: *testing + script: make docker/run/make/testapps/python2/armeabi-v7a TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST TRAVIS_BRANCH=$TRAVIS_BRANCH + - <<: *testapps name: Rebuild updated recipes - script: make docker/run/make/rebuild_updated_recipes + script: make docker/run/make/rebuild_updated_recipes TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST TRAVIS_BRANCH=$TRAVIS_BRANCH diff --git a/Dockerfile.py3 b/Dockerfile.py3 index c4758d89c4..cd6e9f8405 100644 --- a/Dockerfile.py3 +++ b/Dockerfile.py3 @@ -29,7 +29,8 @@ RUN apt -y update -qq > /dev/null && apt -y install -qq --no-install-recommends ca-certificates \ curl \ && apt -y autoremove \ - && apt -y clean + && apt -y clean \ + && rm -rf /var/lib/apt/lists/* # retry helper script, refs: # https://github.com/kivy/python-for-android/issues/1306 @@ -83,7 +84,8 @@ RUN dpkg --add-architecture i386 \ zlib1g-dev \ zlib1g:i386 \ && apt -y autoremove \ - && apt -y clean + && apt -y clean \ + && rm -rf /var/lib/apt/lists/* # prepare non root env RUN useradd --create-home --shell /bin/bash ${USER} @@ -93,7 +95,8 @@ RUN usermod -append --groups sudo ${USER} RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers # install cython for python 2 (for python 3 it's inside the venv) -RUN pip2 install --upgrade Cython==0.28.6 +RUN pip2 install --upgrade Cython==0.28.6 \ + && rm -rf ~/.cache/ WORKDIR ${WORK_DIR} RUN mkdir ${ANDROID_HOME} && chown --recursive ${USER} ${HOME_DIR} ${ANDROID_HOME} @@ -108,6 +111,7 @@ RUN make --file /tmp/android.mk target_os=linux \ COPY --chown=user:user Makefile README.md setup.py pythonforandroid/__init__.py ${WORK_DIR}/ RUN mkdir pythonforandroid \ && mv __init__.py pythonforandroid/ \ - && make virtualenv + && make virtualenv \ + && rm -rf ~/.cache/ COPY --chown=user:user . ${WORK_DIR} diff --git a/Makefile b/Makefile index 05687dffd6..cb83c84093 100644 --- a/Makefile +++ b/Makefile @@ -11,9 +11,18 @@ PYTHON_MINOR_VERSION=6 PYTHON_VERSION=$(PYTHON_MAJOR_VERSION).$(PYTHON_MINOR_VERSION) PYTHON_MAJOR_MINOR=$(PYTHON_MAJOR_VERSION)$(PYTHON_MINOR_VERSION) PYTHON_WITH_VERSION=python$(PYTHON_VERSION) -DOCKER_IMAGE=kivy/python-for-android ANDROID_SDK_HOME ?= $(HOME)/.android/android-sdk ANDROID_NDK_HOME ?= $(HOME)/.android/android-ndk +DOCKER_IMAGE=kivy/python-for-android +# tag to latest only on merge to develop +DOCKER_TAG=latest +ifdef TRAVIS_PULL_REQUEST +ifneq ($(TRAVIS_PULL_REQUEST), false) + DOCKER_TAG=pr-$(TRAVIS_PULL_REQUEST) +else ifneq ($(TRAVIS_BRANCH), develop) + DOCKER_TAG=branch-$(subst /,-,$(TRAVIS_BRANCH)) # slash is an invalid docker tag character +endif +endif all: virtualenv @@ -57,13 +66,16 @@ clean/all: clean rm -rf $(VIRTUAL_ENV) .tox/ docker/pull: - docker pull $(DOCKER_IMAGE):latest || true + docker pull $(DOCKER_IMAGE):$(DOCKER_TAG) || docker pull $(DOCKER_IMAGE):latest || true docker/build: - docker build --cache-from=$(DOCKER_IMAGE) --tag=$(DOCKER_IMAGE) --file=Dockerfile.py3 . + docker build --cache-from=$(DOCKER_IMAGE):$(DOCKER_TAG) --tag=$(DOCKER_IMAGE):$(DOCKER_TAG) --file=Dockerfile.py3 . + +docker/login: + docker login --username $(DOCKER_USERNAME) --password $(DOCKER_PASSWORD) || true -docker/push: - docker push $(DOCKER_IMAGE) +docker/push: docker/login + docker push $(DOCKER_IMAGE) || true docker/run/test: docker/build docker run --rm --env-file=.env $(DOCKER_IMAGE) 'make test' @@ -72,7 +84,7 @@ docker/run/command: docker/build docker run --rm --env-file=.env $(DOCKER_IMAGE) /bin/sh -c "$(COMMAND)" docker/run/make/%: docker/build - docker run --rm --env-file=.env $(DOCKER_IMAGE) make $* + docker run --rm --env-file=.env $(DOCKER_IMAGE):$(DOCKER_TAG) make $* docker/run/shell: docker/build docker run --rm --env-file=.env -it $(DOCKER_IMAGE)