Skip to content

Commit c7d76e4

Browse files
AndreMirasAndreMiras
committed
Travis CI revamp part 1, refs #2008
Revamping Travis and Docker setup introducing a `Makefile`. The idea is to move the CI complexity from .travis.yml to `Makefile`. That makes a single entry point via `make` command and reproducible builds via Docker. It makes it easy to run some commands outside docker, such as: ```sh make testapps/python3/armeabi-v7a ``` Or the same command inside docker: ```sh make docker/run/make/testapps/python3/armeabi-v7a ``` This pull request also starts introducing some docker layer cache optimization as needed by #2009 to speed up docker pull/push and rebuilds from cache. It also introduces other Docker images good practices like ordering dependencies alphabetically or always enforcing `apt update` prior install, refs: https://docs.docker.com/develop/develop-images/dockerfile_best-practices/ Subsequent pull requests would simplify the process furthermore and leverage the cache to speed up builds.
1 parent 080ac01 commit c7d76e4

File tree

4 files changed

+162
-87
lines changed

4 files changed

+162
-87
lines changed

‎.env

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# used by coveralls.io, refs:
2+
# https://coveralls-python.readthedocs.io/en/latest/usage/tox.html#travisci
3+
CI
4+
TRAVIS
5+
TRAVIS_BRANCH
6+
TRAVIS_JOB_ID
7+
TRAVIS_PULL_REQUEST

‎.travis.yml

Lines changed: 17 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,10 @@
11
sudo: required
22

3-
dist: xenial # needed for more recent python 3 and python3-venv
4-
53
language: generic
64

75
stages:
8-
- lint
9-
- test
6+
- unit tests
7+
- build testapps
108

119
services:
1210
- docker
@@ -18,8 +16,8 @@ before_install:
1816

1917
jobs:
2018
include:
21-
- &linting
22-
stage: lint
19+
- &unittests
20+
stage: unit tests
2321
language: python
2422
python: 3.7
2523
before_script:
@@ -36,66 +34,35 @@ jobs:
3634
- pip3.7 install pyOpenSSL
3735
- pip3.7 install coveralls
3836
script:
39-
# we want to fail fast on tox errors without having to `docker build` first
37+
# ignores test_pythonpackage.py since it runs for too long
4038
- tox -- tests/ --ignore tests/test_pythonpackage.py
41-
# (we ignore test_pythonpackage.py since these run way too long!!
42-
# test_pythonpackage_basic.py will still be run.)
4339
name: "Tox Pep8"
4440
env: TOXENV=pep8
45-
- <<: *linting
41+
- <<: *unittests
4642
name: "Tox Python 2"
4743
env: TOXENV=py27
48-
- <<: *linting
44+
- <<: *unittests
4945
name: "Tox Python 3 & Coverage"
5046
env: TOXENV=py3
5147
after_success:
5248
- coveralls
53-
54-
- &testing
55-
stage: test
56-
before_script:
57-
# build docker image
58-
- docker build --tag=p4a --file Dockerfile.py3 .
59-
# Run a background process to make sure that travis will not kill our tests in
60-
# case that the travis log doesn't produce any output for more than 10 minutes
61-
- while sleep 540; do echo "==== Still running (travis, don't kill me) ===="; done &
62-
script:
63-
- >
64-
docker run
65-
-e CI
66-
-e TRAVIS_JOB_ID
67-
-e TRAVIS_BRANCH
68-
-e ANDROID_SDK_HOME="/home/user/.android/android-sdk"
69-
-e ANDROID_NDK_HOME="/home/user/.android/android-ndk"
70-
p4a /bin/sh -c "$COMMAND"
71-
after_script:
72-
# kill the background process started before run docker
73-
- kill %1
49+
- &testapps
7450
name: Python 3 arm64-v8a
75-
# overrides requirements to skip `peewee` pure python module, see:
76-
# https://github.com/kivy/python-for-android/issues/1263#issuecomment-390421054
77-
env:
78-
COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools --arch=arm64-v8a'
79-
- <<: *testing
51+
stage: build testapps
52+
before_script: make docker/pull
53+
script: make docker/run/make/testapps/python3/arm64-v8a
54+
- <<: *testapps
8055
name: Python 3 armeabi-v7a
8156
os: osx
8257
osx_image: xcode11 # since xcode1.3, python3 is the default interpreter
8358
before_script:
8459
# installs java 1.8, android's SDK/NDK and p4a
8560
- make -f ci/makefiles/osx.mk
8661
- export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
87-
# Run a background process (like we do with linux tests)
88-
- while sleep 540; do echo "==== Still running (travis, don't kill me) ===="; done &
89-
script:
90-
- >
91-
cd testapps && python3 setup_testapp_python3_sqlite_openssl.py apk
92-
--sdk-dir $HOME/.android/android-sdk
93-
--ndk-dir $HOME/.android/android-ndk
94-
--requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools
95-
--arch=armeabi-v7a
96-
- <<: *testing
62+
script: make testapps/python3/armeabi-v7a PYTHON_WITH_VERSION=python3
63+
- <<: *testapps
9764
name: Python 2 armeabi-v7a (with numpy)
98-
env: COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools,numpy'
99-
- <<: *testing
65+
script: make docker/run/make/testapps/python2/armeabi-v7a
66+
- <<: *testapps
10067
name: Rebuild updated recipes
101-
env: COMMAND='. venv/bin/activate && ./ci/rebuild_updated_recipes.py'
68+
script: make docker/run/make/rebuild_updated_recipes

‎Dockerfile.py3

Lines changed: 60 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,12 @@ ENV LANG="en_US.UTF-8" \
2525
LANGUAGE="en_US.UTF-8" \
2626
LC_ALL="en_US.UTF-8"
2727

28-
RUN apt -y update -qq \
29-
&& apt -y install -qq --no-install-recommends curl unzip ca-certificates \
30-
&& apt -y autoremove
28+
RUN apt -y update -qq > /dev/null && apt -y install -qq --no-install-recommends \
29+
ca-certificates \
30+
curl \
31+
&& apt -y autoremove \
32+
&& apt -y clean \
33+
&& rm -rf /var/lib/apt/lists/*
3134

3235
# retry helper script, refs:
3336
# https://github.com/kivy/python-for-android/issues/1306
@@ -37,37 +40,52 @@ RUN curl https://raw.githubusercontent.com/kadwanev/retry/1.0.1/retry \
3740

3841
ENV USER="user"
3942
ENV HOME_DIR="/home/${USER}"
40-
ENV ANDROID_HOME="${HOME_DIR}/.android"
41-
ENV WORK_DIR="${HOME_DIR}" \
42-
PATH="${HOME_DIR}/.local/bin:${PATH}"
43+
ENV WORK_DIR="${HOME_DIR}/app" \
44+
PATH="${HOME_DIR}/.local/bin:${PATH}" \
45+
ANDROID_HOME="${HOME_DIR}/.android" \
46+
JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64"
4347

44-
# install system dependencies
45-
RUN ${RETRY} apt -y install -qq --no-install-recommends \
46-
python3 virtualenv python3-pip python3-venv \
47-
wget lbzip2 patch sudo python python-pip \
48-
&& apt -y autoremove
4948

50-
# build dependencies
51-
# https://buildozer.readthedocs.io/en/latest/installation.html#android-on-ubuntu-16-04-64bit
49+
# install system dependencies
5250
RUN dpkg --add-architecture i386 \
53-
&& ${RETRY} apt -y update -qq \
51+
&& ${RETRY} apt -y update -qq > /dev/null \
5452
&& ${RETRY} apt -y install -qq --no-install-recommends \
55-
build-essential ccache git python3 python3-dev \
56-
libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \
57-
libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 \
58-
zip zlib1g-dev zlib1g:i386 \
59-
&& apt -y autoremove
60-
61-
# specific recipes dependencies (e.g. libffi requires autoreconf binary)
62-
RUN ${RETRY} apt -y install -qq --no-install-recommends \
63-
libffi-dev autoconf automake cmake gettext libltdl-dev libtool pkg-config \
53+
autoconf \
54+
automake \
55+
build-essential \
56+
ccache \
57+
cmake \
58+
gettext \
59+
git \
60+
lbzip2 \
61+
libffi-dev \
62+
libgtk2.0-0:i386 \
63+
libidn11:i386 \
64+
libltdl-dev \
65+
libncurses5:i386 \
66+
libpangox-1.0-0:i386 \
67+
libpangoxft-1.0-0:i386 \
68+
libstdc++6:i386 \
69+
libtool \
70+
openjdk-8-jdk \
71+
patch \
72+
pkg-config \
73+
python \
74+
python-pip \
75+
python3 \
76+
python3-dev \
77+
python3-pip \
78+
python3-venv \
79+
sudo \
80+
unzip \
81+
virtualenv \
82+
wget \
83+
zip \
84+
zlib1g-dev \
85+
zlib1g:i386 \
6486
&& apt -y autoremove \
65-
&& apt -y clean
66-
67-
# Install Java and set JAVA_HOME (to accept android's SDK licenses)
68-
RUN ${RETRY} apt -y install -qq --no-install-recommends openjdk-8-jdk \
69-
&& apt -y autoremove && apt -y clean
70-
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
87+
&& apt -y clean \
88+
&& rm -rf /var/lib/apt/lists/*
7189

7290
# prepare non root env
7391
RUN useradd --create-home --shell /bin/bash ${USER}
@@ -77,18 +95,23 @@ RUN usermod -append --groups sudo ${USER}
7795
RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
7896

7997
# install cython for python 2 (for python 3 it's inside the venv)
80-
RUN pip2 install --upgrade Cython==0.28.6
98+
RUN pip2 install --upgrade Cython==0.28.6 \
99+
&& rm -rf ~/.cache/
81100

82101
WORKDIR ${WORK_DIR}
83-
COPY --chown=user:user . ${WORK_DIR}
84-
RUN mkdir ${ANDROID_HOME} && chown --recursive ${USER} ${ANDROID_HOME}
102+
RUN mkdir ${ANDROID_HOME} && chown --recursive ${USER} ${HOME_DIR} ${ANDROID_HOME}
85103
USER ${USER}
86104

87105
# Download and install android's NDK/SDK
88-
RUN make -f ci/makefiles/android.mk target_os=linux
106+
COPY ci/makefiles/android.mk /tmp/android.mk
107+
RUN make --file /tmp/android.mk target_os=linux \
108+
&& sudo rm /tmp/android.mk
89109

90110
# install python-for-android from current branch
91-
RUN virtualenv --python=python3 venv \
92-
&& . venv/bin/activate \
93-
&& pip3 install --upgrade Cython==0.28.6 \
94-
&& pip3 install -e .
111+
COPY --chown=user:user Makefile README.md setup.py pythonforandroid/__init__.py ${WORK_DIR}/
112+
RUN mkdir pythonforandroid \
113+
&& mv __init__.py pythonforandroid/ \
114+
&& make virtualenv \
115+
&& rm -rf ~/.cache/
116+
117+
COPY --chown=user:user . ${WORK_DIR}

‎Makefile

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
VIRTUAL_ENV ?= venv
2+
PIP=$(VIRTUAL_ENV)/bin/pip
3+
TOX=`which tox`
4+
ACTIVATE=$(VIRTUAL_ENV)/bin/activate
5+
PYTHON=$(VIRTUAL_ENV)/bin/python
6+
FLAKE8=$(VIRTUAL_ENV)/bin/flake8
7+
PYTEST=$(VIRTUAL_ENV)/bin/pytest
8+
SOURCES=src/ tests/
9+
PYTHON_MAJOR_VERSION=3
10+
PYTHON_MINOR_VERSION=6
11+
PYTHON_VERSION=$(PYTHON_MAJOR_VERSION).$(PYTHON_MINOR_VERSION)
12+
PYTHON_MAJOR_MINOR=$(PYTHON_MAJOR_VERSION)$(PYTHON_MINOR_VERSION)
13+
PYTHON_WITH_VERSION=python$(PYTHON_VERSION)
14+
DOCKER_IMAGE=kivy/python-for-android
15+
ANDROID_SDK_HOME ?= $(HOME)/.android/android-sdk
16+
ANDROID_NDK_HOME ?= $(HOME)/.android/android-ndk
17+
18+
19+
all: virtualenv
20+
21+
$(VIRTUAL_ENV):
22+
virtualenv --python=$(PYTHON_WITH_VERSION) $(VIRTUAL_ENV)
23+
$(PIP) install Cython==0.28.6
24+
$(PIP) install -e .
25+
26+
virtualenv: $(VIRTUAL_ENV)
27+
28+
# ignores test_pythonpackage.py since it runs for too long
29+
test:
30+
$(TOX) -- tests/ --ignore tests/test_pythonpackage.py
31+
@if test -n "$$CI"; then .tox/py$(PYTHON_MAJOR_MINOR)/bin/coveralls; fi; \
32+
33+
rebuild_updated_recipes: virtualenv
34+
$(PYTHON) ci/rebuild_updated_recipes.py
35+
36+
testapps/python2/armeabi-v7a: virtualenv
37+
. $(ACTIVATE) && cd testapps/ && \
38+
python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
39+
--requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools,numpy
40+
41+
testapps/python3/arm64-v8a: virtualenv
42+
. $(ACTIVATE) && cd testapps/ && \
43+
python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
44+
--arch=arm64-v8a
45+
46+
testapps/python3/armeabi-v7a: virtualenv
47+
. $(ACTIVATE) && cd testapps/ && \
48+
python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
49+
--requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools \
50+
--arch=armeabi-v7a
51+
52+
clean:
53+
find . -type d -name "__pycache__" -exec rm -r {} +
54+
find . -type d -name "*.egg-info" -exec rm -r {} +
55+
56+
clean/all: clean
57+
rm -rf $(VIRTUAL_ENV) .tox/
58+
59+
docker/pull:
60+
docker pull $(DOCKER_IMAGE):latest || true
61+
62+
docker/build:
63+
docker build --cache-from=$(DOCKER_IMAGE) --tag=$(DOCKER_IMAGE) --file=Dockerfile.py3 .
64+
65+
docker/push:
66+
docker push $(DOCKER_IMAGE)
67+
68+
docker/run/test: docker/build
69+
docker run --rm --env-file=.env $(DOCKER_IMAGE) 'make test'
70+
71+
docker/run/command: docker/build
72+
docker run --rm --env-file=.env $(DOCKER_IMAGE) /bin/sh -c "$(COMMAND)"
73+
74+
docker/run/make/%: docker/build
75+
docker run --rm --env-file=.env $(DOCKER_IMAGE) make $*
76+
77+
docker/run/shell: docker/build
78+
docker run --rm --env-file=.env -it $(DOCKER_IMAGE)

0 commit comments

Comments
 (0)