From 61077f3893895b6c5e49d47b0040079c0e2fe76c Mon Sep 17 00:00:00 2001 From: beckermr Date: Fri, 10 Mar 2017 17:20:04 -0600 Subject: [PATCH 01/13] ENH move to a pure pip install --- .condarc | 7 -- Dockerfile | 176 +++++++++++++++++++++++++++-------------------- environment.yml | 49 ------------- requirements.txt | 37 ++++++++++ 4 files changed, 138 insertions(+), 131 deletions(-) delete mode 100644 .condarc delete mode 100644 environment.yml create mode 100644 requirements.txt diff --git a/.condarc b/.condarc deleted file mode 100644 index 364cb1a..0000000 --- a/.condarc +++ /dev/null @@ -1,7 +0,0 @@ -channels: - - defaults - -show_channel_urls: True - -create_default_packages: - - nomkl diff --git a/Dockerfile b/Dockerfile index 9f6493c..44c3d94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,94 +1,120 @@ FROM ubuntu:14.04 MAINTAINER support@civisanalytics.com -# Ensure UTF-8 locale. -RUN locale-gen en_US.UTF-8 +# We need this file to install the right version of numpy. +COPY requirements.txt /requirements.txt +# Python install by wlattner. + +# Eemove several traces of debian python. +RUN apt-get purge -y python.* + +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG=C.UTF-8 \ + BASH_ENV=/etc/profile # Set environment variables for UTF-8, conda, and shell environments -ENV LANG=en_US.UTF-8 \ - LANGUAGE=en_US:en \ - LC_ALL=en_US.UTF-8 \ - CONDARC=/opt/conda/.condarc \ - BASH_ENV=/etc/profile \ - PATH=/opt/conda/bin:$PATH \ - CIVIS_CONDA_VERSION=4.3.11 \ - CIVIS_PYTHON_VERSION=3.6.0 +#ENV LANG=en_US.UTF-8 \ +# LANGUAGE=en_US:en \ +# LC_ALL=en_US.UTF-8 \ +# BASH_ENV=/etc/profile -RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && \ - apt-get install -y --no-install-recommends software-properties-common && \ - apt-get install -y --no-install-recommends \ - make \ - automake \ - libpq-dev \ - libffi-dev \ - gfortran \ - g++ \ - git \ - libboost-program-options-dev \ - libtool \ - libxrender1 \ - wget \ - ca-certificates \ - curl +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + libsqlite3-0 \ + libssl1.0.0 \ + libgfortran3 \ + git \ + postgresql \ + postgresql-contrib \ + libpq-dev && \ + rm -rf /var/lib/apt/lists/* -# Conda install. -# -# Everything is installed in the root environment. This allows for -# upgrades to the packages and eliminates the pain of trying to activate -# some other environment automatically for the many different ways -# people can use a docker image. -# -# Things are pinned to prevent upgrades from conda and force it to -# resolve dependencies relative to a fixed conda & python version. +# We like PGP and so are going through this pain. Notes for the future. Yay! +# I *think* this is how this works. # -# Note that the python version is also listed in the enviornment.yml -# file. The version in CIVIS_PYTHON_VERSION is the source of truth. -# If you want to change the python version, you need to change it in -# **both** places. The python version has been left in the `environment.yml` -# file so that people can create environments equivalent to this -# container. However, change the name of the environment first! +# 1) Figure out who the release manager is for your python version. (I *think* they usually make +# a PEP for it, because, like, they make a PEP for everything. See here for 3.6.0 +# https://www.python.org/dev/peps/pep-0494/.) +# 2) Then go look them up on a key server (This one is nice https://pgp.mit.edu/) +# 3) Search by their name and find the short 8 character key ID. You might have to try a few +# different keys since people can have more than one and it is not always clear which one +# was used to sign the package. +# 4) Put the 8 character key ID below. # -# The ordering of these steps seems to matter. You seem to have to -# install a specific python version by hand and then pin it. -# 1) install conda -# 2) pin conda to the version given by CIVIS_CONDA_VERSION -# 3) install the python version CIVIS_PYTHON_VERSION -# 4) pin the python version -RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ - wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${CIVIS_CONDA_VERSION}-Linux-x86_64.sh && \ - /bin/bash /Miniconda3-${CIVIS_CONDA_VERSION}-Linux-x86_64.sh -b -p /opt/conda && \ - rm Miniconda3-${CIVIS_CONDA_VERSION}-Linux-x86_64.sh && \ - /opt/conda/bin/conda install --yes conda==${CIVIS_CONDA_VERSION} && \ - echo "conda ==${CIVIS_CONDA_VERSION}" > /opt/conda/conda-meta/pinned && \ - conda install --yes python==${CIVIS_PYTHON_VERSION} && \ - echo "\npython ==${CIVIS_PYTHON_VERSION}" >> /opt/conda/conda-meta/pinned +# If everything goes well, you will successfully verify that the download (over https) in fact +# returned the correct, uncorrupted file. Yay! +ENV GPG_KEY AA65421D + +ENV PYTHON_VERSION 3.6.0 + +# If this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" +ENV PYTHON_PIP_VERSION 9.0.1 + +RUN set -ex && \ + buildDeps=' \ + curl \ + gcc \ + g++ \ + gfortran \ + libbz2-dev \ + libc6-dev \ + liblzma-dev \ + libncurses-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + make \ + xz-utils \ + zlib1g-dev' && \ + apt-get update && \ + apt-get install -y $buildDeps --no-install-recommends && \ + rm -rf /var/lib/apt/lists/* && \ + curl -fSL "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" -o python.tar.xz && \ + curl -fSL "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" -o python.tar.xz.asc && \ + export GNUPGHOME="$(mktemp -d)" && \ + gpg --keyserver pgp.mit.edu --recv-keys "$GPG_KEY" && \ + gpg --batch --verify python.tar.xz.asc python.tar.xz && \ + rm -r "$GNUPGHOME" python.tar.xz.asc && \ + mkdir -p /usr/src/python && \ + tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz && \ + rm python.tar.xz && \ + \ + cd /usr/src/python && \ + ./configure --enable-shared --enable-unicode=ucs4 && \ + make -j$(nproc) && \ + make install && \ + ldconfig && \ + pip3 install --no-cache-dir --upgrade --ignore-installed pip==$PYTHON_PIP_VERSION && \ + pip3 install --no-cache-dir `grep 'numpy' /requirements.txt` && \ + find /usr/local -depth \ + \( \ + \( -type d -a -name test -o -name tests \) \ + -o \ + \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \ + \) -exec rm -rf '{}' + && \ + rm -rf /usr/src/python ~/.cache -# Red Hat and Debian use different names for this file. git2R wants the latter. -# See conda-recipes GH 423 -RUN ln -s /opt/conda/lib/libopenblas.so /opt/conda/lib/libblas.so && \ - ln -s /opt/conda/lib/libopenblas.so /opt/conda/lib/liblapack.so && \ - ln -s /opt/conda/lib/libssl.so /opt/conda/lib/libssl.so.6 && \ - ln -s /opt/conda/lib/libcrypto.so /opt/conda/lib/libcrypto.so.6 +# Make some useful symlinks that are expected to exist. +RUN cd /usr/local/bin && \ + ln -s python3 python && \ + ln -s python3-config python-config -# Install boto in the base environment for private s3 channel support. -# Install Python Packages -COPY .condarc /opt/conda/.condarc -COPY environment.yml environment.yml -RUN conda install -y boto && \ - conda install -y nomkl && \ - conda env update -f environment.yml && \ - conda clean --all -y +# Now install the rest of the dependencies from the requirements file. +RUN pip install --no-cache-dir -r /requirements.txt +RUN rm -rf ~/.cache/* # We aren't running a GUI, so force matplotlib to use # the non-interactive "Agg" backend for graphics. ENV MATPLOTLIBRC=${HOME}/.config/matplotlib/matplotlibrc -RUN mkdir -p ${HOME}/.config/matplotlib -RUN echo "backend : Agg" > ${HOME}/.config/matplotlib/matplotlibrc +RUN mkdir -p `dirname ${MATPLOTLIBRC}` +RUN echo "backend : Agg" > ${MATPLOTLIBRC} -# Run matplotlib once to build the font cache +# Run matplotlib once to build the font cache. RUN python -c "import matplotlib.pyplot" -ENV VERSION=2.0.1 \ - VERSION_MAJOR=2 \ +ENV VERSION=3.0.0 \ + VERSION_MAJOR=3 \ VERSION_MINOR=0 \ - VERSION_MICRO=1 + VERSION_MICRO=0 diff --git a/environment.yml b/environment.yml deleted file mode 100644 index 21809c5..0000000 --- a/environment.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: root -dependencies: -- beautifulsoup4=4.5.3 -- boto=2.45.0 -- boto3==1.4.3 -- cython=0.25.2 -- ipython=5.1.0 -- jinja2=2.8 -- jsonschema=2.5.1 -- jupyter=1.0.0 -- libffi=3.2.1 -- libgcc=5.2.0 -- libgfortran=3.0.0 -- libsodium=1.0.10 -- libtiff=4.0.6 -- libxml2=2.9.2 -- matplotlib=2.0.0 -- nomkl=1.0 -- nose=1.3.7 -- numexpr=2.6.2 -- numpy=1.12.0 -- openblas=0.2.19 -- pandas=0.19.2 -- patsy=0.4.1 -- psycopg2=2.6.2 -- pycrypto=2.6.1 -- pytest=3.0.5 -- python=3.6.0 -- pyyaml=3.12 -- requests=2.12.4 -- seaborn=0.7.1 -- scipy=0.18.1 -- scikit-learn=0.18.1 -- statsmodels=0.8.0 -- pip: - - awscli==1.11.60 - - civis==1.3.0 - - dropbox==7.1.1 - - ftputil==3.3.1 - - glmnet==2.0.0 - - joblib==0.11.0 - - muffnn==1.0.0 - - pubnub==4.0.8 - - pysftp==0.2.9 - - python-simple-hipchat==0.4.0 - - requests-toolbelt==0.7.1 - - tensorflow==1.0.0 - - urllib3==1.19 - - xgboost==0.6a2 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7c43033 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,37 @@ +beautifulsoup4==4.5.3 +boto==2.45.0 +boto3==1.4.3 +cython==0.25.2 +ipython==5.1.0 +jinja2==2.8 +jsonschema==2.5.1 +jupyter==1.0.0 +matplotlib==2.0.0 +nose==1.3.7 +numexpr==2.6.2 +numpy==1.12.0 +pandas==0.19.2 +patsy==0.4.1 +psycopg2==2.6.2 +pycrypto==2.6.1 +pytest==3.0.5 +pyyaml==3.12 +requests==2.12.4 +seaborn==0.7.1 +scipy==0.18.1 +scikit-learn==0.18.1 +statsmodels==0.8.0 +awscli==1.11.60 +civis==1.3.0 +dropbox==7.1.1 +ftputil==3.3.1 +glmnet==2.0.0 +joblib==0.11.0 +muffnn==1.0.0 +pubnub==4.0.8 +pysftp==0.2.9 +python-simple-hipchat==0.4.0 +requests-toolbelt==0.7.1 +tensorflow==1.0.0 +urllib3==1.19 +xgboost==0.6a2 From 01798a724303948530f20282605d12619181a88b Mon Sep 17 00:00:00 2001 From: beckermr Date: Tue, 14 Mar 2017 14:40:11 -0500 Subject: [PATCH 02/13] DEP remove old code --- Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 44c3d94..2f998b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,11 +13,6 @@ RUN apt-get purge -y python.* # > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. ENV LANG=C.UTF-8 \ BASH_ENV=/etc/profile -# Set environment variables for UTF-8, conda, and shell environments -#ENV LANG=en_US.UTF-8 \ -# LANGUAGE=en_US:en \ -# LC_ALL=en_US.UTF-8 \ -# BASH_ENV=/etc/profile RUN apt-get update && \ apt-get install -y --no-install-recommends \ From 942d6fb7106625934e2f569d9437f491c9bc7959 Mon Sep 17 00:00:00 2001 From: beckermr Date: Thu, 16 Mar 2017 10:22:37 -0500 Subject: [PATCH 03/13] ENH move to docker build straight from the source! --- Dockerfile | 207 +++++++++++++++++++++++++++++------------------------ 1 file changed, 115 insertions(+), 92 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2f998b9..276a4fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,113 +1,136 @@ -FROM ubuntu:14.04 +FROM buildpack-deps:trusty-curl MAINTAINER support@civisanalytics.com -# We need this file to install the right version of numpy. -COPY requirements.txt /requirements.txt - -# Python install by wlattner. +# Docker build based on Docker official python-slim build. +# +# Copyright (c) 2014-2015 Docker, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# Eemove several traces of debian python. -RUN apt-get purge -y python.* +# ensure local python is preferred over distribution python +ENV BASH_ENV=/etc/profile \ + PATH=/usr/local/bin:$PATH # http://bugs.python.org/issue19846 # > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. -ENV LANG=C.UTF-8 \ - BASH_ENV=/etc/profile - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - libsqlite3-0 \ - libssl1.0.0 \ - libgfortran3 \ - git \ - postgresql \ - postgresql-contrib \ - libpq-dev && \ - rm -rf /var/lib/apt/lists/* +ENV LANG C.UTF-8 -# We like PGP and so are going through this pain. Notes for the future. Yay! -# I *think* this is how this works. -# -# 1) Figure out who the release manager is for your python version. (I *think* they usually make -# a PEP for it, because, like, they make a PEP for everything. See here for 3.6.0 -# https://www.python.org/dev/peps/pep-0494/.) -# 2) Then go look them up on a key server (This one is nice https://pgp.mit.edu/) -# 3) Search by their name and find the short 8 character key ID. You might have to try a few -# different keys since people can have more than one and it is not always clear which one -# was used to sign the package. -# 4) Put the 8 character key ID below. -# -# If everything goes well, you will successfully verify that the download (over https) in fact -# returned the correct, uncorrupted file. Yay! -ENV GPG_KEY AA65421D +# runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgdbm3 \ + libsqlite3-0 \ + libssl1.0.0 \ + make \ + automake \ + gcc \ + g++ \ + gfortran \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* +ENV GPG_KEY 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D ENV PYTHON_VERSION 3.6.0 -# If this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" +# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" ENV PYTHON_PIP_VERSION 9.0.1 -RUN set -ex && \ - buildDeps=' \ - curl \ - gcc \ - g++ \ - gfortran \ - libbz2-dev \ - libc6-dev \ - liblzma-dev \ - libncurses-dev \ - libreadline-dev \ - libsqlite3-dev \ - libssl-dev \ - make \ - xz-utils \ - zlib1g-dev' && \ - apt-get update && \ - apt-get install -y $buildDeps --no-install-recommends && \ - rm -rf /var/lib/apt/lists/* && \ - curl -fSL "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" -o python.tar.xz && \ - curl -fSL "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" -o python.tar.xz.asc && \ - export GNUPGHOME="$(mktemp -d)" && \ - gpg --keyserver pgp.mit.edu --recv-keys "$GPG_KEY" && \ - gpg --batch --verify python.tar.xz.asc python.tar.xz && \ - rm -r "$GNUPGHOME" python.tar.xz.asc && \ - mkdir -p /usr/src/python && \ - tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz && \ - rm python.tar.xz && \ - \ - cd /usr/src/python && \ - ./configure --enable-shared --enable-unicode=ucs4 && \ - make -j$(nproc) && \ - make install && \ - ldconfig && \ - pip3 install --no-cache-dir --upgrade --ignore-installed pip==$PYTHON_PIP_VERSION && \ - pip3 install --no-cache-dir `grep 'numpy' /requirements.txt` && \ - find /usr/local -depth \ - \( \ - \( -type d -a -name test -o -name tests \) \ - -o \ - \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \ - \) -exec rm -rf '{}' + && \ - rm -rf /usr/src/python ~/.cache +RUN set -ex \ + && buildDeps=' \ + libbz2-dev \ + libc6-dev \ + libgdbm-dev \ + liblzma-dev \ + libncurses-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + tcl-dev \ + tk-dev \ + xz-utils \ + zlib1g-dev \ + ' \ + && apt-get update && apt-get install -y $buildDeps --no-install-recommends && rm -rf /var/lib/apt/lists/* \ + \ + && wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \ + && wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \ + && export GNUPGHOME="$(mktemp -d)" \ + && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEY" \ + && gpg --batch --verify python.tar.xz.asc python.tar.xz \ + && rm -r "$GNUPGHOME" python.tar.xz.asc \ + && mkdir -p /usr/src/python \ + && tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \ + && rm python.tar.xz \ + \ + && cd /usr/src/python \ + && ./configure \ + --enable-loadable-sqlite-extensions \ + --enable-shared + --enable-optimization \ + && make -j$(nproc) \ + && make install \ + && ldconfig \ + \ +# explicit path to "pip3" to ensure distribution-provided "pip3" cannot interfere + && if [ ! -e /usr/local/bin/pip3 ]; then : \ + && wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ + && python3 /tmp/get-pip.py "pip==$PYTHON_PIP_VERSION" \ + && rm /tmp/get-pip.py \ + ; fi \ +# we use "--force-reinstall" for the case where the version of pip we're trying to install is the same as the version bundled with Python +# ("Requirement already up-to-date: pip==8.1.2 in /usr/local/lib/python3.6/site-packages") +# https://github.com/docker-library/python/pull/143#issuecomment-241032683 + && pip3 install --no-cache-dir --upgrade --force-reinstall "pip==$PYTHON_PIP_VERSION" \ +# then we use "pip list" to ensure we don't have more than one pip version installed +# https://github.com/docker-library/python/pull/100 + && [ "$(pip list |tac|tac| awk -F '[ ()]+' '$1 == "pip" { print $2; exit }')" = "$PYTHON_PIP_VERSION" ] \ + \ + && find /usr/local -depth \ + \( \ + \( -type d -a -name test -o -name tests \) \ + -o \ + \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \ + \) -exec rm -rf '{}' + \ + && apt-get purge -y --auto-remove $buildDeps \ + && rm -rf /usr/src/python ~/.cache -# Make some useful symlinks that are expected to exist. -RUN cd /usr/local/bin && \ - ln -s python3 python && \ - ln -s python3-config python-config +# make some useful symlinks that are expected to exist +RUN cd /usr/local/bin \ + && { [ -e easy_install ] || ln -s easy_install-* easy_install; } \ + && ln -s idle3 idle \ + && ln -s pydoc3 pydoc \ + && ln -s python3 python \ + && ln -s python3-config python-config -# Now install the rest of the dependencies from the requirements file. -RUN pip install --no-cache-dir -r /requirements.txt -RUN rm -rf ~/.cache/* +# Now install the dependencies from the requirements file. +COPY requirements.txt /requirements.txt +RUN pip install --no-cache-dir `grep 'numpy' /requirements.txt` && \ + pip install --no-cache-dir -r /requirements.txt && \ + rm -rf ~/.cache/* # We aren't running a GUI, so force matplotlib to use # the non-interactive "Agg" backend for graphics. +# Finally, run matplotlib once to build the font cache. ENV MATPLOTLIBRC=${HOME}/.config/matplotlib/matplotlibrc -RUN mkdir -p `dirname ${MATPLOTLIBRC}` -RUN echo "backend : Agg" > ${MATPLOTLIBRC} - -# Run matplotlib once to build the font cache. -RUN python -c "import matplotlib.pyplot" +RUN mkdir -p `dirname ${MATPLOTLIBRC}` && \ + echo "backend : Agg" > ${MATPLOTLIBRC} && \ + python -c "import matplotlib.pyplot" ENV VERSION=3.0.0 \ VERSION_MAJOR=3 \ From fba06f1d46065527e3f58bb417ccb1fff2fc507f Mon Sep 17 00:00:00 2001 From: beckermr Date: Thu, 16 Mar 2017 10:29:43 -0500 Subject: [PATCH 04/13] BUG silly bug --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 276a4fe..7b758b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,8 +81,8 @@ RUN set -ex \ && cd /usr/src/python \ && ./configure \ --enable-loadable-sqlite-extensions \ - --enable-shared - --enable-optimization \ + --enable-shared \ + --enable-optimizations \ && make -j$(nproc) \ && make install \ && ldconfig \ From 8c32807b35d72f2c52e326ba0b87e1cd4ddacb01 Mon Sep 17 00:00:00 2001 From: beckermr Date: Thu, 16 Mar 2017 10:37:30 -0500 Subject: [PATCH 05/13] ENH tabs to spaces, adding libffi --- Dockerfile | 119 +++++++++++++++++++++++++++-------------------------- 1 file changed, 60 insertions(+), 59 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7b758b9..76e3f34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,16 +34,16 @@ ENV LANG C.UTF-8 # runtime dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ - libgdbm3 \ - libsqlite3-0 \ - libssl1.0.0 \ + libgdbm3 \ + libsqlite3-0 \ + libssl1.0.0 \ make \ automake \ gcc \ g++ \ gfortran \ libpq-dev \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* ENV GPG_KEY 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D ENV PYTHON_VERSION 3.6.0 @@ -52,71 +52,72 @@ ENV PYTHON_VERSION 3.6.0 ENV PYTHON_PIP_VERSION 9.0.1 RUN set -ex \ - && buildDeps=' \ - libbz2-dev \ - libc6-dev \ - libgdbm-dev \ - liblzma-dev \ - libncurses-dev \ - libreadline-dev \ - libsqlite3-dev \ - libssl-dev \ - tcl-dev \ - tk-dev \ - xz-utils \ - zlib1g-dev \ - ' \ - && apt-get update && apt-get install -y $buildDeps --no-install-recommends && rm -rf /var/lib/apt/lists/* \ - \ - && wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \ - && wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \ - && export GNUPGHOME="$(mktemp -d)" \ - && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEY" \ - && gpg --batch --verify python.tar.xz.asc python.tar.xz \ - && rm -r "$GNUPGHOME" python.tar.xz.asc \ - && mkdir -p /usr/src/python \ - && tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \ - && rm python.tar.xz \ - \ - && cd /usr/src/python \ - && ./configure \ - --enable-loadable-sqlite-extensions \ - --enable-shared \ + && buildDeps=' \ + libffi-dev \ + libbz2-dev \ + libc6-dev \ + libgdbm-dev \ + liblzma-dev \ + libncurses-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + tcl-dev \ + tk-dev \ + xz-utils \ + zlib1g-dev \ + ' \ + && apt-get update && apt-get install -y $buildDeps --no-install-recommends && rm -rf /var/lib/apt/lists/* \ + \ + && wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \ + && wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \ + && export GNUPGHOME="$(mktemp -d)" \ + && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEY" \ + && gpg --batch --verify python.tar.xz.asc python.tar.xz \ + && rm -r "$GNUPGHOME" python.tar.xz.asc \ + && mkdir -p /usr/src/python \ + && tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \ + && rm python.tar.xz \ + \ + && cd /usr/src/python \ + && ./configure \ + --enable-loadable-sqlite-extensions \ + --enable-shared \ --enable-optimizations \ - && make -j$(nproc) \ - && make install \ - && ldconfig \ - \ + && make -j$(nproc) \ + && make install \ + && ldconfig \ + \ # explicit path to "pip3" to ensure distribution-provided "pip3" cannot interfere - && if [ ! -e /usr/local/bin/pip3 ]; then : \ - && wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ - && python3 /tmp/get-pip.py "pip==$PYTHON_PIP_VERSION" \ - && rm /tmp/get-pip.py \ - ; fi \ + && if [ ! -e /usr/local/bin/pip3 ]; then : \ + && wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ + && python3 /tmp/get-pip.py "pip==$PYTHON_PIP_VERSION" \ + && rm /tmp/get-pip.py \ + ; fi \ # we use "--force-reinstall" for the case where the version of pip we're trying to install is the same as the version bundled with Python # ("Requirement already up-to-date: pip==8.1.2 in /usr/local/lib/python3.6/site-packages") # https://github.com/docker-library/python/pull/143#issuecomment-241032683 - && pip3 install --no-cache-dir --upgrade --force-reinstall "pip==$PYTHON_PIP_VERSION" \ + && pip3 install --no-cache-dir --upgrade --force-reinstall "pip==$PYTHON_PIP_VERSION" \ # then we use "pip list" to ensure we don't have more than one pip version installed # https://github.com/docker-library/python/pull/100 - && [ "$(pip list |tac|tac| awk -F '[ ()]+' '$1 == "pip" { print $2; exit }')" = "$PYTHON_PIP_VERSION" ] \ - \ - && find /usr/local -depth \ - \( \ - \( -type d -a -name test -o -name tests \) \ - -o \ - \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \ - \) -exec rm -rf '{}' + \ - && apt-get purge -y --auto-remove $buildDeps \ - && rm -rf /usr/src/python ~/.cache + && [ "$(pip list |tac|tac| awk -F '[ ()]+' '$1 == "pip" { print $2; exit }')" = "$PYTHON_PIP_VERSION" ] \ + \ + && find /usr/local -depth \ + \( \ + \( -type d -a -name test -o -name tests \) \ + -o \ + \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \ + \) -exec rm -rf '{}' + \ + && apt-get purge -y --auto-remove $buildDeps \ + && rm -rf /usr/src/python ~/.cache # make some useful symlinks that are expected to exist RUN cd /usr/local/bin \ - && { [ -e easy_install ] || ln -s easy_install-* easy_install; } \ - && ln -s idle3 idle \ - && ln -s pydoc3 pydoc \ - && ln -s python3 python \ - && ln -s python3-config python-config + && { [ -e easy_install ] || ln -s easy_install-* easy_install; } \ + && ln -s idle3 idle \ + && ln -s pydoc3 pydoc \ + && ln -s python3 python \ + && ln -s python3-config python-config # Now install the dependencies from the requirements file. COPY requirements.txt /requirements.txt From 994ef93cd50b7fcb44587ebc26ebca633cbfa7df Mon Sep 17 00:00:00 2001 From: beckermr Date: Thu, 16 Mar 2017 10:46:33 -0500 Subject: [PATCH 06/13] BUG maybe this is it? --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 76e3f34..0dbc6aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -83,7 +83,6 @@ RUN set -ex \ && ./configure \ --enable-loadable-sqlite-extensions \ --enable-shared \ - --enable-optimizations \ && make -j$(nproc) \ && make install \ && ldconfig \ From 555300d3568d9f1bb24967ca0834baa56c373dd9 Mon Sep 17 00:00:00 2001 From: beckermr Date: Thu, 16 Mar 2017 10:59:44 -0500 Subject: [PATCH 07/13] BUg add postgressql --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 0dbc6aa..9e03e42 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,6 +43,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ g++ \ gfortran \ libpq-dev \ + postgresql \ + postgresql-contrib \ && rm -rf /var/lib/apt/lists/* ENV GPG_KEY 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D @@ -83,6 +85,7 @@ RUN set -ex \ && ./configure \ --enable-loadable-sqlite-extensions \ --enable-shared \ + --with-lto \ && make -j$(nproc) \ && make install \ && ldconfig \ From 3f4488ff099efb8ea547579a0914379be73040b7 Mon Sep 17 00:00:00 2001 From: beckermr Date: Thu, 16 Mar 2017 11:23:47 -0500 Subject: [PATCH 08/13] ENH added more deps I hope --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 9e03e42..a02adff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpq-dev \ postgresql \ postgresql-contrib \ + postgresql-server-dev \ && rm -rf /var/lib/apt/lists/* ENV GPG_KEY 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D From 486cd73331d5467a57e0259dee665bafa9669dd9 Mon Sep 17 00:00:00 2001 From: beckermr Date: Thu, 16 Mar 2017 11:41:14 -0500 Subject: [PATCH 09/13] ENH try again... --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index a02adff..39aa473 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpq-dev \ postgresql \ postgresql-contrib \ - postgresql-server-dev \ + postgresql-server-dev-9.3 \ + libc6-dev \ + libssl-dev \ + zlib1g-dev \ && rm -rf /var/lib/apt/lists/* ENV GPG_KEY 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D @@ -58,17 +61,14 @@ RUN set -ex \ && buildDeps=' \ libffi-dev \ libbz2-dev \ - libc6-dev \ libgdbm-dev \ liblzma-dev \ libncurses-dev \ libreadline-dev \ libsqlite3-dev \ - libssl-dev \ tcl-dev \ tk-dev \ xz-utils \ - zlib1g-dev \ ' \ && apt-get update && apt-get install -y $buildDeps --no-install-recommends && rm -rf /var/lib/apt/lists/* \ \ From e23b1ac3baccf4835aca5da6ebd21eda6f451abd Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Fri, 17 Mar 2017 09:24:01 -0500 Subject: [PATCH 10/13] ENH add miniconda and git --- Dockerfile | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 39aa473..a4470ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,17 +26,20 @@ MAINTAINER support@civisanalytics.com # ensure local python is preferred over distribution python ENV BASH_ENV=/etc/profile \ - PATH=/usr/local/bin:$PATH + PATH=/opt/conda/bin:/usr/local/bin:$PATH \ + CIVIS_CONDA_VERSION=4.3.11 \ + PYTHON_VERSION=3.6.0 # http://bugs.python.org/issue19846 # > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. ENV LANG C.UTF-8 # runtime dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y --no-install-recommends \ libgdbm3 \ libsqlite3-0 \ libssl1.0.0 \ + git \ make \ automake \ gcc \ @@ -52,12 +55,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* ENV GPG_KEY 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D -ENV PYTHON_VERSION 3.6.0 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" ENV PYTHON_PIP_VERSION 9.0.1 -RUN set -ex \ +RUN DEBIAN_FRONTEND=noninteractive set -ex \ && buildDeps=' \ libffi-dev \ libbz2-dev \ @@ -122,6 +124,22 @@ RUN cd /usr/local/bin \ && ln -s python3 python \ && ln -s python3-config python-config +RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ + wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${CIVIS_CONDA_VERSION}-Linux-x86_64.sh && \ + /bin/bash /Miniconda3-${CIVIS_CONDA_VERSION}-Linux-x86_64.sh -b -p /opt/conda && \ + rm Miniconda3-${CIVIS_CONDA_VERSION}-Linux-x86_64.sh && \ + /opt/conda/bin/conda install --yes conda==${CIVIS_CONDA_VERSION} && \ + echo "conda ==${CIVIS_CONDA_VERSION}" > /opt/conda/conda-meta/pinned && \ + conda install -y nomkl && \ + conda clean --all -y + +# Red Hat and Debian use different names for this file. git2R wants the latter. +# See conda-recipes GH 423 +RUN ln -s /opt/conda/lib/libopenblas.so /opt/conda/lib/libblas.so && \ + ln -s /opt/conda/lib/libopenblas.so /opt/conda/lib/liblapack.so && \ + ln -s /opt/conda/lib/libssl.so /opt/conda/lib/libssl.so.6 && \ + ln -s /opt/conda/lib/libcrypto.so /opt/conda/lib/libcrypto.so.6 + # Now install the dependencies from the requirements file. COPY requirements.txt /requirements.txt RUN pip install --no-cache-dir `grep 'numpy' /requirements.txt` && \ From f17cdd3d1c7f6105e78e0142ffafb009373020c2 Mon Sep 17 00:00:00 2001 From: beckermr Date: Fri, 17 Mar 2017 09:31:43 -0500 Subject: [PATCH 11/13] ENH more conda stuff --- .condarc | 7 +++++++ Dockerfile | 12 ++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 .condarc diff --git a/.condarc b/.condarc new file mode 100644 index 0000000..b71c0f8 --- /dev/null +++ b/.condarc @@ -0,0 +1,7 @@ +channels: + - defaults + +show_channel_urls: True + +create_default_packages: + - nomkl \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index a4470ae..c84c92e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,8 @@ MAINTAINER support@civisanalytics.com ENV BASH_ENV=/etc/profile \ PATH=/opt/conda/bin:/usr/local/bin:$PATH \ CIVIS_CONDA_VERSION=4.3.11 \ - PYTHON_VERSION=3.6.0 + PYTHON_VERSION=3.6.0 \ + CONDARC=/opt/conda/.condarc # http://bugs.python.org/issue19846 # > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. @@ -124,6 +125,7 @@ RUN cd /usr/local/bin \ && ln -s python3 python \ && ln -s python3-config python-config +# A simple miniconda install for those who want it... RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${CIVIS_CONDA_VERSION}-Linux-x86_64.sh && \ /bin/bash /Miniconda3-${CIVIS_CONDA_VERSION}-Linux-x86_64.sh -b -p /opt/conda && \ @@ -132,13 +134,7 @@ RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ echo "conda ==${CIVIS_CONDA_VERSION}" > /opt/conda/conda-meta/pinned && \ conda install -y nomkl && \ conda clean --all -y - -# Red Hat and Debian use different names for this file. git2R wants the latter. -# See conda-recipes GH 423 -RUN ln -s /opt/conda/lib/libopenblas.so /opt/conda/lib/libblas.so && \ - ln -s /opt/conda/lib/libopenblas.so /opt/conda/lib/liblapack.so && \ - ln -s /opt/conda/lib/libssl.so /opt/conda/lib/libssl.so.6 && \ - ln -s /opt/conda/lib/libcrypto.so /opt/conda/lib/libcrypto.so.6 +COPY .condarc /opt/conda/.condarc # Now install the dependencies from the requirements file. COPY requirements.txt /requirements.txt From 0c770a84f4cb302d5c48e860fc12a2aaad9d9c67 Mon Sep 17 00:00:00 2001 From: beckermr Date: Fri, 17 Mar 2017 09:32:53 -0500 Subject: [PATCH 12/13] ENH roll back condarc changes --- .condarc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.condarc b/.condarc index b71c0f8..364cb1a 100644 --- a/.condarc +++ b/.condarc @@ -1,7 +1,7 @@ channels: - - defaults + - defaults show_channel_urls: True create_default_packages: - - nomkl \ No newline at end of file + - nomkl From f711221a1f138a5dc192451a1c9b98366d4c463d Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Fri, 17 Mar 2017 09:45:36 -0500 Subject: [PATCH 13/13] ENH force conda to remove python --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c84c92e..0b4a8a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -132,7 +132,7 @@ RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ rm Miniconda3-${CIVIS_CONDA_VERSION}-Linux-x86_64.sh && \ /opt/conda/bin/conda install --yes conda==${CIVIS_CONDA_VERSION} && \ echo "conda ==${CIVIS_CONDA_VERSION}" > /opt/conda/conda-meta/pinned && \ - conda install -y nomkl && \ + conda uninstall python -y && \ conda clean --all -y COPY .condarc /opt/conda/.condarc