civisanalytics · beckermr · Mar 10, 2017 · Mar 14, 2017 · Mar 16, 2017 · Mar 16, 2017
diff --git a/Dockerfile b/Dockerfile
@@ -1,94 +1,156 @@
-FROM ubuntu:14.04
+FROM buildpack-deps:trusty-curl
 MAINTAINER support@civisanalytics.com
 
-# Ensure UTF-8 locale.
-RUN locale-gen en_US.UTF-8
-
-# Set environment variables for UTF-8, conda, and shell environments
-ENV LANG=en_US.UTF-8 \
-    LANGUAGE=en_US:en \
-    LC_ALL=en_US.UTF-8 \
-    CONDARC=/opt/conda/.condarc \
-    BASH_ENV=/etc/profile \
-    PATH=/opt/conda/bin:$PATH \
-    CIVIS_CONDA_VERSION=4.3.11 \
-    CIVIS_PYTHON_VERSION=3.6.0
-
-RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && \
-  apt-get install -y --no-install-recommends software-properties-common && \
-  apt-get install -y --no-install-recommends \
-      make \
-      automake \
-      libpq-dev \
-      libffi-dev \
-      gfortran \
-      g++ \
-      git \
-      libboost-program-options-dev \
-      libtool \
-      libxrender1 \
-      wget \
-      ca-certificates \
-      curl
-
-# Conda install.
+# Docker build based on Docker official python-slim build.
 #
-# Everything is installed in the root environment. This allows for
-# upgrades to the packages and eliminates the pain of trying to activate
-# some other environment automatically for the many different ways
-# people can use a docker image.
+# Copyright (c) 2014-2015 Docker, Inc.
 #
-# Things are pinned to prevent upgrades from conda and force it to
-# resolve dependencies relative to a fixed conda & python version.
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
 #
-# Note that the python version is also listed in the enviornment.yml
-# file. The version in CIVIS_PYTHON_VERSION is the source of truth.
-# If you want to change the python version, you need to change it in
-# **both** places. The python version has been left in the `environment.yml`
-# file so that people can create environments equivalent to this
-# container. However, change the name of the environment first!
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
 #
-# The ordering of these steps seems to matter. You seem to have to
-# install a specific python version by hand and then pin it.
-# 1) install conda
-# 2) pin conda to the version given by CIVIS_CONDA_VERSION
-# 3) install the python version CIVIS_PYTHON_VERSION
-# 4) pin the python version
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# ensure local python is preferred over distribution python
+ENV BASH_ENV=/etc/profile \
+    PATH=/opt/conda/bin:/usr/local/bin:$PATH \
+    CIVIS_CONDA_VERSION=4.3.11 \
+    PYTHON_VERSION=3.6.0 \
+    CONDARC=/opt/conda/.condarc
+
+# http://bugs.python.org/issue19846
+# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
+ENV LANG C.UTF-8
+
+# runtime dependencies
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y --no-install-recommends \
+        libgdbm3 \
+        libsqlite3-0 \
+        libssl1.0.0 \
+        git \
+        make \
+        automake \
+        gcc \
+        g++ \
+        gfortran \
+        libpq-dev \
+        postgresql \
+        postgresql-contrib \
+        postgresql-server-dev-9.3 \
+        libc6-dev \
+        libssl-dev \
+        zlib1g-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV GPG_KEY 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D
+
+# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
+ENV PYTHON_PIP_VERSION 9.0.1
+
+RUN DEBIAN_FRONTEND=noninteractive set -ex \
+    && buildDeps=' \
+        libffi-dev \
+        libbz2-dev \
+        libgdbm-dev \
+        liblzma-dev \
+        libncurses-dev \
+        libreadline-dev \
+        libsqlite3-dev \
+        tcl-dev \
+        tk-dev \
+        xz-utils \
+    ' \
+    && apt-get update && apt-get install -y $buildDeps --no-install-recommends && rm -rf /var/lib/apt/lists/* \
+    \
+    && wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
+    && wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
+    && export GNUPGHOME="$(mktemp -d)" \
+    && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEY" \
+    && gpg --batch --verify python.tar.xz.asc python.tar.xz \
+    && rm -r "$GNUPGHOME" python.tar.xz.asc \
+    && mkdir -p /usr/src/python \
+    && tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \
+    && rm python.tar.xz \
+    \
+    && cd /usr/src/python \
+    && ./configure \
+        --enable-loadable-sqlite-extensions \
+        --enable-shared \
+        --with-lto \
+    && make -j$(nproc) \
+    && make install \
+    && ldconfig \
+    \
+# explicit path to "pip3" to ensure distribution-provided "pip3" cannot interfere
+    && if [ ! -e /usr/local/bin/pip3 ]; then : \
+        && wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \
+        && python3 /tmp/get-pip.py "pip==$PYTHON_PIP_VERSION" \
+        && rm /tmp/get-pip.py \
+    ; fi \
+# we use "--force-reinstall" for the case where the version of pip we're trying to install is the same as the version bundled with Python
+# ("Requirement already up-to-date: pip==8.1.2 in /usr/local/lib/python3.6/site-packages")
+# https://github.com/docker-library/python/pull/143#issuecomment-241032683
+    && pip3 install --no-cache-dir --upgrade --force-reinstall "pip==$PYTHON_PIP_VERSION" \
+# then we use "pip list" to ensure we don't have more than one pip version installed
+# https://github.com/docker-library/python/pull/100
+    && [ "$(pip list |tac|tac| awk -F '[ ()]+' '$1 == "pip" { print $2; exit }')" = "$PYTHON_PIP_VERSION" ] \
+    \
+    && find /usr/local -depth \
+        \( \
+            \( -type d -a -name test -o -name tests \) \
+            -o \
+            \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \
+        \) -exec rm -rf '{}' + \
+    && apt-get purge -y --auto-remove $buildDeps \
+    && rm -rf /usr/src/python ~/.cache
+
+# make some useful symlinks that are expected to exist
+RUN cd /usr/local/bin \
+    && { [ -e easy_install ] || ln -s easy_install-* easy_install; } \
+    && ln -s idle3 idle \
+    && ln -s pydoc3 pydoc \
+    && ln -s python3 python \
+    && ln -s python3-config python-config
+
+# A simple miniconda install for those who want it...
 RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \
     wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${CIVIS_CONDA_VERSION}-Linux-x86_64.sh && \
     /bin/bash /Miniconda3-${CIVIS_CONDA_VERSION}-Linux-x86_64.sh -b -p /opt/conda && \
     rm Miniconda3-${CIVIS_CONDA_VERSION}-Linux-x86_64.sh && \
     /opt/conda/bin/conda install --yes conda==${CIVIS_CONDA_VERSION} && \
     echo "conda ==${CIVIS_CONDA_VERSION}" > /opt/conda/conda-meta/pinned && \
-    conda install --yes python==${CIVIS_PYTHON_VERSION} && \
-    echo "\npython ==${CIVIS_PYTHON_VERSION}" >> /opt/conda/conda-meta/pinned
-
-# Red Hat and Debian use different names for this file. git2R wants the latter.
-# See conda-recipes GH 423
-RUN ln -s /opt/conda/lib/libopenblas.so /opt/conda/lib/libblas.so && \
-    ln -s /opt/conda/lib/libopenblas.so /opt/conda/lib/liblapack.so && \
-    ln -s /opt/conda/lib/libssl.so /opt/conda/lib/libssl.so.6 && \
-    ln -s /opt/conda/lib/libcrypto.so /opt/conda/lib/libcrypto.so.6
-
-# Install boto in the base environment for private s3 channel support.
-# Install Python Packages
-COPY .condarc /opt/conda/.condarc
-COPY environment.yml environment.yml
-RUN conda install -y boto && \
-    conda install -y nomkl && \
-    conda env update -f environment.yml && \
+    conda uninstall python -y && \
     conda clean --all -y
+COPY .condarc /opt/conda/.condarc
+
+# Now install the dependencies from the requirements file.
+COPY requirements.txt /requirements.txt
+RUN pip install --no-cache-dir `grep 'numpy' /requirements.txt` && \
+    pip install --no-cache-dir -r /requirements.txt && \
+    rm -rf ~/.cache/*
 
 # We aren't running a GUI, so force matplotlib to use
 # the non-interactive "Agg" backend for graphics.
+# Finally, run matplotlib once to build the font cache.
 ENV MATPLOTLIBRC=${HOME}/.config/matplotlib/matplotlibrc
-RUN mkdir -p ${HOME}/.config/matplotlib
-RUN echo "backend      : Agg" > ${HOME}/.config/matplotlib/matplotlibrc
-
-# Run matplotlib once to build the font cache
-RUN python -c "import matplotlib.pyplot"
+RUN mkdir -p `dirname ${MATPLOTLIBRC}` && \
+    echo "backend      : Agg" > ${MATPLOTLIBRC} && \
+    python -c "import matplotlib.pyplot"
 
-ENV VERSION=2.0.1 \
-    VERSION_MAJOR=2 \
+ENV VERSION=3.0.0 \
+    VERSION_MAJOR=3 \
     VERSION_MINOR=0 \
-    VERSION_MICRO=1
+    VERSION_MICRO=0
diff --git a/environment.yml b/environment.yml
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,37 @@
+beautifulsoup4==4.5.3
+boto==2.45.0
+boto3==1.4.3
+cython==0.25.2
+ipython==5.1.0
+jinja2==2.8
+jsonschema==2.5.1
+jupyter==1.0.0
+matplotlib==2.0.0
+nose==1.3.7
+numexpr==2.6.2
+numpy==1.12.0
+pandas==0.19.2
+patsy==0.4.1
+psycopg2==2.6.2
+pycrypto==2.6.1
+pytest==3.0.5
+pyyaml==3.12
+requests==2.12.4
+seaborn==0.7.1
+scipy==0.18.1
+scikit-learn==0.18.1
+statsmodels==0.8.0
+awscli==1.11.60
+civis==1.3.0
+dropbox==7.1.1
+ftputil==3.3.1
+glmnet==2.0.0
+joblib==0.11.0
+muffnn==1.0.0
+pubnub==4.0.8
+pysftp==0.2.9
+python-simple-hipchat==0.4.0
+requests-toolbelt==0.7.1
+tensorflow==1.0.0
+urllib3==1.19
+xgboost==0.6a2