From 4cbf73d8b9584f84dea8eaaa4eba712c82f6eba3 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 10:12:12 -0800 Subject: [PATCH 01/21] update .gitignore and .dockerignore --- .dockerignore | 40 ++++++++++++++++++++++++++++++++++++---- .gitignore | 3 ++- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/.dockerignore b/.dockerignore index f41447e32c..4d176df9e5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,39 @@ -.git/ -*.pyc -*.egg-info +# python cache +__pycache__/**/* __pycache__ +*.pyc + +# python distribution +build/**/* +build +dist/**/* +dist +nipype.egg-info/**/* +nipype.egg-info +.eggs/**/* +.eggs +src/**/* +src/ docker/nipype_* docker/test-* -.coverage \ No newline at end of file + +# releasing +Makefile + +# git +.gitignore +.git/**/* +.git + +# other +docs/**/* +docs/ +.coverage +.coveragerc +codecov.yml +rtd_requirements.txt +circle.yml +Vagrantfile +.travis.yml +.noserc + diff --git a/.gitignore b/.gitignore index 8d472e7389..4e07cd5b27 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ .project .settings .pydevproject +.eggs .idea/ /documentation.zip .DS_Store @@ -24,4 +25,4 @@ htmlcov/ __pycache__/ *~ .ipynb_checkpoints/ -.ruby-version \ No newline at end of file +.ruby-version From e56151aecb353fc9d9444908604186dee042762d Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 10:57:51 -0800 Subject: [PATCH 02/21] first revision of general Dockerfile --- Dockerfile | 218 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 183 insertions(+), 35 deletions(-) diff --git a/Dockerfile b/Dockerfile index 478f09eda8..d742e335a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,49 +26,197 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM nipype/testnipypedata:latest -MAINTAINER Stanford Center for Reproducible Neuroscience -# Preparations -RUN ln -snf /bin/bash /bin/sh +# +# Based on https://github.com/poldracklab/fmriprep/blob/9c92a3de9112f8ef1655b876de060a2ad336ffb0/Dockerfile +# +FROM ubuntu:xenial-20161213 + +# Prepare environment +RUN apt-get update && \ + apt-get install -y --no-install-recommends curl bzip2 ca-certificates xvfb && \ + curl -sSL http://neuro.debian.net/lists/xenial.us-ca.full >> /etc/apt/sources.list.d/neurodebian.sources.list && \ + apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 && \ + apt-get update + +# Installing freesurfer +RUN curl -sSL https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.0/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.0.tar.gz | tar zxv -C /opt \ + --exclude='freesurfer/trctrain' \ + --exclude='freesurfer/subjects/fsaverage_sym' \ + --exclude='freesurfer/subjects/fsaverage3' \ + --exclude='freesurfer/subjects/fsaverage4' \ + --exclude='freesurfer/subjects/fsaverage5' \ + --exclude='freesurfer/subjects/fsaverage6' \ + --exclude='freesurfer/subjects/cvs_avg35' \ + --exclude='freesurfer/subjects/cvs_avg35_inMNI152' \ + --exclude='freesurfer/subjects/bert' \ + --exclude='freesurfer/subjects/V1_average' \ + --exclude='freesurfer/average/mult-comp-cor' \ + --exclude='freesurfer/lib/cuda' \ + --exclude='freesurfer/lib/qt' + +ENV FSL_DIR=/usr/share/fsl/5.0 \ + OS=Linux \ + FS_OVERRIDE=0 \ + FIX_VERTEX_AREA= \ + FSF_OUTPUT_FORMAT=nii.gz \ + FREESURFER_HOME=/opt/freesurfer +ENV SUBJECTS_DIR=$FREESURFER_HOME/subjects \ + FUNCTIONALS_DIR=$FREESURFER_HOME/sessions \ + MNI_DIR=$FREESURFER_HOME/mni \ + LOCAL_DIR=$FREESURFER_HOME/local \ + FSFAST_HOME=$FREESURFER_HOME/fsfast \ + MINC_BIN_DIR=$FREESURFER_HOME/mni/bin \ + MINC_LIB_DIR=$FREESURFER_HOME/mni/lib \ + MNI_DATAPATH=$FREESURFER_HOME/mni/data \ + FMRI_ANALYSIS_DIR=$FREESURFER_HOME/fsfast +ENV PERL5LIB=$MINC_LIB_DIR/perl5/5.8.5 \ + MNI_PERL5LIB=$MINC_LIB_DIR/perl5/5.8.5 \ + PATH=$FREESURFER_HOME/bin:$FSFAST_HOME/bin:$FREESURFER_HOME/tktools:$MINC_BIN_DIR:$PATH +RUN echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh + +# Installing Neurodebian packages (FSL, AFNI, git) +RUN apt-get install -y --no-install-recommends \ + fsl-core=5.0.9-1~nd+1+nd16.04+1 \ + afni=16.2.07~dfsg.1-2~nd16.04+1 + +ENV FSLDIR=/usr/share/fsl/5.0 \ + FSLOUTPUTTYPE=NIFTI_GZ \ + FSLMULTIFILEQUIT=TRUE \ + POSSUMDIR=/usr/share/fsl/5.0 \ + LD_LIBRARY_PATH=/usr/lib/fsl/5.0:$LD_LIBRARY_PATH \ + FSLTCLSH=/usr/bin/tclsh \ + FSLWISH=/usr/bin/wish \ + AFNI_MODELPATH=/usr/lib/afni/models \ + AFNI_IMSAVE_WARNINGS=NO \ + AFNI_TTATLAS_DATASET=/usr/share/afni/atlases \ + AFNI_PLUGINPATH=/usr/lib/afni/plugins \ + PATH=/usr/lib/fsl/5.0:/usr/lib/afni/bin:$PATH + +# Installing and setting up ANTs +RUN mkdir -p /opt/ants && \ + curl -sSL "https://github.com/stnava/ANTs/releases/download/v2.1.0/Linux_Ubuntu14.04.tar.bz2" \ + | tar -xjC /opt/ants --strip-components 1 + +ENV ANTSPATH=/opt/ants \ + PATH=$ANTSPATH:$PATH + +# Installing and setting up c3d +RUN mkdir -p /opt/c3d && \ + curl -sSL "http://downloads.sourceforge.net/project/c3d/c3d/1.0.0/c3d-1.0.0-Linux-x86_64.tar.gz" \ + | tar -xzC /opt/c3d --strip-components 1 + +ENV C3DPATH=/opt/c3d/ \ + PATH=$C3DPATH/bin:$PATH + +# Install some other required tools +ENV apt-get install -y --no-install-recommends \ + unzip \ + apt-utils \ + fusefat \ + graphviz \ + make \ + ruby && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Install fake-S3 +ENV GEM_HOME /usr/local/bundle +ENV BUNDLE_PATH="$GEM_HOME" \ + BUNDLE_BIN="$GEM_HOME/bin" \ + BUNDLE_SILENCE_ROOT_WARNING=1 \ + BUNDLE_APP_CONFIG="$GEM_HOME" +ENV PATH $BUNDLE_BIN:$PATH +RUN mkdir -p "$GEM_HOME" "$BUNDLE_BIN" && \ + chmod 777 "$GEM_HOME" "$BUNDLE_BIN" + +RUN gem install fakes3 + +# Install Matlab MCR: from the good old install_spm_mcr.sh of @chrisfilo +WORKDIR /opt +RUN echo "destinationFolder=/opt/mcr" > mcr_options.txt && \ + echo "agreeToLicense=yes" >> mcr_options.txt && \ + echo "outputFile=/tmp/matlabinstall_log" >> mcr_options.txt && \ + echo "mode=silent" >> mcr_options.txt && \ + mkdir -p matlab_installer && \ + curl -sSL http://www.mathworks.com/supportfiles/downloads/R2015a/deployment_files/R2015a/installers/glnxa64/MCR_R2015a_glnxa64_installer.zip \ + -o matlab_installer/installer.zip && \ + unzip matlab_installer/installer.zip -d matlab_installer/ && \ + matlab_installer/install -inputFile mcr_options.txt && \ + rm -rf matlab_installer mcr_options.txt + +# Install SPM +RUN curl -sSL http://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/dev/spm12_r6472_Linux_R2015a.zip -o spm12.zip && \ + unzip spm12.zip && \ + rm -rf spm12.zip + +ENV MATLABCMD="/opt/mcr/v85/toolbox/matlab" \ + SPMMCRCMD="/opt/spm12/run_spm12.sh /opt/mcr/v85/ script" \ + FORCE_SPMMCR=1 + -# Install this branch's code -WORKDIR /root/src +# Installing and setting up miniconda +RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-4.2.12-Linux-x86_64.sh && \ + bash Miniconda3-4.2.12-Linux-x86_64.sh -b -p /usr/local/miniconda && \ + rm Miniconda3-4.2.12-Linux-x86_64.sh -# Install matplotlib, sphinx and coverage to build documentation -# and run tests with coverage -RUN source activate nipypetests-2.7 && \ - pip install matplotlib sphinx coverage && \ - source activate nipypetests-3.4 && \ - pip install matplotlib sphinx coverage && \ - source activate nipypetests-3.5 && \ - pip install matplotlib sphinx coverage +ENV PATH=/usr/local/miniconda/bin:$PATH \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + ACCEPT_INTEL_PYTHON_EULA=yes -ADD . nipype/ +# Installing precomputed python packages +RUN conda config --add channels intel && \ + conda config --set always_yes yes --set changeps1 no && \ + conda update -q conda && \ + conda install -y mkl=2017.0.1 \ + numpy=1.11.2 \ + scipy=0.18.1 \ + scikit-learn=0.17.1 \ + matplotlib=1.5.3 \ + pandas=0.19.0 \ + libxml2=2.9.4 \ + libxslt=1.1.29 \ + traits=4.6.0 \ + icu && \ + chmod +x /usr/local/miniconda/bin/* && \ + conda clean --all -y -# Install the checked out version of nipype, check that requirements are -# installed and install it for each of the three environments. -RUN cd nipype/ && \ - source activate nipypetests-2.7 && \ - pip install -r requirements.txt && \ - pip install -e . +# Precaching fonts +RUN python -c "from matplotlib import font_manager" -RUN cd nipype/ && \ - source activate nipypetests-3.4 && \ - pip install -r requirements.txt && \ - pip install -e . +# Installing Ubuntu packages and cleaning up +RUN apt-get install -y --no-install-recommends \ + git=1:2.7.4-0ubuntu1 \ + graphviz=2.38.0-12ubuntu2 && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -RUN cd nipype/ && \ - source activate nipypetests-3.5 && \ - pip install -r requirements.txt && \ - pip install -e . +# Unless otherwise specified each process should only use one thread - nipype +# will handle parallelization +ENV MKL_NUM_THREADS=1 \ + OMP_NUM_THREADS=1 -WORKDIR /scratch +# Installing dev requirements (packages that are not in pypi) +WORKDIR /root/ +ADD requirements.txt requirements.txt +RUN pip install -r requirements.txt && \ + rm -rf ~/.cache/pip -# Install entrypoints -ADD docker/circleci/run_* /usr/bin/ -RUN chmod +x /usr/bin/run_* +# Installing nipype +COPY . /root/src/nipype +RUN cd /root/src/nipype && \ + pip install -e .[all] && \ + rm -rf ~/.cache/pip -# RUN echo 'source /etc/profile.d/nipype_tests.sh' >> /etc/bash.bashrc -ENTRYPOINT ["/usr/bin/run_examples.sh"] +WORKDIR /root/ +ARG BUILD_DATE +ARG VCS_REF +ARG VERSION +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="NIPYPE" \ + org.label-schema.description="NIPYPE - Neuroimaging in Python: Pipelines and Interfaces" \ + org.label-schema.url="http://nipype.readthedocs.io" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vcs-url="https://github.com/nipy/nipype" \ + org.label-schema.version=$VERSION \ + org.label-schema.schema-version="1.0" From 39393380a0556bca7faa5aae4ed9078c8f01f916 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 11:09:22 -0800 Subject: [PATCH 03/21] add automatic deployment of tags from circleci (commented out for now) --- circle.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/circle.yml b/circle.yml index 7af4186b9c..4b5441ccab 100644 --- a/circle.yml +++ b/circle.yml @@ -46,3 +46,15 @@ general: artifacts: - "~/docs" - "~/logs" + +# To enable when ready +# deployment: +# production: +# tag: /.*/ +# commands: +# - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker push poldracklab/fmriprep:latest; fi : +# timeout: 21600 +# - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker tag poldracklab/fmriprep poldracklab/fmriprep:$CIRCLE_TAG && docker push poldracklab/fmriprep:$CIRCLE_TAG; fi : +# timeout: 21600 +# - printf "[distutils]\nindex-servers =\n pypi\n\n[pypi]\nusername:$PYPI_USER\npassword:$PYPI_PASS\n" > ~/.pypirc +# - python setup.py sdist upload -r pypi \ No newline at end of file From f62d5356753b3732efd4a828cad8f34ea30358b0 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 11:11:06 -0800 Subject: [PATCH 04/21] remove deprecated dockerfiles which only added noise --- docker/test-image-base/Dockerfile | 82 ----------------------------- docker/test-image-data/Dockerfile | 58 -------------------- docker/test-image-nipype/Dockerfile | 73 ------------------------- 3 files changed, 213 deletions(-) delete mode 100644 docker/test-image-base/Dockerfile delete mode 100644 docker/test-image-data/Dockerfile delete mode 100644 docker/test-image-nipype/Dockerfile diff --git a/docker/test-image-base/Dockerfile b/docker/test-image-base/Dockerfile deleted file mode 100644 index 8d940ef2ce..0000000000 --- a/docker/test-image-base/Dockerfile +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) 2016, The developers of nipype -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of crn_base nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -FROM neurodebian:latest -MAINTAINER Nipype developers - -# Preparations -RUN ln -snf /bin/bash /bin/sh -ARG DEBIAN_FRONTEND=noninteractive - -RUN sed -i -e 's,main$,main contrib non-free,g' /etc/apt/sources.list.d/neurodebian.sources.list && \ - apt-get -y update && \ - apt-get install -y curl \ - git \ - xvfb \ - bzip2 \ - unzip \ - apt-utils \ - fusefat \ - graphviz \ - fsl-core && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ - echo ". /etc/fsl/fsl.sh" >> /etc/bash.bashrc - -ENV FSLDIR=/usr/share/fsl/5.0 -ENV FSLOUTPUTTYPE=NIFTI_GZ -ENV PATH=/usr/lib/fsl/5.0:$PATH -ENV FSLMULTIFILEQUIT=TRUE -ENV POSSUMDIR=/usr/share/fsl/5.0 -ENV LD_LIBRARY_PATH=/usr/lib/fsl/5.0:$LD_LIBRARY_PATH -ENV FSLTCLSH=/usr/bin/tclsh -ENV FSLWISH=/usr/bin/wish - -# Install Matlab: from the good old install_spm_mcr.sh of @chrisfilo -WORKDIR /opt - -RUN echo "destinationFolder=/opt/mcr" > mcr_options.txt && \ - echo "agreeToLicense=yes" >> mcr_options.txt && \ - echo "outputFile=/tmp/matlabinstall_log" >> mcr_options.txt && \ - echo "mode=silent" >> mcr_options.txt && \ - mkdir -p matlab_installer && \ - curl -sSL http://www.mathworks.com/supportfiles/downloads/R2015a/deployment_files/R2015a/installers/glnxa64/MCR_R2015a_glnxa64_installer.zip \ - -o matlab_installer/installer.zip && \ - unzip matlab_installer/installer.zip -d matlab_installer/ && \ - matlab_installer/install -inputFile mcr_options.txt && \ - rm -rf matlab_installer mcr_options.txt - -ENV SPMMCRCMD "/opt/spm12/run_spm12.sh /opt/mcr/v85/ script" -ENV FORCE_SPMMCR 1 - -# Install SPM -RUN curl -sSL http://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/dev/spm12_r6472_Linux_R2015a.zip -o spm12.zip && \ - unzip spm12.zip && \ - rm -rf spm12.zip - -CMD ["/bin/bash"] - diff --git a/docker/test-image-data/Dockerfile b/docker/test-image-data/Dockerfile deleted file mode 100644 index cff45a3acb..0000000000 --- a/docker/test-image-data/Dockerfile +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2016, The developers of the Stanford CRN -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of crn_base nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -FROM nipype/testnipype:latest -MAINTAINER Stanford Center for Reproducible Neuroscience - -# Preparations -RUN ln -snf /bin/bash /bin/sh -WORKDIR /root - -# Install fsl-feeds and the nipype-tutorial data -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && \ - apt-get install -y fsl-feeds && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ - mkdir -p ~/examples/ && \ - ln -sf /usr/share/fsl-feeds/ ~/examples/feeds && \ - curl -sSL "https://dl.dropbox.com/s/jzgq2nupxyz36bp/nipype-tutorial.tar.bz2" -o nipype-tutorial.tar.bz2 && \ - tar jxf nipype-tutorial.tar.bz2 -C /root/examples - -# RUN curl -sSL "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt1.tar.gz" -o fdt1.tar.gz && \ -# curl -sSL "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt2.tar.gz" -o fdt2.tar.gz && \ -# curl -sSL "http://fsl.fmrib.ox.ac.uk/fslcourse/tbss.tar.gz" -o tbss.tar.gz && \ -# mkdir ~/examples/fsl_course_data && \ -# echo 'Untarring fsl course data...' && \ -# tar zxf fdt1.tar.gz -C ~/examples/fsl_course_data && \ -# tar zxf fdt2.tar.gz -C ~/examples/fsl_course_data && \ -# tar zxf tbss.tar.gz -C ~/examples/fsl_course_data && \ -# echo 'export FSL_COURSE_DATA=/root/examples/fsl_course_data' >> /etc/profile.d/nipype_data.sh -# ENV FSL_COURSE_DATA /root/examples/fsl_course_data -# RUN echo 'source /etc/profile.d/nipype_data.sh' >> /etc/bash.bashrc - -CMD ["/bin/bash"] diff --git a/docker/test-image-nipype/Dockerfile b/docker/test-image-nipype/Dockerfile deleted file mode 100644 index 07fe38281c..0000000000 --- a/docker/test-image-nipype/Dockerfile +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (c) 2016, The developers of the Stanford CRN -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of crn_base nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -FROM nipype/testbase:latest -MAINTAINER Stanford Center for Reproducible Neuroscience - -# Preparations -RUN ln -snf /bin/bash /bin/sh -WORKDIR /root - -# Install miniconda -RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh && \ - /bin/bash Miniconda-latest-Linux-x86_64.sh -b -p /usr/local/miniconda && \ - rm Miniconda-latest-Linux-x86_64.sh && \ - echo '#!/bin/bash' >> /etc/profile.d/nipype.sh && \ - echo 'export PATH=/usr/local/miniconda/bin:$PATH' >> /etc/profile.d/nipype.sh - -ENV PATH /usr/local/miniconda/bin:$PATH - -# http://bugs.python.org/issue19846 -# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. -ENV LANG C.UTF-8 - -# Add conda-forge channel in conda -RUN conda config --add channels conda-forge - -# Create conda environment -RUN conda create -y -n nipypetests-2.7 lockfile nipype && \ - echo '#!/bin/bash' >> /etc/profile.d/nipype.sh && \ - echo '#!/bin/bash' >> /etc/bashrc && \ - echo 'source activate nipypetests-2.7' >> /etc/profile.d/nipype.sh - -# Create conda environment -RUN conda create -y -n nipypetests-3.4 lockfile nipype python=3.4 - -# Create conda environment -RUN conda create -y -n nipypetests-3.5 lockfile nipype python=3.5 - -# Install dipy -RUN source activate nipypetests-2.7 && \ - pip install dipy && \ - source activate nipypetests-3.4 && \ - pip install dipy && \ - source activate nipypetests-3.5 && \ - pip install dipy - -RUN echo "source /etc/profile.d/nipype.sh" >> /etc/bash.bashrc -CMD ["/bin/bash"] From d1ffaa99a57fbf142b4e526fae2a9878e9efcce6 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 11:25:32 -0800 Subject: [PATCH 05/21] Move dockerfiles one level up, rename docker/files, fix circle.yml leftovers from fmriprep --- circle.yml | 8 ++++---- docker/{nipype_test => }/Dockerfile_base | 0 docker/{nipype_test => }/Dockerfile_py27 | 2 +- docker/{nipype_test => }/Dockerfile_py34 | 2 +- docker/{nipype_test => }/Dockerfile_py35 | 2 +- docker/{circleci => files}/run_builddocs.sh | 0 docker/{circleci => files}/run_examples.sh | 0 docker/{circleci => files}/run_pytests.sh | 0 docker/{circleci => files}/teardown.sh | 0 docker/{circleci => files}/tests.sh | 2 +- 10 files changed, 8 insertions(+), 8 deletions(-) rename docker/{nipype_test => }/Dockerfile_base (100%) rename docker/{nipype_test => }/Dockerfile_py27 (98%) rename docker/{nipype_test => }/Dockerfile_py34 (98%) rename docker/{nipype_test => }/Dockerfile_py35 (98%) rename docker/{circleci => files}/run_builddocs.sh (100%) rename docker/{circleci => files}/run_examples.sh (100%) rename docker/{circleci => files}/run_pytests.sh (100%) rename docker/{circleci => files}/teardown.sh (100%) rename docker/{circleci => files}/tests.sh (99%) diff --git a/circle.yml b/circle.yml index 4b5441ccab..ff55a0806b 100644 --- a/circle.yml +++ b/circle.yml @@ -27,9 +27,9 @@ dependencies: - if [[ -e ~/docker/image.tar ]]; then mv -n ~/docker/image.tar ~/docker/image_27.tar; fi - if [[ -e ~/docker/image_27.tar ]]; then docker load -i ~/docker/image_27.tar; fi - if [[ -e ~/docker/image_35.tar ]]; then docker load -i ~/docker/image_35.tar; fi - - docker build -f docker/nipype_test/Dockerfile_py27 -t nipype/nipype_test:py27 . : + - docker build -f docker/Dockerfile_py27 -t nipype/nipype_test:py27 . : timeout: 1600 - - docker build -f docker/nipype_test/Dockerfile_py35 -t nipype/nipype_test:py35 . : + - docker build -f docker/Dockerfile_py35 -t nipype/nipype_test:py35 . : timeout: 1600 - docker save nipype/nipype_test:py27 > ~/docker/image_27.tar : timeout: 1600 @@ -52,9 +52,9 @@ general: # production: # tag: /.*/ # commands: -# - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker push poldracklab/fmriprep:latest; fi : +# - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker push nipype/nipype:latest; fi : # timeout: 21600 -# - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker tag poldracklab/fmriprep poldracklab/fmriprep:$CIRCLE_TAG && docker push poldracklab/fmriprep:$CIRCLE_TAG; fi : +# - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker tag nipype/nipype nipype/nipype:$CIRCLE_TAG && docker push nipype/nipype:$CIRCLE_TAG; fi : # timeout: 21600 # - printf "[distutils]\nindex-servers =\n pypi\n\n[pypi]\nusername:$PYPI_USER\npassword:$PYPI_PASS\n" > ~/.pypirc # - python setup.py sdist upload -r pypi \ No newline at end of file diff --git a/docker/nipype_test/Dockerfile_base b/docker/Dockerfile_base similarity index 100% rename from docker/nipype_test/Dockerfile_base rename to docker/Dockerfile_base diff --git a/docker/nipype_test/Dockerfile_py27 b/docker/Dockerfile_py27 similarity index 98% rename from docker/nipype_test/Dockerfile_py27 rename to docker/Dockerfile_py27 index dc464b9e15..ba03902108 100644 --- a/docker/nipype_test/Dockerfile_py27 +++ b/docker/Dockerfile_py27 @@ -34,7 +34,7 @@ RUN conda update -y conda && \ conda update --all -y python=2.7 && \ pip install configparser -COPY docker/circleci/run_* /usr/bin/ +COPY docker/files/run_* /usr/bin/ RUN chmod +x /usr/bin/run_* # Speed up building diff --git a/docker/nipype_test/Dockerfile_py34 b/docker/Dockerfile_py34 similarity index 98% rename from docker/nipype_test/Dockerfile_py34 rename to docker/Dockerfile_py34 index 9f49f86206..6802923817 100644 --- a/docker/nipype_test/Dockerfile_py34 +++ b/docker/Dockerfile_py34 @@ -33,7 +33,7 @@ MAINTAINER The nipype developers https://github.com/nipy/nipype RUN conda update -y conda && \ conda update --all -y python=3.4 -COPY docker/circleci/run_* /usr/bin/ +COPY docker/files/run_* /usr/bin/ RUN chmod +x /usr/bin/run_* # Replace imglob with a Python3 compatible version diff --git a/docker/nipype_test/Dockerfile_py35 b/docker/Dockerfile_py35 similarity index 98% rename from docker/nipype_test/Dockerfile_py35 rename to docker/Dockerfile_py35 index a5107b9bad..ec64d3fa4b 100644 --- a/docker/nipype_test/Dockerfile_py35 +++ b/docker/Dockerfile_py35 @@ -31,7 +31,7 @@ MAINTAINER The nipype developers https://github.com/nipy/nipype WORKDIR /root -COPY docker/circleci/run_* /usr/bin/ +COPY docker/files/run_* /usr/bin/ RUN chmod +x /usr/bin/run_* # Replace imglob with a Python3 compatible version diff --git a/docker/circleci/run_builddocs.sh b/docker/files/run_builddocs.sh similarity index 100% rename from docker/circleci/run_builddocs.sh rename to docker/files/run_builddocs.sh diff --git a/docker/circleci/run_examples.sh b/docker/files/run_examples.sh similarity index 100% rename from docker/circleci/run_examples.sh rename to docker/files/run_examples.sh diff --git a/docker/circleci/run_pytests.sh b/docker/files/run_pytests.sh similarity index 100% rename from docker/circleci/run_pytests.sh rename to docker/files/run_pytests.sh diff --git a/docker/circleci/teardown.sh b/docker/files/teardown.sh similarity index 100% rename from docker/circleci/teardown.sh rename to docker/files/teardown.sh diff --git a/docker/circleci/tests.sh b/docker/files/tests.sh similarity index 99% rename from docker/circleci/tests.sh rename to docker/files/tests.sh index 40eaa1c7b4..2c28c4b155 100644 --- a/docker/circleci/tests.sh +++ b/docker/files/tests.sh @@ -36,7 +36,7 @@ case ${CIRCLE_NODE_INDEX} in esac # Put the artifacts in place -bash docker/circleci/teardown.sh +bash docker/files/teardown.sh # Send coverage data to codecov.io curl -so codecov.io https://codecov.io/bash From f9969bd186980b5fff63e1d19a6e11828d01fc50 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 11:25:50 -0800 Subject: [PATCH 06/21] few fixes on Dockerfile --- Dockerfile | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index d742e335a1..a0713bddf4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -109,12 +109,19 @@ RUN mkdir -p /opt/c3d && \ ENV C3DPATH=/opt/c3d/ \ PATH=$C3DPATH/bin:$PATH +# Installing Ubuntu packages and cleaning up +RUN apt-get install -y --no-install-recommends \ + git=1:2.7.4-0ubuntu1 \ + graphviz=2.38.0-12ubuntu2 && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + # Install some other required tools ENV apt-get install -y --no-install-recommends \ + git=1:2.7.4-0ubuntu1 \ + graphviz=2.38.0-12ubuntu2 \ unzip \ apt-utils \ fusefat \ - graphviz \ make \ ruby && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -165,7 +172,7 @@ ENV PATH=/usr/local/miniconda/bin:$PATH \ ACCEPT_INTEL_PYTHON_EULA=yes # Installing precomputed python packages -RUN conda config --add channels intel && \ +RUN conda config --add channels intel conda-forge && \ conda config --set always_yes yes --set changeps1 no && \ conda update -q conda && \ conda install -y mkl=2017.0.1 \ @@ -177,18 +184,14 @@ RUN conda config --add channels intel && \ libxml2=2.9.4 \ libxslt=1.1.29 \ traits=4.6.0 \ - icu && \ + psutil=5.0.1 \ + icu=58.1 && \ chmod +x /usr/local/miniconda/bin/* && \ conda clean --all -y -# Precaching fonts -RUN python -c "from matplotlib import font_manager" - -# Installing Ubuntu packages and cleaning up -RUN apt-get install -y --no-install-recommends \ - git=1:2.7.4-0ubuntu1 \ - graphviz=2.38.0-12ubuntu2 && \ - apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +# matplotlib cleanups: set default backend, precaching fonts +RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python2.7/site-packages/matplotlib/mpl-data/matplotlibrc && \ + python -c "from matplotlib import font_manager" # Unless otherwise specified each process should only use one thread - nipype # will handle parallelization From 2958961ecc09fd6f2c4350bf770b0e92ea1679d9 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 11:46:24 -0800 Subject: [PATCH 07/21] fix several RUN commands on Dockerfile --- Dockerfile | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index a0713bddf4..a60b578af4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -109,25 +109,20 @@ RUN mkdir -p /opt/c3d && \ ENV C3DPATH=/opt/c3d/ \ PATH=$C3DPATH/bin:$PATH -# Installing Ubuntu packages and cleaning up -RUN apt-get install -y --no-install-recommends \ - git=1:2.7.4-0ubuntu1 \ - graphviz=2.38.0-12ubuntu2 && \ - apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - # Install some other required tools -ENV apt-get install -y --no-install-recommends \ +RUN apt-get install -y --no-install-recommends \ git=1:2.7.4-0ubuntu1 \ graphviz=2.38.0-12ubuntu2 \ unzip \ apt-utils \ fusefat \ make \ - ruby && \ + ruby=1:2.3.0+1 && \ + apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Install fake-S3 -ENV GEM_HOME /usr/local/bundle +ENV GEM_HOME /usr/lib/ruby/gems/2.3 ENV BUNDLE_PATH="$GEM_HOME" \ BUNDLE_BIN="$GEM_HOME/bin" \ BUNDLE_SILENCE_ROOT_WARNING=1 \ From 33b47efb6f0ef13857277bc33aa49f22cb7d66bb Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 11:59:08 -0800 Subject: [PATCH 08/21] base all test images in main docker image --- circle.yml | 14 ++++++-------- docker/Dockerfile_py27 | 13 +++---------- docker/Dockerfile_py34 | 14 ++++---------- docker/Dockerfile_py35 | 19 +++---------------- 4 files changed, 16 insertions(+), 44 deletions(-) diff --git a/circle.yml b/circle.yml index ff55a0806b..a7f2e658b4 100644 --- a/circle.yml +++ b/circle.yml @@ -20,21 +20,19 @@ dependencies: - sudo apt-get -y update && sudo apt-get install -y wget bzip2 override: - - mkdir -p ~/docker ~/examples ~/scratch/pytest ~/scratch/logs + - mkdir -p $HOME/docker $HOME/examples $HOME/scratch/pytest $HOME/scratch/logs - if [[ ! -d ~/examples/nipype-tutorial ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-tutorial.tar.bz2 "${DATA_NIPYPE_TUTORIAL_URL}" && tar xjf nipype-tutorial.tar.bz2 -C ~/examples/; fi - if [[ ! -d ~/examples/nipype-fsl_course_data ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-fsl_course_data.tar.gz "${DATA_NIPYPE_FSL_COURSE}" && tar xzf nipype-fsl_course_data.tar.gz -C ~/examples/; fi - if [[ ! -d ~/examples/feeds ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O fsl-5.0.9-feeds.tar.gz "${DATA_NIPYPE_FSL_FEEDS}" && tar xzf fsl-5.0.9-feeds.tar.gz -C ~/examples/; fi - - if [[ -e ~/docker/image.tar ]]; then mv -n ~/docker/image.tar ~/docker/image_27.tar; fi - - if [[ -e ~/docker/image_27.tar ]]; then docker load -i ~/docker/image_27.tar; fi - - if [[ -e ~/docker/image_35.tar ]]; then docker load -i ~/docker/image_35.tar; fi + - if [[ -e $HOME/docker/image.tar ]]; then docker load -i $HOME/docker/image.tar; fi + - sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py + - e=1 && for i in {1..5}; do docker build -t nipype/nipype:latest --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] : + timeout: 21600 + - docker save nipype/nipype:latest > $HOME/docker/image.tar - docker build -f docker/Dockerfile_py27 -t nipype/nipype_test:py27 . : timeout: 1600 - docker build -f docker/Dockerfile_py35 -t nipype/nipype_test:py35 . : timeout: 1600 - - docker save nipype/nipype_test:py27 > ~/docker/image_27.tar : - timeout: 1600 - - docker save nipype/nipype_test:py35 > ~/docker/image_35.tar : - timeout: 1600 test: override: diff --git a/docker/Dockerfile_py27 b/docker/Dockerfile_py27 index ba03902108..6e20ad7c3f 100644 --- a/docker/Dockerfile_py27 +++ b/docker/Dockerfile_py27 @@ -26,23 +26,16 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM nipype/nipype_test:base-0.0.2 +FROM nipype/nipype:latest MAINTAINER The nipype developers https://github.com/nipy/nipype # Downgrade python to 2.7 -RUN conda update -y conda && \ - conda update --all -y python=2.7 && \ - pip install configparser +RUN conda install python=2.7 && \ + conda update --all -y python=2.7 COPY docker/files/run_* /usr/bin/ RUN chmod +x /usr/bin/run_* -# Speed up building -RUN mkdir -p /root/src/nipype -COPY requirements.txt /root/src/nipype/requirements.txt -RUN pip install -r /root/src/nipype/requirements.txt && \ - sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python2.7/site-packages/matplotlib/mpl-data/matplotlibrc - # Re-install nipype COPY . /root/src/nipype RUN rm -r /usr/local/miniconda/lib/python2.7/site-packages/nipype* && \ diff --git a/docker/Dockerfile_py34 b/docker/Dockerfile_py34 index 6802923817..bd595f0fe2 100644 --- a/docker/Dockerfile_py34 +++ b/docker/Dockerfile_py34 @@ -26,11 +26,11 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM nipype/nipype_test:base-0.0.2 +FROM nipype/nipype:latest MAINTAINER The nipype developers https://github.com/nipy/nipype # Downgrade python to 3.4 -RUN conda update -y conda && \ +RUN conda install python=3.4 && \ conda update --all -y python=3.4 COPY docker/files/run_* /usr/bin/ @@ -42,16 +42,10 @@ RUN rm -r ${FSLDIR}/bin/imglob && \ chmod +x /usr/bin/fsl_imglob.py && \ ln -s /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob -# Speed up building -RUN mkdir -p /root/src/nipype -COPY requirements.txt /root/src/nipype/requirements.txt -RUN pip install -r /root/src/nipype/requirements.txt && \ - sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python3.4/site-packages/matplotlib/mpl-data/matplotlibrc - # Re-install nipype COPY . /root/src/nipype -RUN rm -r /usr/local/miniconda/lib/python3.4/site-packages/nipype* && \ +RUN rm -r /usr/local/miniconda/lib/python2.7/site-packages/nipype* && \ cd /root/src/nipype && \ pip install -e .[all] -CMD ["/bin/bash"] +CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/Dockerfile_py35 b/docker/Dockerfile_py35 index ec64d3fa4b..cbdd88c8f8 100644 --- a/docker/Dockerfile_py35 +++ b/docker/Dockerfile_py35 @@ -29,27 +29,14 @@ FROM nipype/nipype_test:base-0.0.2 MAINTAINER The nipype developers https://github.com/nipy/nipype -WORKDIR /root - -COPY docker/files/run_* /usr/bin/ -RUN chmod +x /usr/bin/run_* - # Replace imglob with a Python3 compatible version COPY nipype/external/fsl_imglob.py /usr/bin/fsl_imglob.py RUN rm -r ${FSLDIR}/bin/imglob && \ chmod +x /usr/bin/fsl_imglob.py && \ ln -s /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob +COPY docker/files/run_* /usr/bin/ +RUN chmod +x /usr/bin/run_* -# Speed up building -RUN mkdir -p /root/src/nipype -COPY requirements.txt /root/src/nipype/requirements.txt -RUN pip install -r /root/src/nipype/requirements.txt && \ - sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python3.5/site-packages/matplotlib/mpl-data/matplotlibrc - -# Re-install nipype -COPY . /root/src/nipype -RUN rm -r /usr/local/miniconda/lib/python3.5/site-packages/nipype* && \ - cd /root/src/nipype && \ - pip install -e .[all] +WORKDIR /root CMD ["/bin/bash"] From fb5edf09b82d6539f9d1509ee79308465261c234 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 12:36:56 -0800 Subject: [PATCH 09/21] first building version of Dockerfile, fixes in Dockerfiles for tests with python<3.5 --- Dockerfile | 7 +++---- docker/Dockerfile_py27 | 4 ++++ docker/Dockerfile_py34 | 4 ++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index a60b578af4..00426a22a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -167,7 +167,7 @@ ENV PATH=/usr/local/miniconda/bin:$PATH \ ACCEPT_INTEL_PYTHON_EULA=yes # Installing precomputed python packages -RUN conda config --add channels intel conda-forge && \ +RUN conda config --add channels intel --add channels conda-forge && \ conda config --set always_yes yes --set changeps1 no && \ conda update -q conda && \ conda install -y mkl=2017.0.1 \ @@ -181,11 +181,10 @@ RUN conda config --add channels intel conda-forge && \ traits=4.6.0 \ psutil=5.0.1 \ icu=58.1 && \ - chmod +x /usr/local/miniconda/bin/* && \ - conda clean --all -y + chmod +x /usr/local/miniconda/bin/* # matplotlib cleanups: set default backend, precaching fonts -RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python2.7/site-packages/matplotlib/mpl-data/matplotlibrc && \ +RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python3.5/site-packages/matplotlib/mpl-data/matplotlibrc && \ python -c "from matplotlib import font_manager" # Unless otherwise specified each process should only use one thread - nipype diff --git a/docker/Dockerfile_py27 b/docker/Dockerfile_py27 index 6e20ad7c3f..fb82d0a5ec 100644 --- a/docker/Dockerfile_py27 +++ b/docker/Dockerfile_py27 @@ -36,6 +36,10 @@ RUN conda install python=2.7 && \ COPY docker/files/run_* /usr/bin/ RUN chmod +x /usr/bin/run_* +# matplotlib cleanups: set default backend, precaching fonts +RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python2.7/site-packages/matplotlib/mpl-data/matplotlibrc && \ + python -c "from matplotlib import font_manager" + # Re-install nipype COPY . /root/src/nipype RUN rm -r /usr/local/miniconda/lib/python2.7/site-packages/nipype* && \ diff --git a/docker/Dockerfile_py34 b/docker/Dockerfile_py34 index bd595f0fe2..29af1ef29c 100644 --- a/docker/Dockerfile_py34 +++ b/docker/Dockerfile_py34 @@ -42,6 +42,10 @@ RUN rm -r ${FSLDIR}/bin/imglob && \ chmod +x /usr/bin/fsl_imglob.py && \ ln -s /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob +# matplotlib cleanups: set default backend, precaching fonts +RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python3.4/site-packages/matplotlib/mpl-data/matplotlibrc && \ + python -c "from matplotlib import font_manager" + # Re-install nipype COPY . /root/src/nipype RUN rm -r /usr/local/miniconda/lib/python2.7/site-packages/nipype* && \ From 25219c1e26d096eaf4dadf1c15f23fa2475d8178 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 12:39:11 -0800 Subject: [PATCH 10/21] add conda-forge before intel --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 00426a22a7..17e8eacda4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -167,7 +167,7 @@ ENV PATH=/usr/local/miniconda/bin:$PATH \ ACCEPT_INTEL_PYTHON_EULA=yes # Installing precomputed python packages -RUN conda config --add channels intel --add channels conda-forge && \ +RUN conda config --add channels conda-forge --add channels intel && \ conda config --set always_yes yes --set changeps1 no && \ conda update -q conda && \ conda install -y mkl=2017.0.1 \ From 37132771e672090987eb50aca594b517684ea0bf Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 12:58:29 -0800 Subject: [PATCH 11/21] final modifications to test dockerfiles --- docker/Dockerfile_py27 | 6 ++---- docker/Dockerfile_py34 | 3 +-- docker/Dockerfile_py35 | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/docker/Dockerfile_py27 b/docker/Dockerfile_py27 index fb82d0a5ec..827faf35aa 100644 --- a/docker/Dockerfile_py27 +++ b/docker/Dockerfile_py27 @@ -30,8 +30,7 @@ FROM nipype/nipype:latest MAINTAINER The nipype developers https://github.com/nipy/nipype # Downgrade python to 2.7 -RUN conda install python=2.7 && \ - conda update --all -y python=2.7 +RUN conda install python=2.7 COPY docker/files/run_* /usr/bin/ RUN chmod +x /usr/bin/run_* @@ -42,8 +41,7 @@ RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python2.7/sit # Re-install nipype COPY . /root/src/nipype -RUN rm -r /usr/local/miniconda/lib/python2.7/site-packages/nipype* && \ - cd /root/src/nipype && \ +RUN cd /root/src/nipype && \ pip install -e .[all] CMD ["/bin/bash"] diff --git a/docker/Dockerfile_py34 b/docker/Dockerfile_py34 index 29af1ef29c..88e84841e6 100644 --- a/docker/Dockerfile_py34 +++ b/docker/Dockerfile_py34 @@ -48,8 +48,7 @@ RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python3.4/sit # Re-install nipype COPY . /root/src/nipype -RUN rm -r /usr/local/miniconda/lib/python2.7/site-packages/nipype* && \ - cd /root/src/nipype && \ +RUN cd /root/src/nipype && \ pip install -e .[all] CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/Dockerfile_py35 b/docker/Dockerfile_py35 index cbdd88c8f8..f08c655941 100644 --- a/docker/Dockerfile_py35 +++ b/docker/Dockerfile_py35 @@ -26,7 +26,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM nipype/nipype_test:base-0.0.2 +FROM nipype/nipype:latest MAINTAINER The nipype developers https://github.com/nipy/nipype # Replace imglob with a Python3 compatible version From b1ed84f7765261b2eaad1e880f40b377f0b10f6e Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 13:08:41 -0800 Subject: [PATCH 12/21] update docker version in circle --- circle.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index a7f2e658b4..153cc995df 100644 --- a/circle.yml +++ b/circle.yml @@ -1,10 +1,12 @@ machine: + pre: + - sudo curl -L -o /usr/bin/docker 'https://s3-external-1.amazonaws.com/circle-downloads/docker-1.9.1-circleci' + - sudo chmod 0755 /usr/bin/docker environment: OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" DATA_NIPYPE_TUTORIAL_URL: "${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" DATA_NIPYPE_FSL_COURSE: "${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" DATA_NIPYPE_FSL_FEEDS: "${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" - services: - docker From 46b8476d58bc2728931bb121d5ae04faba0fccaf Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 13:55:57 -0800 Subject: [PATCH 13/21] make sure conda has the right perms all the times --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 17e8eacda4..60aada866f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -168,8 +168,10 @@ ENV PATH=/usr/local/miniconda/bin:$PATH \ # Installing precomputed python packages RUN conda config --add channels conda-forge --add channels intel && \ + chmod +x /usr/local/miniconda/bin/* && \ conda config --set always_yes yes --set changeps1 no && \ conda update -q conda && \ + chmod +x /usr/local/miniconda/bin/* && \ conda install -y mkl=2017.0.1 \ numpy=1.11.2 \ scipy=0.18.1 \ @@ -181,7 +183,6 @@ RUN conda config --add channels conda-forge --add channels intel && \ traits=4.6.0 \ psutil=5.0.1 \ icu=58.1 && \ - chmod +x /usr/local/miniconda/bin/* # matplotlib cleanups: set default backend, precaching fonts RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python3.5/site-packages/matplotlib/mpl-data/matplotlibrc && \ From b48da5b98bfb2e060f4b91e4d3fbae627169e9d4 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 18:13:57 -0800 Subject: [PATCH 14/21] fix Dockerfile, tested in local and pushed into docker hub --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 60aada866f..16a44fd99e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -182,7 +182,7 @@ RUN conda config --add channels conda-forge --add channels intel && \ libxslt=1.1.29 \ traits=4.6.0 \ psutil=5.0.1 \ - icu=58.1 && \ + icu=58.1 # matplotlib cleanups: set default backend, precaching fonts RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python3.5/site-packages/matplotlib/mpl-data/matplotlibrc && \ From 1f17d28110d99e9992545bd7ab1af6f8bc542da9 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 19:13:01 -0800 Subject: [PATCH 15/21] enable circleCI deploy to Docker Hub, hopefully fix permissions after conda update --- circle.yml | 25 +++++++++++++++---------- docker/Dockerfile_py27 | 8 ++++---- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/circle.yml b/circle.yml index 153cc995df..8136a88e0e 100644 --- a/circle.yml +++ b/circle.yml @@ -10,6 +10,11 @@ machine: services: - docker +# Disable python +checkout: + post: + - rm -rf virtualenvs venv .pyenv + dependencies: cache_directories: - "~/docker" @@ -47,14 +52,14 @@ general: - "~/docs" - "~/logs" -# To enable when ready -# deployment: -# production: -# tag: /.*/ -# commands: -# - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker push nipype/nipype:latest; fi : -# timeout: 21600 -# - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker tag nipype/nipype nipype/nipype:$CIRCLE_TAG && docker push nipype/nipype:$CIRCLE_TAG; fi : -# timeout: 21600 +deployment: + production: + tag: /.*/ + commands: + - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker push nipype/nipype:latest; fi : + timeout: 21600 + - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker tag nipype/nipype nipype/nipype:$CIRCLE_TAG && docker push nipype/nipype:$CIRCLE_TAG; fi : + timeout: 21600 +# Automatic deployment to Pypi: # - printf "[distutils]\nindex-servers =\n pypi\n\n[pypi]\nusername:$PYPI_USER\npassword:$PYPI_PASS\n" > ~/.pypirc -# - python setup.py sdist upload -r pypi \ No newline at end of file +# - python setup.py sdist upload -r pypi diff --git a/docker/Dockerfile_py27 b/docker/Dockerfile_py27 index 827faf35aa..b1264294c3 100644 --- a/docker/Dockerfile_py27 +++ b/docker/Dockerfile_py27 @@ -29,14 +29,14 @@ FROM nipype/nipype:latest MAINTAINER The nipype developers https://github.com/nipy/nipype -# Downgrade python to 2.7 -RUN conda install python=2.7 - COPY docker/files/run_* /usr/bin/ RUN chmod +x /usr/bin/run_* +# Downgrade python to 2.7 # matplotlib cleanups: set default backend, precaching fonts -RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python2.7/site-packages/matplotlib/mpl-data/matplotlibrc && \ +RUN conda install python=2.7 && \ + find /usr/local/miniconda/ -exec chmod 775 {} + && \ + sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python2.7/site-packages/matplotlib/mpl-data/matplotlibrc && \ python -c "from matplotlib import font_manager" # Re-install nipype From c612230c939b4f655e112b156daa2d0d4f7daa28 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 19:48:51 -0800 Subject: [PATCH 16/21] revert removing virtualenvs, fix tests path --- circle.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/circle.yml b/circle.yml index 8136a88e0e..b7d59f664f 100644 --- a/circle.yml +++ b/circle.yml @@ -10,11 +10,6 @@ machine: services: - docker -# Disable python -checkout: - post: - - rm -rf virtualenvs venv .pyenv - dependencies: cache_directories: - "~/docker" @@ -43,7 +38,7 @@ dependencies: test: override: - - bash docker/circleci/tests.sh : + - bash docker/files/tests.sh : timeout: 7200 parallel: true From 7a4144ed258b4abf7de0f7faaa703e9c95270c43 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 20:39:16 -0800 Subject: [PATCH 17/21] fix permissions in circleci --- circle.yml | 6 ++++-- docker/files/tests.sh | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/circle.yml b/circle.yml index b7d59f664f..ef2344a9b5 100644 --- a/circle.yml +++ b/circle.yml @@ -7,6 +7,7 @@ machine: DATA_NIPYPE_TUTORIAL_URL: "${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" DATA_NIPYPE_FSL_COURSE: "${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" DATA_NIPYPE_FSL_FEEDS: "${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" + SCRATCH: "$HOME/scratch" services: - docker @@ -20,9 +21,10 @@ dependencies: # Let CircleCI cache the apt archive - mkdir -p ~/.apt-cache/partial && sudo rm -rf /var/cache/apt/archives && sudo ln -s ~/.apt-cache /var/cache/apt/archives - sudo apt-get -y update && sudo apt-get install -y wget bzip2 - + # Create scratch folder and force group permissions + - mkdir -p $SCRATCH && sudo setfacl -d -m group:ubuntu:rwx $SCRATCH && sudo setfacl -m group:ubuntu:rwx $SCRATCH + - mkdir -p $HOME/docker $HOME/examples $SCRATCH/pytest $SCRATCH override: - - mkdir -p $HOME/docker $HOME/examples $HOME/scratch/pytest $HOME/scratch/logs - if [[ ! -d ~/examples/nipype-tutorial ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-tutorial.tar.bz2 "${DATA_NIPYPE_TUTORIAL_URL}" && tar xjf nipype-tutorial.tar.bz2 -C ~/examples/; fi - if [[ ! -d ~/examples/nipype-fsl_course_data ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-fsl_course_data.tar.gz "${DATA_NIPYPE_FSL_COURSE}" && tar xzf nipype-fsl_course_data.tar.gz -C ~/examples/; fi - if [[ ! -d ~/examples/feeds ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O fsl-5.0.9-feeds.tar.gz "${DATA_NIPYPE_FSL_FEEDS}" && tar xzf fsl-5.0.9-feeds.tar.gz -C ~/examples/; fi diff --git a/docker/files/tests.sh b/docker/files/tests.sh index 2c28c4b155..18f7db0121 100644 --- a/docker/files/tests.sh +++ b/docker/files/tests.sh @@ -14,24 +14,24 @@ fi # They may need to be rebalanced in the future. case ${CIRCLE_NODE_INDEX} in 0) - time docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ level1 + time docker run -v /etc/localtime:/etc/localtime:ro -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ level1 ;; 1) - time docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ l2pipeline - time docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow3d - time docker run -v /etc/localtime:/etc/localtime:ro -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /root/src/nipype nipype/nipype_test:py35 /usr/bin/run_pytests.sh py35 - time docker run -v /etc/localtime:/etc/localtime:ro -e NIPYPE_NUMBER_OF_CPUS=4 -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ level1 + time docker run -v /etc/localtime:/etc/localtime:ro -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ l2pipeline + time docker run -v /etc/localtime:/etc/localtime:ro -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow3d + time docker run -v /etc/localtime:/etc/localtime:ro -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /root/src/nipype nipype/nipype_test:py35 /usr/bin/run_pytests.sh py35 + time docker run -v /etc/localtime:/etc/localtime:ro -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ level1 ;; 2) - time docker run -v /etc/localtime:/etc/localtime:ro -e NIPYPE_NUMBER_OF_CPUS=4 -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ l2pipeline - time docker run -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /root/src/nipype/doc nipype/nipype_test:py35 /usr/bin/run_builddocs.sh - time docker run -v /etc/localtime:/etc/localtime:ro -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /root/src/nipype nipype/nipype_test:py27 /usr/bin/run_pytests.sh py27 - time docker run -v /etc/localtime:/etc/localtime:ro -e NIPYPE_NUMBER_OF_CPUS=4 -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ level1 + time docker run -v /etc/localtime:/etc/localtime:ro -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ l2pipeline + time docker run -v /etc/localtime:/etc/localtime:ro -v $SCRATCH:/scratch -w /root/src/nipype/doc nipype/nipype_test:py35 /usr/bin/run_builddocs.sh + time docker run -v /etc/localtime:/etc/localtime:ro -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /root/src/nipype nipype/nipype_test:py27 /usr/bin/run_pytests.sh py27 + time docker run -v /etc/localtime:/etc/localtime:ro -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ level1 ;; 3) - time docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow4d - time docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_fsl_feeds Linear /root/examples/ l1pipeline - time docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_fsl_reuse Linear /root/examples/ level1_workflow + time docker run -v /etc/localtime:/etc/localtime:ro -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow4d + time docker run -v /etc/localtime:/etc/localtime:ro -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_fsl_feeds Linear /root/examples/ l1pipeline + time docker run -v /etc/localtime:/etc/localtime:ro -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_fsl_reuse Linear /root/examples/ level1_workflow ;; esac From 7fd0106a4cf003cfa9456f900ff9f377e6907b81 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 14 Feb 2017 22:04:37 -0800 Subject: [PATCH 18/21] fix docker run tests --- circle.yml | 2 +- docker/files/tests.sh | 26 ++++++++++++-------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/circle.yml b/circle.yml index ef2344a9b5..b307653b75 100644 --- a/circle.yml +++ b/circle.yml @@ -23,7 +23,7 @@ dependencies: - sudo apt-get -y update && sudo apt-get install -y wget bzip2 # Create scratch folder and force group permissions - mkdir -p $SCRATCH && sudo setfacl -d -m group:ubuntu:rwx $SCRATCH && sudo setfacl -m group:ubuntu:rwx $SCRATCH - - mkdir -p $HOME/docker $HOME/examples $SCRATCH/pytest $SCRATCH + - mkdir -p $HOME/docker $HOME/examples $SCRATCH/pytest $SCRATCH/logs override: - if [[ ! -d ~/examples/nipype-tutorial ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-tutorial.tar.bz2 "${DATA_NIPYPE_TUTORIAL_URL}" && tar xjf nipype-tutorial.tar.bz2 -C ~/examples/; fi - if [[ ! -d ~/examples/nipype-fsl_course_data ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-fsl_course_data.tar.gz "${DATA_NIPYPE_FSL_COURSE}" && tar xzf nipype-fsl_course_data.tar.gz -C ~/examples/; fi diff --git a/docker/files/tests.sh b/docker/files/tests.sh index 18f7db0121..41b3531b33 100644 --- a/docker/files/tests.sh +++ b/docker/files/tests.sh @@ -14,24 +14,24 @@ fi # They may need to be rebalanced in the future. case ${CIRCLE_NODE_INDEX} in 0) - time docker run -v /etc/localtime:/etc/localtime:ro -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ level1 + time docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ level1 ;; 1) - time docker run -v /etc/localtime:/etc/localtime:ro -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ l2pipeline - time docker run -v /etc/localtime:/etc/localtime:ro -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow3d - time docker run -v /etc/localtime:/etc/localtime:ro -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /root/src/nipype nipype/nipype_test:py35 /usr/bin/run_pytests.sh py35 - time docker run -v /etc/localtime:/etc/localtime:ro -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ level1 + time docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ l2pipeline + time docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow3d + time docker run --rm -it -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /root/src/nipype nipype/nipype_test:py35 /usr/bin/run_pytests.sh py35 + time docker run --rm -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ level1 ;; 2) - time docker run -v /etc/localtime:/etc/localtime:ro -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ l2pipeline - time docker run -v /etc/localtime:/etc/localtime:ro -v $SCRATCH:/scratch -w /root/src/nipype/doc nipype/nipype_test:py35 /usr/bin/run_builddocs.sh - time docker run -v /etc/localtime:/etc/localtime:ro -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /root/src/nipype nipype/nipype_test:py27 /usr/bin/run_pytests.sh py27 - time docker run -v /etc/localtime:/etc/localtime:ro -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ level1 + time docker run --rm -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ l2pipeline + time docker run --rm -it -v $SCRATCH:/scratch -w /root/src/nipype/doc nipype/nipype_test:py35 /usr/bin/run_builddocs.sh + time docker run --rm -it -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /root/src/nipype nipype/nipype_test:py27 /usr/bin/run_pytests.sh py27 + time docker run --rm -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ level1 ;; 3) - time docker run -v /etc/localtime:/etc/localtime:ro -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow4d - time docker run -v /etc/localtime:/etc/localtime:ro -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_fsl_feeds Linear /root/examples/ l1pipeline - time docker run -v /etc/localtime:/etc/localtime:ro -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_fsl_reuse Linear /root/examples/ level1_workflow + time docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow4d + time docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_fsl_feeds Linear /root/examples/ l1pipeline + time docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_fsl_reuse Linear /root/examples/ level1_workflow ;; esac @@ -46,5 +46,3 @@ find "${CIRCLE_TEST_REPORTS}/pytest" -name 'coverage*.xml' -print0 | \ find "${CIRCLE_TEST_REPORTS}/pytest" -name 'smoketests*.xml' -print0 | \ xargs -0 -I file ./codecov.io -f file -t "${CODECOV_TOKEN}" -F smoketests - - From d1a8e518b217fdc5711cf1d7557b21da0fca25ab Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 15 Feb 2017 10:16:09 -0800 Subject: [PATCH 19/21] Update CHANGES --- CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index 8720078b98..2f72451f0a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,8 @@ Upcoming release 0.13 ===================== +* ENH: Revised all Dockerfiles and automated deployment to Docker Hub + from CircleCI (https://github.com/nipy/nipype/pull/1815) * FIX: Semaphore capture using MultiProc plugin (https://github.com/nipy/nipype/pull/1689) * REF: Refactor AFNI interfaces (https://github.com/nipy/nipype/pull/1678, https://github.com/nipy/nipype/pull/1680) * ENH: Move nipype commands to group command using click (https://github.com/nipy/nipype/pull/1608) From a23df788dfb30633a464a5e69bc573d317632450 Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 15 Feb 2017 14:03:22 -0800 Subject: [PATCH 20/21] update dockerignore --- .dockerignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 4d176df9e5..71191e2e56 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,8 +14,6 @@ nipype.egg-info .eggs src/**/* src/ -docker/nipype_* -docker/test-* # releasing Makefile From 4d4d8422b57af7cfc3b65f38c50f7e415c07283d Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 15 Feb 2017 14:05:35 -0800 Subject: [PATCH 21/21] save nipype_test docker images into cached dir --- circle.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index b307653b75..01b0dfd601 100644 --- a/circle.yml +++ b/circle.yml @@ -29,14 +29,18 @@ dependencies: - if [[ ! -d ~/examples/nipype-fsl_course_data ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-fsl_course_data.tar.gz "${DATA_NIPYPE_FSL_COURSE}" && tar xzf nipype-fsl_course_data.tar.gz -C ~/examples/; fi - if [[ ! -d ~/examples/feeds ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O fsl-5.0.9-feeds.tar.gz "${DATA_NIPYPE_FSL_FEEDS}" && tar xzf fsl-5.0.9-feeds.tar.gz -C ~/examples/; fi - if [[ -e $HOME/docker/image.tar ]]; then docker load -i $HOME/docker/image.tar; fi + - if [[ -e $HOME/docker/image27.tar ]]; then docker load -i $HOME/docker/image27.tar; fi + - if [[ -e $HOME/docker/image35.tar ]]; then docker load -i $HOME/docker/image35.tar; fi - sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py - e=1 && for i in {1..5}; do docker build -t nipype/nipype:latest --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] : timeout: 21600 - - docker save nipype/nipype:latest > $HOME/docker/image.tar - docker build -f docker/Dockerfile_py27 -t nipype/nipype_test:py27 . : timeout: 1600 - docker build -f docker/Dockerfile_py35 -t nipype/nipype_test:py35 . : timeout: 1600 + - docker save nipype/nipype:latest > $HOME/docker/image.tar + - docker save nipype/nipype_test:py27 > $HOME/docker/image27.tar + - docker save nipype/nipype_test:py35 > $HOME/docker/image35.tar test: override: