From 0e7f32323863a93f68433eae0624417b9620edc4 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Sat, 17 Oct 2015 11:16:38 -0700 Subject: [PATCH 1/7] BF: fix and test sdist command I broke setup.py sdist in the Python 3 refactor, because we were previously using a version of numpy.distutils build_py, and the refactor reverted (accidentally) to using the default disutils setup.py. We also forgot to add the new position of the fff files to the MANIFEST.in, so the sdist archives was broken. --- .travis.yml | 20 +++++++++++++++++++- MANIFEST.in | 2 +- setup.py | 3 ++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4e1454ba4b..edc5f73447 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ cache: env: global: - DEPENDS="numpy scipy sympy matplotlib nibabel" + - INSTALL_TYPE="setup" python: - 2.6 - 3.2 @@ -37,6 +38,13 @@ matrix: - python: 3.4 env: - NIPY_EXTERNAL_LAPACK=1 + - python: 3.4 + env: + - INSTALL_TYPE=sdist + - python: 3.4 + env: + - INSTALL_TYPE=wheel + before_install: - source tools/travis_tools.sh - virtualenv --python=python venv @@ -52,7 +60,17 @@ before_install: # command to install dependencies # e.g. pip install -r requirements.txt # --use-mirrors install: - - python setup.py install + - | + if [ "$INSTALL_TYPE" == "setup" ]; then + python setup.py install + elif [ "$INSTALL_TYPE" == "sdist" ]; then + python setup.py sdist + pip install dist/*.tar.gz + elif [ "$INSTALL_TYPE" == "wheel" ]; then + pip install wheel + python setup.py bdist_wheel + pip install dist/*.whl + fi # command to run tests, e.g. python setup.py test script: # Change into an innocuous directory and find tests from installation diff --git a/MANIFEST.in b/MANIFEST.in index b1817c29a0..1cd5277a28 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,7 +3,7 @@ include Changelog TODO include *.py include site.* recursive-include nipy *.c *.h *.pyx *.pxd -recursive-include libcstat *.c *.h *.pyx *.pxd +recursive-include lib *.c *.h *.pyx *.pxd remake recursive-include scripts * recursive-include tools * # put this stuff back into setup.py (package_data) once I'm enlightened diff --git a/setup.py b/setup.py index 47bac007d5..f6d47f261c 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,8 @@ build_src.generate_a_pyrex_source = generate_a_pyrex_source # Add custom commit-recording build command -cmdclass['build_py'] = get_comrec_build('nipy') +from numpy.distutils.command.build_py import build_py as _build_py +cmdclass['build_py'] = get_comrec_build('nipy', _build_py) def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration From 3debbe23f8df1ac758bbd20679b82c931988d650 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Sat, 17 Oct 2015 11:36:49 -0700 Subject: [PATCH 2/7] RF: ship get_comrec_build routine from nibabel Ship routine to make build_py command. Working towards allowing setup.py to run without nibabel. --- setup.py | 4 +-- setup_helpers.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f6d47f261c..d25794208f 100755 --- a/setup.py +++ b/setup.py @@ -13,12 +13,12 @@ # Import build helpers try: - from nisext.sexts import package_check, get_comrec_build + from nisext.sexts import package_check except ImportError: raise RuntimeError('Need nisext package from nibabel installation' ' - please install nibabel first') -from setup_helpers import (generate_a_pyrex_source, +from setup_helpers import (generate_a_pyrex_source, get_comrec_build, cmdclass, INFO_VARS) # monkey-patch numpy distutils to use Cython instead of Pyrex diff --git a/setup_helpers.py b/setup_helpers.py index 47a7492464..094040a722 100755 --- a/setup_helpers.py +++ b/setup_helpers.py @@ -23,10 +23,16 @@ from distutils.cmd import Command from distutils.command.clean import clean from distutils.command.install_scripts import install_scripts +from distutils.command.build_py import build_py from distutils.version import LooseVersion from distutils.dep_util import newer_group from distutils.errors import DistutilsError +try: + from ConfigParser import ConfigParser +except ImportError: # Python 3 + from configparser import ConfigParser + from numpy.distutils.misc_util import appendpath from numpy.distutils import log @@ -308,6 +314,63 @@ def run(self): with open(bat_file, 'wt') as fobj: fobj.write(bat_contents) +# This copied from nibabel/nisext/sexts.py +# We'll probably drop this way of doing versioning soon +def get_comrec_build(pkg_dir, build_cmd=build_py): + """ Return extended build command class for recording commit + + The extended command tries to run git to find the current commit, getting + the empty string if it fails. It then writes the commit hash into a file + in the `pkg_dir` path, named ``COMMIT_INFO.txt``. + + In due course this information can be used by the package after it is + installed, to tell you what commit it was installed from if known. + + To make use of this system, you need a package with a COMMIT_INFO.txt file - + e.g. ``myproject/COMMIT_INFO.txt`` - that might well look like this:: + + # This is an ini file that may contain information about the code state + [commit hash] + # The line below may contain a valid hash if it has been substituted during 'git archive' + archive_subst_hash=$Format:%h$ + # This line may be modified by the install process + install_hash= + + The COMMIT_INFO file above is also designed to be used with git substitution + - so you probably also want a ``.gitattributes`` file in the root directory + of your working tree that contains something like this:: + + myproject/COMMIT_INFO.txt export-subst + + That will cause the ``COMMIT_INFO.txt`` file to get filled in by ``git + archive`` - useful in case someone makes such an archive - for example with + via the github 'download source' button. + + Although all the above will work as is, you might consider having something + like a ``get_info()`` function in your package to display the commit + information at the terminal. See the ``pkg_info.py`` module in the nipy + package for an example. + """ + class MyBuildPy(build_cmd): + ''' Subclass to write commit data into installation tree ''' + def run(self): + build_cmd.run(self) + import subprocess + proc = subprocess.Popen('git rev-parse --short HEAD', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True) + repo_commit, _ = proc.communicate() + # Fix for python 3 + repo_commit = str(repo_commit) + # We write the installation commit even if it's empty + cfg_parser = ConfigParser() + cfg_parser.read(pjoin(pkg_dir, 'COMMIT_INFO.txt')) + cfg_parser.set('commit hash', 'install_hash', repo_commit) + out_pth = pjoin(self.build_lib, pkg_dir, 'COMMIT_INFO.txt') + cfg_parser.write(open(out_pth, 'wt')) + return MyBuildPy + # The command classes for distutils, used by setup.py cmdclass = {'api_docs': APIDocs, From f9cc179449201f0b619c9490a358316fbb0d5149 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Sat, 17 Oct 2015 12:20:05 -0700 Subject: [PATCH 3/7] RF: let setuptools do some dependency checking Let setuptools do dependency checking when packages are not installed, or they are easy to install. --- setup.py | 66 +++++++++++++++++------------------------------- setup_helpers.py | 33 ++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 43 deletions(-) diff --git a/setup.py b/setup.py index d25794208f..6f334bbfbd 100755 --- a/setup.py +++ b/setup.py @@ -11,15 +11,9 @@ # update it when the contents of directories change. if exists('MANIFEST'): os.remove('MANIFEST') -# Import build helpers -try: - from nisext.sexts import package_check -except ImportError: - raise RuntimeError('Need nisext package from nibabel installation' - ' - please install nibabel first') - from setup_helpers import (generate_a_pyrex_source, get_comrec_build, - cmdclass, INFO_VARS) + cmdclass, INFO_VARS, get_pkg_version, + version_error_msg) # monkey-patch numpy distutils to use Cython instead of Pyrex from numpy.distutils.command.build_src import build_src @@ -60,42 +54,28 @@ def configuration(parent_package='',top_path=None): if not 'extra_setuptools_args' in globals(): extra_setuptools_args = dict() - # Hard and soft dependency checking -package_check('numpy', INFO_VARS['NUMPY_MIN_VERSION']) -package_check('scipy', INFO_VARS['SCIPY_MIN_VERSION']) -package_check('nibabel', INFO_VARS['NIBABEL_MIN_VERSION']) -package_check('sympy', INFO_VARS['SYMPY_MIN_VERSION']) -def _mayavi_version(pkg_name): - """Mayavi2 pruned enthought. namespace at 4.0.0 - """ - v = '' - try: - from mayavi import version - v = version.version - if v == '': - v = '4.0.0' # must be the one in Debian - except ImportError: - from enthought.mayavi import version - v = version.version - return v -package_check('mayavi', - INFO_VARS['MAYAVI_MIN_VERSION'], - optional=True, - version_getter=_mayavi_version) -# Cython can be a build dependency -def _cython_version(pkg_name): - from Cython.Compiler.Version import version - return version -package_check('cython', - INFO_VARS['CYTHON_MIN_VERSION'], - optional=True, - version_getter=_cython_version, - messages={'opt suffix': ' - you will not be able ' - 'to rebuild Cython source files into C files', - 'missing opt': 'Missing optional build-time ' - 'package "%s"'} - ) +DEPS = ( + ('numpy', INFO_VARS['NUMPY_MIN_VERSION'], 'setup_requires', True), + ('scipy', INFO_VARS['SCIPY_MIN_VERSION'], 'install_requires', True), + ('nibabel', INFO_VARS['NIBABEL_MIN_VERSION'], 'install_requires', False), + ('sympy', INFO_VARS['SYMPY_MIN_VERSION'], 'install_requires', False)) + +using_setuptools = 'setuptools' in sys.modules + +for name, min_ver, req_type, heavy in DEPS: + found_ver = get_pkg_version(name) + ver_err_msg = version_error_msg(name, found_ver, min_ver) + if not using_setuptools: + if ver_err_msg != None: + raise RuntimeError(ver_err_msg) + else: # Using setuptools + # Add packages to given section of setup/install_requires + if ver_err_msg != None or not heavy: + new_req = '{0}>={1}'.format(name, min_ver) + old_reqs = extra_setuptools_args.get(req_type, []) + extra_setuptools_args[req_type] = old_reqs + [new_req] + ################################################################################ # commands for installing the data diff --git a/setup_helpers.py b/setup_helpers.py index 094040a722..09fa12adb2 100755 --- a/setup_helpers.py +++ b/setup_helpers.py @@ -372,6 +372,39 @@ def run(self): return MyBuildPy +def get_pkg_version(pkg_name): + """ Return package version for `pkg_name` if installed + + Returns + ------- + pkg_version : str or None + Return None if package not importable. Return 'unknown' if standard + ``__version__`` string not present. Otherwise return version string. + """ + try: + pkg = __import__(pkg_name) + except ImportError: + return None + try: + return pkg.__version__ + except AttributeError: + return 'unknown' + + +def version_error_msg(pkg_name, found_ver, min_ver): + """ Return informative error message for version or None + """ + if found_ver is None: + return 'We need package {0}, but not importable'.format(pkg_name) + if found_ver == 'unknown': + msg = 'We need {0} version {1}, but cannot get version'.format( + pkg_name, min_ver) + if LooseVersion(found_ver) >= LooseVersion(min_ver): + return None + return 'We need {0} version {1}, but found version {2}'.format( + pkg_name, found_ver, min_ver) + + # The command classes for distutils, used by setup.py cmdclass = {'api_docs': APIDocs, 'clean': Clean, From db28f0a3aaa6db11821c02836ec7784587d7d1e4 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Sat, 17 Oct 2015 12:55:15 -0700 Subject: [PATCH 4/7] RF: do not use internal copy of six for setup.py Using internal copy requires import from the nipy tree, which means that the needs to be importable when setup.py runs. --- nipy/setup.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/nipy/setup.py b/nipy/setup.py index 7a4eb661ec..c2f8b4a700 100644 --- a/nipy/setup.py +++ b/nipy/setup.py @@ -1,11 +1,19 @@ -from __future__ import absolute_import # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: +from __future__ import absolute_import import os +import sys -from nipy.externals.six import string_types -from nipy.externals.six.moves.configparser import ConfigParser +# Cannot use internal copy of six because can't import from nipy tree +# This is to allow setup.py to run without a full nipy +PY3 = sys.version_info[0] == 3 +if PY3: + string_types = str, + from configparser import ConfigParser +else: + string_types = basestring, + from ConfigParser import ConfigParser NIPY_DEFAULTS = dict() From 2727fe36d59419dfc8b88755a435bf4a480f6716 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Sat, 17 Oct 2015 13:15:09 -0700 Subject: [PATCH 5/7] TST: test pip install with minimal system Don't preinstall packages, except numpy; let pip install dependencies for some tests. --- .travis.yml | 23 +++++++++++++++++------ nipy/info.py | 2 ++ requirements.txt | 5 +++++ 3 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 requirements.txt diff --git a/.travis.yml b/.travis.yml index edc5f73447..16874d7022 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,17 +33,24 @@ matrix: # Absolute minimum dependencies - python: 2.7 env: + # Definitive source for these in nipy/info.py - DEPENDS="numpy==1.6.0 scipy==0.9.0 sympy==0.7.0 nibabel==1.2.0" # Test compiling against external lapack - python: 3.4 env: - NIPY_EXTERNAL_LAPACK=1 - - python: 3.4 + - python: 2.7 env: - INSTALL_TYPE=sdist - - python: 3.4 + - DEPENDS="numpy==1.6.0" + - python: 2.7 env: - INSTALL_TYPE=wheel + - DEPENDS="numpy==1.6.0" + - python: 2.7 + env: + - INSTALL_TYPE=requirements + - DEPENDS= before_install: - source tools/travis_tools.sh @@ -64,12 +71,16 @@ install: if [ "$INSTALL_TYPE" == "setup" ]; then python setup.py install elif [ "$INSTALL_TYPE" == "sdist" ]; then - python setup.py sdist - pip install dist/*.tar.gz + python setup_egg.py egg_info # check egg_info while we're here + python setup_egg.py sdist + wheelhouse_pip_install dist/*.tar.gz elif [ "$INSTALL_TYPE" == "wheel" ]; then pip install wheel - python setup.py bdist_wheel - pip install dist/*.whl + python setup_egg.py bdist_wheel + wheelhouse_pip_install dist/*.whl + elif [ "$INSTALL_TYPE" == "requirements" ]; then + wheelhouse_pip_install -r requirements.txt + python setup.py install fi # command to run tests, e.g. python setup.py test script: diff --git a/nipy/info.py b/nipy/info.py index 760e8e21f3..d0622991bd 100644 --- a/nipy/info.py +++ b/nipy/info.py @@ -131,6 +131,8 @@ # minimum versions # Update in readme text above +# Update in .travis.yml +# Update in requirements.txt NUMPY_MIN_VERSION='1.6.0' SCIPY_MIN_VERSION = '0.9.0' NIBABEL_MIN_VERSION = '1.2' diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..61e7e7284c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +# See nipy/info.py for requirement definitions +numpy>=1.6.0 +scipy>=0.9.0 +sympy>=0.7.0 +nibabel>=1.2.0 From 9fbe26cd30b17915859f66726f2b8194083e3a9a Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Sat, 17 Oct 2015 14:33:40 -0700 Subject: [PATCH 6/7] BF: add testing data to MANIFEST.in Nipy should add these as data in the various setup.py files, but this does not seem to be working. Add them explicitly in the MANIFEST.in --- MANIFEST.in | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 1cd5277a28..c6603c2aeb 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,6 +9,11 @@ recursive-include tools * # put this stuff back into setup.py (package_data) once I'm enlightened # enough to accomplish this herculean task recursive-include nipy/algorithms/tests/data * +include nipy/testing/*.nii.gz +include nipy/algorithms/diagnostics/tests/data/*.mat +include nipy/algorithms/statistics/models/tests/*.bin +include nipy/modalities/fmri/tests/*.npz +include nipy/modalities/fmri/tests/*.mat include nipy/COMMIT_INFO.txt include LICENSE graft examples From 72df1759d0215de57da80289040192c0e416e430 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Sat, 17 Oct 2015 14:48:56 -0700 Subject: [PATCH 7/7] RF: let setup.py sdist etc to run without nibabel Data package message needed nibabel. Make this optional. --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6f334bbfbd..21af1e04fc 100755 --- a/setup.py +++ b/setup.py @@ -84,7 +84,12 @@ def configuration(parent_package='',top_path=None): def data_install_msgs(): # Check whether we have data packages - from nibabel.data import datasource_or_bomber + try: # Allow setup.py to run without nibabel + from nibabel.data import datasource_or_bomber + except ImportError: + log.warn('Cannot check for optional data packages: see: ' + 'http://nipy.org/nipy/stable/users/install_data.html') + return DATA_PKGS = INFO_VARS['DATA_PKGS'] templates = datasource_or_bomber(DATA_PKGS['nipy-templates']) example_data = datasource_or_bomber(DATA_PKGS['nipy-data'])