Merge pull request #377 from matthew-brett/refactor-setup.py

matthew-brett · matthew-brett · commit 125245e676f8 · 2015-10-17T17:47:24.000-07:00
MRG: refactoring setup.py to fix sdist etc python setup.py sdist was broken, because I wasn't adding some files to the MANIFEST.in, and because I wasn't correctly using the numpy build_py command. In other news, it was annoying to have to install sympy etc to run python setup.py egg_info, which was in turn required when installing via pip - see gh-320. Fix both of these by dropping checks for Mayavi and Cython, dropping nibabel/nisext dependency (by copying code and refactoring), and letting setuptools take care of dependency checking for missing packages.
diff --git a/.travis.yml b/.travis.yml
@@ -17,6 +17,7 @@ cache:
 env:
     global:
         - DEPENDS="numpy scipy sympy matplotlib nibabel"
+        - INSTALL_TYPE="setup"
 python:
     - 2.6
     - 3.2
@@ -32,11 +33,25 @@ matrix:
     # Absolute minimum dependencies
     - python: 2.7
       env:
+        # Definitive source for these in nipy/info.py
         - DEPENDS="numpy==1.6.0 scipy==0.9.0 sympy==0.7.0 nibabel==1.2.0"
     # Test compiling against external lapack
     - python: 3.4
       env:
         - NIPY_EXTERNAL_LAPACK=1
+    - python: 2.7
+      env:
+        - INSTALL_TYPE=sdist
+        - DEPENDS="numpy==1.6.0"
+    - python: 2.7
+      env:
+        - INSTALL_TYPE=wheel
+        - DEPENDS="numpy==1.6.0"
+    - python: 2.7
+      env:
+        - INSTALL_TYPE=requirements
+        - DEPENDS=
+
 before_install:
     - source tools/travis_tools.sh
     - virtualenv --python=python venv
@@ -52,7 +67,21 @@ before_install:
 # command to install dependencies
 # e.g. pip install -r requirements.txt # --use-mirrors
 install:
-    - python setup.py install
+    - |
+      if [ "$INSTALL_TYPE" == "setup" ]; then
+          python setup.py install
+      elif [ "$INSTALL_TYPE" == "sdist" ]; then
+        python setup_egg.py egg_info  # check egg_info while we're here
+        python setup_egg.py sdist
+        wheelhouse_pip_install dist/*.tar.gz
+      elif [ "$INSTALL_TYPE" == "wheel" ]; then
+        pip install wheel
+        python setup_egg.py bdist_wheel
+        wheelhouse_pip_install dist/*.whl
+      elif [ "$INSTALL_TYPE" == "requirements" ]; then
+        wheelhouse_pip_install -r requirements.txt
+        python setup.py install
+      fi
 # command to run tests, e.g. python setup.py test
 script:
     # Change into an innocuous directory and find tests from installation
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -3,12 +3,17 @@ include Changelog TODO
 include *.py
 include site.*
 recursive-include nipy *.c *.h *.pyx *.pxd
-recursive-include libcstat *.c *.h *.pyx *.pxd
+recursive-include lib *.c *.h *.pyx *.pxd remake
 recursive-include scripts *
 recursive-include tools *
 # put this stuff back into setup.py (package_data) once I'm enlightened
 # enough to accomplish this herculean task
 recursive-include nipy/algorithms/tests/data *
+include nipy/testing/*.nii.gz
+include nipy/algorithms/diagnostics/tests/data/*.mat
+include nipy/algorithms/statistics/models/tests/*.bin
+include nipy/modalities/fmri/tests/*.npz
+include nipy/modalities/fmri/tests/*.mat
 include nipy/COMMIT_INFO.txt
 include LICENSE
 graft examples
diff --git a/nipy/info.py b/nipy/info.py
@@ -131,6 +131,8 @@
 
 # minimum versions
 # Update in readme text above
+# Update in .travis.yml
+# Update in requirements.txt
 NUMPY_MIN_VERSION='1.6.0'
 SCIPY_MIN_VERSION = '0.9.0'
 NIBABEL_MIN_VERSION = '1.2'
diff --git a/nipy/setup.py b/nipy/setup.py
@@ -1,11 +1,19 @@
-from __future__ import absolute_import
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
 # vi: set ft=python sts=4 ts=4 sw=4 et:
+from __future__ import absolute_import
 
 import os
+import sys
 
-from nipy.externals.six import string_types
-from nipy.externals.six.moves.configparser import ConfigParser
+# Cannot use internal copy of six because can't import from nipy tree
+# This is to allow setup.py to run without a full nipy
+PY3 = sys.version_info[0] == 3
+if PY3:
+    string_types = str,
+    from configparser import ConfigParser
+else:
+    string_types = basestring,
+    from ConfigParser import ConfigParser
 
 NIPY_DEFAULTS = dict()
 
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,5 @@
+# See nipy/info.py for requirement definitions
+numpy>=1.6.0
+scipy>=0.9.0
+sympy>=0.7.0
+nibabel>=1.2.0
diff --git a/setup.py b/setup.py
@@ -11,22 +11,17 @@
 # update it when the contents of directories change.
 if exists('MANIFEST'): os.remove('MANIFEST')
 
-# Import build helpers
-try:
-    from nisext.sexts import package_check, get_comrec_build
-except ImportError:
-    raise RuntimeError('Need nisext package from nibabel installation'
-                       ' - please install nibabel first')
-
-from setup_helpers import (generate_a_pyrex_source,
-                           cmdclass, INFO_VARS)
+from setup_helpers import (generate_a_pyrex_source, get_comrec_build,
+                           cmdclass, INFO_VARS, get_pkg_version,
+                           version_error_msg)
 
 # monkey-patch numpy distutils to use Cython instead of Pyrex
 from numpy.distutils.command.build_src import build_src
 build_src.generate_a_pyrex_source = generate_a_pyrex_source
 
 # Add custom commit-recording build command
-cmdclass['build_py'] = get_comrec_build('nipy')
+from numpy.distutils.command.build_py import build_py as _build_py
+cmdclass['build_py'] = get_comrec_build('nipy', _build_py)
 
 def configuration(parent_package='',top_path=None):
     from numpy.distutils.misc_util import Configuration
@@ -59,42 +54,28 @@ def configuration(parent_package='',top_path=None):
 if not 'extra_setuptools_args' in globals():
     extra_setuptools_args = dict()
 
-
 # Hard and soft dependency checking
-package_check('numpy', INFO_VARS['NUMPY_MIN_VERSION'])
-package_check('scipy', INFO_VARS['SCIPY_MIN_VERSION'])
-package_check('nibabel', INFO_VARS['NIBABEL_MIN_VERSION'])
-package_check('sympy', INFO_VARS['SYMPY_MIN_VERSION'])
-def _mayavi_version(pkg_name):
-    """Mayavi2 pruned enthought. namespace at 4.0.0
-    """
-    v = ''
-    try:
-        from mayavi import version
-        v = version.version
-        if v == '':
-            v = '4.0.0' # must be the one in Debian
-    except ImportError:
-        from enthought.mayavi import version
-        v = version.version
-    return v
-package_check('mayavi',
-              INFO_VARS['MAYAVI_MIN_VERSION'],
-              optional=True,
-              version_getter=_mayavi_version)
-# Cython can be a build dependency
-def _cython_version(pkg_name):
-    from Cython.Compiler.Version import version
-    return version
-package_check('cython',
-              INFO_VARS['CYTHON_MIN_VERSION'],
-              optional=True,
-              version_getter=_cython_version,
-              messages={'opt suffix': ' - you will not be able '
-                        'to rebuild Cython source files into C files',
-                        'missing opt': 'Missing optional build-time '
-                        'package "%s"'}
-              )
+DEPS = (
+    ('numpy', INFO_VARS['NUMPY_MIN_VERSION'], 'setup_requires', True),
+    ('scipy', INFO_VARS['SCIPY_MIN_VERSION'], 'install_requires', True),
+    ('nibabel', INFO_VARS['NIBABEL_MIN_VERSION'], 'install_requires', False),
+    ('sympy', INFO_VARS['SYMPY_MIN_VERSION'], 'install_requires', False))
+
+using_setuptools = 'setuptools' in sys.modules
+
+for name, min_ver, req_type, heavy in DEPS:
+    found_ver = get_pkg_version(name)
+    ver_err_msg = version_error_msg(name, found_ver, min_ver)
+    if not using_setuptools:
+        if ver_err_msg != None:
+            raise RuntimeError(ver_err_msg)
+    else:  # Using setuptools
+        # Add packages to given section of setup/install_requires
+        if ver_err_msg != None or not heavy:
+            new_req = '{0}>={1}'.format(name, min_ver)
+            old_reqs = extra_setuptools_args.get(req_type, [])
+            extra_setuptools_args[req_type] = old_reqs + [new_req]
+
 
 ################################################################################
 # commands for installing the data
@@ -103,7 +84,12 @@ def _cython_version(pkg_name):
 
 def data_install_msgs():
     # Check whether we have data packages
-    from nibabel.data import datasource_or_bomber
+    try:  # Allow setup.py to run without nibabel
+        from nibabel.data import datasource_or_bomber
+    except ImportError:
+        log.warn('Cannot check for optional data packages: see: '
+                 'http://nipy.org/nipy/stable/users/install_data.html')
+        return
     DATA_PKGS = INFO_VARS['DATA_PKGS']
     templates = datasource_or_bomber(DATA_PKGS['nipy-templates'])
     example_data = datasource_or_bomber(DATA_PKGS['nipy-data'])
diff --git a/setup_helpers.py b/setup_helpers.py
@@ -23,10 +23,16 @@
 from distutils.cmd import Command
 from distutils.command.clean import clean
 from distutils.command.install_scripts import install_scripts
+from distutils.command.build_py import build_py
 from distutils.version import LooseVersion
 from distutils.dep_util import newer_group
 from distutils.errors import DistutilsError
 
+try:
+    from ConfigParser import ConfigParser
+except ImportError:  # Python 3
+    from configparser import ConfigParser
+
 from numpy.distutils.misc_util import appendpath
 from numpy.distutils import log
 
@@ -308,6 +314,96 @@ def run(self):
             with open(bat_file, 'wt') as fobj:
                 fobj.write(bat_contents)
 
+# This copied from nibabel/nisext/sexts.py
+# We'll probably drop this way of doing versioning soon
+def get_comrec_build(pkg_dir, build_cmd=build_py):
+    """ Return extended build command class for recording commit
+
+    The extended command tries to run git to find the current commit, getting
+    the empty string if it fails.  It then writes the commit hash into a file
+    in the `pkg_dir` path, named ``COMMIT_INFO.txt``.
+
+    In due course this information can be used by the package after it is
+    installed, to tell you what commit it was installed from if known.
+
+    To make use of this system, you need a package with a COMMIT_INFO.txt file -
+    e.g. ``myproject/COMMIT_INFO.txt`` - that might well look like this::
+
+        # This is an ini file that may contain information about the code state
+        [commit hash]
+        # The line below may contain a valid hash if it has been substituted during 'git archive'
+        archive_subst_hash=$Format:%h$
+        # This line may be modified by the install process
+        install_hash=
+
+    The COMMIT_INFO file above is also designed to be used with git substitution
+    - so you probably also want a ``.gitattributes`` file in the root directory
+    of your working tree that contains something like this::
+
+       myproject/COMMIT_INFO.txt export-subst
+
+    That will cause the ``COMMIT_INFO.txt`` file to get filled in by ``git
+    archive`` - useful in case someone makes such an archive - for example with
+    via the github 'download source' button.
+
+    Although all the above will work as is, you might consider having something
+    like a ``get_info()`` function in your package to display the commit
+    information at the terminal.  See the ``pkg_info.py`` module in the nipy
+    package for an example.
+    """
+    class MyBuildPy(build_cmd):
+        ''' Subclass to write commit data into installation tree '''
+        def run(self):
+            build_cmd.run(self)
+            import subprocess
+            proc = subprocess.Popen('git rev-parse --short HEAD',
+                                    stdout=subprocess.PIPE,
+                                    stderr=subprocess.PIPE,
+                                    shell=True)
+            repo_commit, _ = proc.communicate()
+            # Fix for python 3
+            repo_commit = str(repo_commit)
+            # We write the installation commit even if it's empty
+            cfg_parser = ConfigParser()
+            cfg_parser.read(pjoin(pkg_dir, 'COMMIT_INFO.txt'))
+            cfg_parser.set('commit hash', 'install_hash', repo_commit)
+            out_pth = pjoin(self.build_lib, pkg_dir, 'COMMIT_INFO.txt')
+            cfg_parser.write(open(out_pth, 'wt'))
+    return MyBuildPy
+
+
+def get_pkg_version(pkg_name):
+    """ Return package version for `pkg_name` if installed
+
+    Returns
+    -------
+    pkg_version : str or None
+        Return None if package not importable.  Return 'unknown' if standard
+        ``__version__`` string not present. Otherwise return version string.
+    """
+    try:
+        pkg = __import__(pkg_name)
+    except ImportError:
+        return None
+    try:
+        return pkg.__version__
+    except AttributeError:
+        return 'unknown'
+
+
+def version_error_msg(pkg_name, found_ver, min_ver):
+    """ Return informative error message for version or None
+    """
+    if found_ver is None:
+        return 'We need package {0}, but not importable'.format(pkg_name)
+    if found_ver == 'unknown':
+        msg = 'We need {0} version {1}, but cannot get version'.format(
+            pkg_name, min_ver)
+    if LooseVersion(found_ver) >= LooseVersion(min_ver):
+        return None
+    return 'We need {0} version {1}, but found version {2}'.format(
+        pkg_name, found_ver, min_ver)
+
 
 # The command classes for distutils, used by setup.py
 cmdclass = {'api_docs': APIDocs,
