From 0da24de7063016b42aee9fce032710d5b7d2d96b Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Mon, 31 Aug 2015 11:16:57 +0100 Subject: [PATCH 1/3] CONFIG: add config option for LAPACK compile Add environment variable and setup.cfg option for compiling against external lapack. Test on travis --- .travis.yml | 10 ++++++++- nipy/labs/setup.py | 56 ++++++++++++++++++++++++++++++++++++---------- setup.cfg | 7 ++++++ 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index c9d439199d..0ee772f25f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,11 @@ # - There can't be any leading "-"s - All newlines will be removed, so use # ";"s sudo: false # To use travis container infrastructure +addons: + apt: + packages: + - libblas-dev + - liblapack-dev language: python cache: directories: @@ -27,6 +32,10 @@ matrix: - python: 2.7 env: - DEPENDS="numpy==1.6.0 scipy==0.7.0 sympy==0.7.0 nibabel==1.2.0" + # Test compiling against external lapack + - python: 3.4 + env: + - NIPY_EXTERNAL_LAPACK=1 before_install: - source tools/travis_tools.sh - virtualenv --python=python venv @@ -34,7 +43,6 @@ before_install: - python --version # just to check - retry pip install nose # always - wheelhouse_pip_install $DEPENDS - # - sudo apt-get install -qq libatlas-dev libatlas-base-dev gfortran libpng-dev - if [ "${COVERAGE}" == "1" ]; then pip install coverage; pip install coveralls; diff --git a/nipy/labs/setup.py b/nipy/labs/setup.py index 108116d8a9..0151a68aa4 100644 --- a/nipy/labs/setup.py +++ b/nipy/labs/setup.py @@ -3,14 +3,42 @@ import os from distutils import log +try: + from configparser import ConfigParser, NoSectionError, NoOptionError +except ImportError: + from ConfigParser import ConfigParser, NoSectionError, NoOptionError + # Global variables LIBS = os.path.realpath('lib') -# The following variable disables linking with Lapack by default. So -# far, the only way to attempt at linking with Lapack is to edit this -# file and set WANT_LAPACK_LINKING=True. We might later want to pass -# in this variable as an optional argument of `python setup.py build`. -WANT_LAPACK_LINKING = False +# Stuff for reading setup file +SETUP_FILE = 'setup.cfg' +SECTION = 'lapack' +KEY = 'external' +EXTERNAL_LAPACK_VAR = 'NIPY_EXTERNAL_LAPACK' + +def get_link_external(): + """ Return True if we should link to system BLAS / LAPACK + + If True, attempt to link to system BLAS / LAPACK. Otherwise, compile + lapack_lite, and link to that. + + First check environment variable NIPY_EXTERNAL_LAPACK for value other than + 'false' or '0'. + + If this variable is undefined, read ``setup.cfg`` file for section + ``[lapack]`` key ``external`` + """ + external_link = os.environ.get(EXTERNAL_LAPACK_VAR) + if external_link is None: + config = ConfigParser() + try: + config.read(SETUP_FILE) + external_link = config.get(SECTION, KEY) + except (IOError, KeyError, NoOptionError, NoSectionError): + return False + return external_link.lower() not in ('0', 'false') + def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration, get_numpy_include_dirs @@ -50,16 +78,20 @@ def configuration(parent_package='',top_path=None): # If lapack linking not required or no lapack install is found, we # use the rescue lapack lite distribution included in the package # (sources have been translated to C using f2c) - if not WANT_LAPACK_LINKING or not lapack_info: - if WANT_LAPACK_LINKING: - log.warn('Lapack not found') - log.warn('Building Lapack lite distribution') + want_lapack_link = get_link_external() + if not want_lapack_link: + log.warn('Building with (slow) Lapack lite distribution: ' + 'set {0} environment variable or use setup.cfg ' + 'to enable link to optimized BLAS / LAPACK'.format( + EXTERNAL_LAPACK_VAR) + ) sources.append(os.path.join(LIBS,'lapack_lite','*.c')) library_dirs = [] libraries = [] - - # Best-case scenario: lapack found - else: + else: # Best-case scenario: external lapack found + if not lapack_info: + raise RuntimeError('Specified external lapack linking but ' + 'numpy does not report external lapack') log.warn('Linking with system Lapack') library_dirs = lapack_info['library_dirs'] libraries = lapack_info['libraries'] diff --git a/setup.cfg b/setup.cfg index 7e365711c7..f4690f0f4d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,3 +19,10 @@ cover-package = nipy #with-doctest=1 doctest-extension=rst +# In this section specify if you want to link to external BLAS / LAPACK +[lapack] +# Value of 0 or False implies compile of, link to lapack_lite +# Value of 1 or True will cause setup to try and link to external BLAS / +# LAPACK as identified with the numpy configuration. Default is False. +# Override with NIPY_EXTERNAL_LAPACK environment variable +#external = False From 2147974bb2900108f1d35b4e48e5c200988f0078 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Mon, 31 Aug 2015 16:25:50 +0100 Subject: [PATCH 2/3] RF: value in setup.cfg file overrides env var Value for lapack linking in setup.cfg now has priority over the environment variable. --- nipy/labs/setup.py | 24 +++++++++++++----------- setup.cfg | 3 ++- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/nipy/labs/setup.py b/nipy/labs/setup.py index 0151a68aa4..97f13168b3 100644 --- a/nipy/labs/setup.py +++ b/nipy/labs/setup.py @@ -23,20 +23,22 @@ def get_link_external(): If True, attempt to link to system BLAS / LAPACK. Otherwise, compile lapack_lite, and link to that. - First check environment variable NIPY_EXTERNAL_LAPACK for value other than - 'false' or '0'. + First check ``setup.cfg`` file for section ``[lapack]`` key ``external``. - If this variable is undefined, read ``setup.cfg`` file for section - ``[lapack]`` key ``external`` + If this value undefined, then get string from environment variable + NIPY_EXTERNAL_LAPACK. + + If value from ``setup.cfg`` or environment variable is not 'False' or '0', + then return True. """ - external_link = os.environ.get(EXTERNAL_LAPACK_VAR) + config = ConfigParser() + try: + config.read(SETUP_FILE) + external_link = config.get(SECTION, KEY) + except (IOError, KeyError, NoOptionError, NoSectionError): + external_link = os.environ.get(EXTERNAL_LAPACK_VAR) if external_link is None: - config = ConfigParser() - try: - config.read(SETUP_FILE) - external_link = config.get(SECTION, KEY) - except (IOError, KeyError, NoOptionError, NoSectionError): - return False + return False return external_link.lower() not in ('0', 'false') diff --git a/setup.cfg b/setup.cfg index f4690f0f4d..d1dba3e565 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,5 +24,6 @@ doctest-extension=rst # Value of 0 or False implies compile of, link to lapack_lite # Value of 1 or True will cause setup to try and link to external BLAS / # LAPACK as identified with the numpy configuration. Default is False. -# Override with NIPY_EXTERNAL_LAPACK environment variable +# The value in this file overrides the equivalent setting in the environment +# variable NIPY_EXTERNAL_LAPACK. #external = False From a2888799c63b1e08c7e0843855a946330f110280 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Wed, 2 Sep 2015 12:49:55 -0700 Subject: [PATCH 3/3] RF: remove extra references to lapack info Trying to sort out compiles with lapack-lite, remove references to lapack info when compiling sub-packages. --- nipy/algorithms/clustering/setup.py | 15 --------------- nipy/labs/bindings/setup.py | 18 +++--------------- nipy/labs/glm/setup.py | 11 +---------- nipy/labs/group/setup.py | 17 +++-------------- nipy/labs/setup.py | 2 +- nipy/labs/utils/setup.py | 12 +----------- 6 files changed, 9 insertions(+), 66 deletions(-) diff --git a/nipy/algorithms/clustering/setup.py b/nipy/algorithms/clustering/setup.py index 7cc08118c2..43950bd18f 100644 --- a/nipy/algorithms/clustering/setup.py +++ b/nipy/algorithms/clustering/setup.py @@ -7,21 +7,6 @@ def configuration(parent_package='',top_path=None): config = Configuration('clustering', parent_package, top_path) config.add_subpackage('tests') - # We need this because libcstat.a is linked to lapack, which can - # be a fortran library, and the linker needs this information. - from numpy.distutils.system_info import get_info - lapack_info = get_info('lapack_opt',0) - if 'libraries' not in lapack_info: - # But on OSX that may not give us what we need, so try with 'lapack' - # instead. NOTE: scipy.linalg uses lapack_opt, not 'lapack'... - lapack_info = get_info('lapack',0) - - #config.add_extension( - # '_clustering', - # sources=['clustering.c'], - # libraries=['cstat'], - # extra_info=lapack_info, - # ) return config diff --git a/nipy/labs/bindings/setup.py b/nipy/labs/bindings/setup.py index ff51ac6305..24f57d0b36 100644 --- a/nipy/labs/bindings/setup.py +++ b/nipy/labs/bindings/setup.py @@ -1,29 +1,17 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -import os, sys def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration config = Configuration('bindings', parent_package, top_path) config.add_subpackage('tests') config.add_subpackage('benchmarks') - # We need this because libcstat.a is linked to lapack, which can - # be a fortran library, and the linker needs this information. - from numpy.distutils.system_info import get_info - lapack_info = get_info('lapack_opt',0) - if 'libraries' not in lapack_info: - # But on OSX that may not give us what we need, so try with 'lapack' - # instead. NOTE: scipy.linalg uses lapack_opt, not 'lapack'... - lapack_info = get_info('lapack',0) config.add_extension('linalg', sources=['linalg.pyx'], - libraries=['cstat'], - extra_info=lapack_info) + libraries=['cstat']) config.add_extension('array', sources=['array.pyx'], - libraries=['cstat'], - extra_info=lapack_info) + libraries=['cstat']) config.add_extension('wrapper', sources=['wrapper.pyx'], - libraries=['cstat'], - extra_info=lapack_info) + libraries=['cstat']) return config diff --git a/nipy/labs/glm/setup.py b/nipy/labs/glm/setup.py index f53457fe5b..e65d272e79 100644 --- a/nipy/labs/glm/setup.py +++ b/nipy/labs/glm/setup.py @@ -3,22 +3,13 @@ def configuration(parent_package='', top_path=None): from numpy.distutils.misc_util import Configuration - # We need this because libcstat.a is linked to lapack, which can - # be a fortran library, and the linker needs this information. - from numpy.distutils.system_info import get_info - lapack_info = get_info('lapack_opt', 0) - if 'libraries' not in lapack_info: - # But on OSX that may not give us what we need, so try with 'lapack' - # instead. NOTE: scipy.linalg uses lapack_opt, not 'lapack'... - lapack_info = get_info('lapack', 0) config = Configuration('glm', parent_package, top_path) config.add_subpackage('tests') config.add_subpackage('benchmarks') config.add_extension( 'kalman', sources=['kalman.pyx'], - libraries=['cstat'], - extra_info=lapack_info, + libraries=['cstat'] ) return config diff --git a/nipy/labs/group/setup.py b/nipy/labs/group/setup.py index 48e5832567..780dde32c5 100644 --- a/nipy/labs/group/setup.py +++ b/nipy/labs/group/setup.py @@ -3,33 +3,22 @@ def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration # We need this because libcstat.a is linked to lapack, which can # be a fortran library, and the linker needs this information. - from numpy.distutils.system_info import get_info - # First, try 'lapack_info', as that seems to provide more details on Linux - # (both 32 and 64 bits): - lapack_info = get_info('lapack_opt', 0) - if 'libraries' not in lapack_info: - # But on OSX that may not give us what we need, so try with 'lapack' - # instead. NOTE: scipy.linalg uses lapack_opt, not 'lapack'... - lapack_info = get_info('lapack',0) config = Configuration('group', parent_package, top_path) config.add_subpackage('tests') config.add_extension( 'onesample', sources=['onesample.pyx'], - libraries=['cstat'], - extra_info=lapack_info, + libraries=['cstat'] ) config.add_extension( 'twosample', sources=['twosample.pyx'], - libraries=['cstat'], - extra_info=lapack_info, + libraries=['cstat'] ) config.add_extension( 'glm_twolevel', sources=['glm_twolevel.pyx'], - libraries=['cstat'], - extra_info=lapack_info, + libraries=['cstat'] ) return config diff --git a/nipy/labs/setup.py b/nipy/labs/setup.py index 97f13168b3..edf995a01b 100644 --- a/nipy/labs/setup.py +++ b/nipy/labs/setup.py @@ -44,7 +44,7 @@ def get_link_external(): def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration, get_numpy_include_dirs - from numpy.distutils.system_info import get_info, system_info + from numpy.distutils.system_info import get_info config = Configuration('labs', parent_package, top_path) diff --git a/nipy/labs/utils/setup.py b/nipy/labs/utils/setup.py index 93bd0ad617..bb782430cc 100644 --- a/nipy/labs/utils/setup.py +++ b/nipy/labs/utils/setup.py @@ -3,22 +3,12 @@ def configuration(parent_package='', top_path=None): from numpy.distutils.misc_util import Configuration - # We need this because libcstat.a is linked to lapack, which can - # be a fortran library, and the linker needs this information. - from numpy.distutils.system_info import get_info - lapack_info = get_info('lapack_opt', 0) - if 'libraries' not in lapack_info: - # But on OSX that may not give us what we need, so try with 'lapack' - # instead. NOTE: scipy.linalg uses lapack_opt, not 'lapack'... - lapack_info = get_info('lapack', 0) - config = Configuration('utils', parent_package, top_path) config.add_subpackage('tests') config.add_extension( 'routines', sources=['routines.pyx'], - libraries=['cstat'], - extra_info=lapack_info, + libraries=['cstat'] ) return config