From 8b87cce398722fc2df7e49d80283d699b5fc2dee Mon Sep 17 00:00:00 2001 From: Holger Peters Date: Thu, 16 Oct 2014 16:50:11 +0200 Subject: [PATCH 1/8] Change matplotlib's nose dependency to a test dependency - add setup.py test command for test execution - modify setupext's logic for dependency determination for a test requirements list - add nose to test requirements - update documentation accordingly This reverts commit 3d622fe8f01cd2c7ac496b5e61d96ec70f030228. install mock for dock building --- .travis.yml | 13 ++-- Makefile | 4 +- README.rst | 2 +- doc/devel/release_guide.rst | 4 +- doc/devel/testing.rst | 50 ++++++++++----- setup.py | 120 ++++++++++++++++++++++++++++++++++++ setupext.py | 8 ++- tox.ini | 1 + 8 files changed, 173 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index d1f0f20bbbef..55ca15fc937b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,9 +25,9 @@ env: - secure: E7OCdqhZ+PlwJcn+Hd6ns9TDJgEUXiUNEI0wu7xjxB2vBRRIKtZMbuaZjd+iKDqCKuVOJKu0ClBUYxmgmpLicTwi34CfTUYt6D4uhrU+8hBBOn1iiK51cl/aBvlUUrqaRLVhukNEBGZcyqAjXSA/Qsnp2iELEmAfOUa92ZYo1sk= - secure: "dfjNqGKzQG5bu3FnDNwLG8H/C4QoieFo4PfFmZPdM2RY7WIzukwKFNT6kiDfOrpwt+2bR7FhzjOGlDECGtlGOtYPN8XuXGjhcP4a4IfakdbDfF+D3NPIpf5VlE6776k0VpvcZBTMYJKNFIMc7QPkOwjvNJ2aXyfe3hBuGlKJzQU=" - BUILD_DOCS=false - - TEST_ARGS=--no-pep8 - NUMPY=numpy - NPROC=2 + - TEST_ARGS=--omit-pep8 language: python @@ -40,7 +40,7 @@ matrix: - python: 3.3 - python: 3.4 - python: 2.7 - env: TEST_ARGS=--pep8 + env: TEST_ARGS=--pep8-only - python: 2.7 env: BUILD_DOCS=true MOCK=true - python: "nightly" @@ -79,7 +79,7 @@ install: # Neihter is installed as a dependency of IPython since they are not used by the IPython console. - | if [[ $BUILD_DOCS == true ]]; then - pip install $PRE numpydoc ipython jsonschema mistune + pip install $PRE numpydoc ipython jsonschema mistune mock pip install -q $PRE linkchecker wget https://github.com/google/fonts/blob/master/ofl/felipa/Felipa-Regular.ttf?raw=true -O Felipa-Regular.ttf wget http://mirrors.kernel.org/ubuntu/pool/universe/f/fonts-humor-sans/fonts-humor-sans_1.0-1_all.deb @@ -103,9 +103,10 @@ script: - | if [[ $BUILD_DOCS == false ]]; then export MPL_REPO_DIR=$PWD # needed for pep8-conformance test of the examples - mkdir ../tmp_test_dir - cd ../tmp_test_dir - gdb -return-child-result -batch -ex r -ex bt --args python ../matplotlib/tests.py -s --processes=$NPROC --process-timeout=300 $TEST_ARGS + # mkdir ../tmp_test_dir + # cd ../tmp_test_dir + # gdb -return-child-result -batch -ex r -ex bt --args python ../matplotlib/tests.py -s --processes=$NPROC --process-timeout=300 $TEST_ARGS + gdb -return-child-result -batch -ex r -ex bt --args python setup.py test --nocapture --nose-verbose --processes=8 --process-timeout=300 $TEST_ARGS else cd doc python make.py html --small --warningsaserrors diff --git a/Makefile b/Makefile index 626b78d009fa..397e45c61513 100644 --- a/Makefile +++ b/Makefile @@ -45,10 +45,10 @@ jdh_doc_snapshot: test: - ${PYTHON} tests.py + ${PYTHON} setup.py test test-coverage: - ${PYTHON} tests.py --with-coverage --cover-package=matplotlib + ${PYTHON} setup.py test --with-coverage --cover-package=matplotlib diff --git a/README.rst b/README.rst index 6e01ea7275cc..af3dbdd95b17 100644 --- a/README.rst +++ b/README.rst @@ -20,7 +20,7 @@ Testing After installation, you can launch the test suite:: - python tests.py + python setup.py tests Consider reading http://matplotlib.org/devel/coding_guide.html#testing for more information. diff --git a/doc/devel/release_guide.rst b/doc/devel/release_guide.rst index 458b33ab8a6b..14f5420f085e 100644 --- a/doc/devel/release_guide.rst +++ b/doc/devel/release_guide.rst @@ -13,8 +13,8 @@ A guide for developers who are doing a matplotlib release. Testing ======= -* Run all of the regression tests by running the `tests.py` script at - the root of the source tree. +* Run all of the regression tests by running ``python setup.py tests`` script + at the root of the source tree. * Run :file:`unit/memleak_hawaii3.py` and make sure there are no memory leaks diff --git a/doc/devel/testing.rst b/doc/devel/testing.rst index 1a62dab34532..5e74eba380f5 100644 --- a/doc/devel/testing.rst +++ b/doc/devel/testing.rst @@ -37,14 +37,24 @@ Running the tests ----------------- Running the tests is simple. Make sure you have nose installed and run -the script :file:`tests.py` in the root directory of the distribution. -The script can take any of the usual `nosetest arguments`_, such as +the setup script's ``test`` command:: -=================== =========== -``-v`` increase verbosity -``-d`` detailed error messages -``--with-coverage`` enable collecting coverage information -=================== =========== + python setup.py test + +in the root directory of the distribution. The script takes a set of +commands, such as: + +======================== =========== +``--pep8-only`` pep8 checks +``--omit-pep8`` Do not perform pep8 checks +``--nocapture`` do not capture stdout (nosetests) +``--nose-verbose`` be verbose (nosetests) +``--processes`` number of processes (nosetests) +``--process-timeout`` process timeout (nosetests) +``--with-coverage`` with coverage +``--detailed-error-msg`` detailed error message (nosetest) +``--tests`` comma separated selection of tests (nosetest) +======================== =========== Additionally it is possible to run only coding standard test or disable them: @@ -57,28 +67,34 @@ To run a single test from the command line, you can provide a dot-separated path to the module followed by the function separated by a colon, e.g., (this is assuming the test is installed):: - python tests.py matplotlib.tests.test_simplification:test_clipping - -If you want to run the full test suite, but want to save wall time try running the -tests in parallel:: + python setup.py test --tests=matplotlib.tests.test_simplification:test_clipping - python ../matplotlib/tests.py -sv --processes=5 --process-timeout=300 +If you want to run the full test suite, but want to save wall time try +running the tests in parallel:: -as we do on Travis.ci. + python setup.py test --nocapture --nose-verbose --processes=5 --process-timeout=300 An alternative implementation that does not look at command line -arguments works from within Python:: +arguments works from within Python is to run the tests from the +matplotlib library function :func:`matplotlib.test`:: import matplotlib matplotlib.test() +.. hint:: + + You might need to install nose for this:: + + pip install nose + + .. _`nosetest arguments`: http://nose.readthedocs.org/en/latest/usage.html -Running tests by any means other than `matplotlib.test()` -does not load the nose "knownfailureif" (Known failing tests) plugin, -causing known-failing tests to fail for real. +Running tests by any means other than `matplotlib.test()` does not +load the nose "knownfailureif" (Known failing tests) plugin, causing +known-failing tests to fail for real. Writing a simple test --------------------- diff --git a/setup.py b/setup.py index 8c02117db85a..81b29ec0018f 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,7 @@ # This needs to be the very first thing to use distribute from distribute_setup import use_setuptools use_setuptools() +from setuptools.command.test import test as TestCommand import sys @@ -121,6 +122,121 @@ 'Topic :: Scientific/Engineering :: Visualization', ] + +class NoseTestCommand(TestCommand): + """Invoke unit tests using nose after an in-place build.""" + + description = "Invoke unit tests using nose after an in-place build." + user_options = [ + ("pep8-only", None, "pep8 checks"), + ("omit-pep8", None, "Do not perform pep8 checks"), + ("nocapture", None, "do not capture stdout (nosetests)"), + ("nose-verbose", None, "be verbose (nosetests)"), + ("processes=", None, "number of processes (nosetests)"), + ("process-timeout=", None, "process timeout (nosetests)"), + ("with-coverage", None, "with coverage"), + ("detailed-error-msg", None, "detailed error message (nosetest)"), + ("tests=", None, "comma separated selection of tests (nosetest)"), + ] + + def initialize_options(self): + self.pep8_only = None + self.omit_pep8 = None + + # parameters passed to nose tests + self.processes = None + self.process_timeout = None + self.nose_verbose = None + self.nocapture = None + self.with_coverage = None + self.detailed_error_msg = None + self.tests = None + + def finalize_options(self): + self.test_args = [] + if self.pep8_only: + self.pep8_only = True + if self.omit_pep8: + self.omit_pep8 = True + + if self.pep8_only and self.omit_pep8: + from distutils.errors import DistutilsOptionError + raise DistutilsOptionError( + "You are using several options for the test command in an " + "incompatible manner. Please use either one of --pep8-only," + "--omit-pep8" + ) + + if self.processes: + self.test_args.append("--processes={prc}".format( + prc=self.processes)) + + if self.process_timeout: + self.test_args.append("--process-timeout={tout}".format( + tout=self.process_timeout)) + + if self.nose_verbose: + self.test_args.append("--verbose") + + if self.nocapture: + self.test_args.append("--nocapture") + + if self.with_coverage: + self.test_args.append("--with-coverage") + + if self.detailed_error_msg: + self.test_args.append("-d") + + if self.tests: + self.test_args.append("--tests={names}".format(names=self.tests)) + + + def run(self): + if self.distribution.install_requires: + self.distribution.fetch_build_eggs( + self.distribution.install_requires) + if self.distribution.tests_require: + self.distribution.fetch_build_eggs(self.distribution.tests_require) + + self.announce('running unittests with nose') + self.with_project_on_sys_path(self.run_tests) + + + def run_tests(self): + try: + import matplotlib + matplotlib.use('agg') + import nose + from matplotlib.testing.noseclasses import KnownFailure + from matplotlib import default_test_modules as testmodules + from matplotlib import font_manager + import time + # Make sure the font caches are created before starting any possibly + # parallel tests + if font_manager._fmcache is not None: + while not os.path.exists(font_manager._fmcache): + time.sleep(0.5) + plugins = [KnownFailure] + + # Nose doesn't automatically instantiate all of the plugins in the + # child processes, so we have to provide the multiprocess plugin + # with a list. + from nose.plugins import multiprocess + multiprocess._instantiate_plugins = plugins + + if self.omit_pep8: + testmodules.remove('matplotlib.tests.test_coding_standards') + elif self.pep8_only: + testmodules = ['matplotlib.tests.test_coding_standards'] + + nose.main(addplugins=[x() for x in plugins], + defaultTest=testmodules, + argv=['nosetests'] + self.test_args, + exit=False) + except ImportError: + sys.exit(-1) + + # One doesn't normally see `if __name__ == '__main__'` blocks in a setup.py, # however, this is needed on Windows to avoid creating infinite subprocesses # when using multiprocessing. @@ -135,6 +251,7 @@ package_dir = {'': 'lib'} install_requires = [] setup_requires = [] + tests_require = [] default_backend = None # Go through all of the packages and figure out which ones we are @@ -195,6 +312,7 @@ package_data[key] = list(set(val + package_data[key])) install_requires.extend(package.get_install_requires()) setup_requires.extend(package.get_setup_requires()) + tests_require.extend(package.get_tests_require()) # Write the default matplotlibrc file if default_backend is None: @@ -254,11 +372,13 @@ # List third-party Python packages that we require install_requires=install_requires, setup_requires=setup_requires, + tests_require=tests_require, # matplotlib has C/C++ extensions, so it's not zip safe. # Telling setuptools this prevents it from doing an automatic # check for zip safety. zip_safe=False, + cmdclass={'test': NoseTestCommand}, **extra_args ) diff --git a/setupext.py b/setupext.py index d33b59e73323..c0461b9b1355 100755 --- a/setupext.py +++ b/setupext.py @@ -415,6 +415,12 @@ def get_setup_requires(self): """ return [] + def get_tests_require(self): + """ + Get a list of Python packages that we require for executing tests. + """ + return [] + def _check_for_pkg_config(self, package, include_file, min_version=None, version=None): """ @@ -694,7 +700,7 @@ def get_package_data(self): 'sphinxext/tests/tinypages/_static/*', ]} - def get_install_requires(self): + def get_tests_require(self): requires = ['nose>=%s' % self.nose_min_version] if not sys.version_info >= (3, 3): requires += ['mock'] diff --git a/tox.ini b/tox.ini index b34e6ea7af13..296cefb56281 100644 --- a/tox.ini +++ b/tox.ini @@ -13,4 +13,5 @@ commands = {envpython} {toxinidir}/tests.py --processes=-1 --process-timeout=300 deps = nose + mock numpy From 4286abe9135d98485b176c8c693c89dda9f26f5c Mon Sep 17 00:00:00 2001 From: Jens Hedegaard Nielsen Date: Sat, 11 Jul 2015 14:54:52 -0500 Subject: [PATCH 2/8] Fixes to .travis to match local changes --- .travis.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 55ca15fc937b..3d28df727e89 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,7 +79,7 @@ install: # Neihter is installed as a dependency of IPython since they are not used by the IPython console. - | if [[ $BUILD_DOCS == true ]]; then - pip install $PRE numpydoc ipython jsonschema mistune mock + pip install $PRE numpydoc ipython jsonschema mistune pip install -q $PRE linkchecker wget https://github.com/google/fonts/blob/master/ofl/felipa/Felipa-Regular.ttf?raw=true -O Felipa-Regular.ttf wget http://mirrors.kernel.org/ubuntu/pool/universe/f/fonts-humor-sans/fonts-humor-sans_1.0-1_all.deb @@ -103,10 +103,7 @@ script: - | if [[ $BUILD_DOCS == false ]]; then export MPL_REPO_DIR=$PWD # needed for pep8-conformance test of the examples - # mkdir ../tmp_test_dir - # cd ../tmp_test_dir - # gdb -return-child-result -batch -ex r -ex bt --args python ../matplotlib/tests.py -s --processes=$NPROC --process-timeout=300 $TEST_ARGS - gdb -return-child-result -batch -ex r -ex bt --args python setup.py test --nocapture --nose-verbose --processes=8 --process-timeout=300 $TEST_ARGS + gdb -return-child-result -batch -ex r -ex bt --args python setup.py test --nocapture --processes=$NPROC --process-timeout=300 $TEST_ARGS else cd doc python make.py html --small --warningsaserrors From 9767d498807c244b40a5d0b43422b7f26db7ab92 Mon Sep 17 00:00:00 2001 From: Jens Hedegaard Nielsen Date: Sat, 11 Jul 2015 15:25:57 -0500 Subject: [PATCH 3/8] Require sphinx for tests --- setupext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setupext.py b/setupext.py index c0461b9b1355..23c4ff899aa1 100755 --- a/setupext.py +++ b/setupext.py @@ -701,7 +701,7 @@ def get_package_data(self): ]} def get_tests_require(self): - requires = ['nose>=%s' % self.nose_min_version] + requires = ['nose>=%s' % self.nose_min_version, 'sphinx'] if not sys.version_info >= (3, 3): requires += ['mock'] return requires From ff34bb931a16eb8040cd837a7bf971e42df415c6 Mon Sep 17 00:00:00 2001 From: Jens Hedegaard Nielsen Date: Sat, 11 Jul 2015 16:41:19 -0500 Subject: [PATCH 4/8] Clarify docs of testing --- README.rst | 7 ++++++- doc/devel/release_guide.rst | 2 +- doc/devel/testing.rst | 4 ---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index af3dbdd95b17..2f7d114654ff 100644 --- a/README.rst +++ b/README.rst @@ -20,7 +20,12 @@ Testing After installation, you can launch the test suite:: - python setup.py tests + python setup.py test + +Or from the python interpreter:: + + import matplotlib + matplotlib.test() Consider reading http://matplotlib.org/devel/coding_guide.html#testing for more information. diff --git a/doc/devel/release_guide.rst b/doc/devel/release_guide.rst index 14f5420f085e..faf5cf619636 100644 --- a/doc/devel/release_guide.rst +++ b/doc/devel/release_guide.rst @@ -13,7 +13,7 @@ A guide for developers who are doing a matplotlib release. Testing ======= -* Run all of the regression tests by running ``python setup.py tests`` script +* Run all of the regression tests by running ``python setup.py test`` script at the root of the source tree. * Run :file:`unit/memleak_hawaii3.py` and make sure there are no diff --git a/doc/devel/testing.rst b/doc/devel/testing.rst index 5e74eba380f5..c84b34946b16 100644 --- a/doc/devel/testing.rst +++ b/doc/devel/testing.rst @@ -92,10 +92,6 @@ matplotlib library function :func:`matplotlib.test`:: .. _`nosetest arguments`: http://nose.readthedocs.org/en/latest/usage.html -Running tests by any means other than `matplotlib.test()` does not -load the nose "knownfailureif" (Known failing tests) plugin, causing -known-failing tests to fail for real. - Writing a simple test --------------------- From 8c30aa362d08afa1c95d5e04beb7b814c95cd129 Mon Sep 17 00:00:00 2001 From: Jens Hedegaard Nielsen Date: Sat, 11 Jul 2015 16:56:17 -0500 Subject: [PATCH 5/8] Setup.py dont catch import error It is more informative to raise the error here --- setup.py | 63 +++++++++++++++++++++++++++----------------------------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/setup.py b/setup.py index 81b29ec0018f..0e0aaa3b7f5e 100644 --- a/setup.py +++ b/setup.py @@ -163,7 +163,7 @@ def finalize_options(self): from distutils.errors import DistutilsOptionError raise DistutilsOptionError( "You are using several options for the test command in an " - "incompatible manner. Please use either one of --pep8-only," + "incompatible manner. Please use either --pep8-only or " "--omit-pep8" ) @@ -203,38 +203,35 @@ def run(self): def run_tests(self): - try: - import matplotlib - matplotlib.use('agg') - import nose - from matplotlib.testing.noseclasses import KnownFailure - from matplotlib import default_test_modules as testmodules - from matplotlib import font_manager - import time - # Make sure the font caches are created before starting any possibly - # parallel tests - if font_manager._fmcache is not None: - while not os.path.exists(font_manager._fmcache): - time.sleep(0.5) - plugins = [KnownFailure] - - # Nose doesn't automatically instantiate all of the plugins in the - # child processes, so we have to provide the multiprocess plugin - # with a list. - from nose.plugins import multiprocess - multiprocess._instantiate_plugins = plugins - - if self.omit_pep8: - testmodules.remove('matplotlib.tests.test_coding_standards') - elif self.pep8_only: - testmodules = ['matplotlib.tests.test_coding_standards'] - - nose.main(addplugins=[x() for x in plugins], - defaultTest=testmodules, - argv=['nosetests'] + self.test_args, - exit=False) - except ImportError: - sys.exit(-1) + import matplotlib + matplotlib.use('agg') + import nose + from matplotlib.testing.noseclasses import KnownFailure + from matplotlib import default_test_modules as testmodules + from matplotlib import font_manager + import time + # Make sure the font caches are created before starting any possibly + # parallel tests + if font_manager._fmcache is not None: + while not os.path.exists(font_manager._fmcache): + time.sleep(0.5) + plugins = [KnownFailure] + + # Nose doesn't automatically instantiate all of the plugins in the + # child processes, so we have to provide the multiprocess plugin + # with a list. + from nose.plugins import multiprocess + multiprocess._instantiate_plugins = plugins + + if self.omit_pep8: + testmodules.remove('matplotlib.tests.test_coding_standards') + elif self.pep8_only: + testmodules = ['matplotlib.tests.test_coding_standards'] + + nose.main(addplugins=[x() for x in plugins], + defaultTest=testmodules, + argv=['nosetests'] + self.test_args, + exit=False) # One doesn't normally see `if __name__ == '__main__'` blocks in a setup.py, From 2fe771581b7449ed76201e99641658b998127afe Mon Sep 17 00:00:00 2001 From: Jens Hedegaard Nielsen Date: Sat, 11 Jul 2015 17:32:07 -0500 Subject: [PATCH 6/8] ignore .eggs directory created when installing mock/nose --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7a34c4005989..341741a83fdd 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ doc/_build dist # Egg metadata *.egg-info +.eggs # tox testing tool .tox From 33493f702a2678fdf6f1448cc77acb19da8311a7 Mon Sep 17 00:00:00 2001 From: Jens Hedegaard Nielsen Date: Sun, 12 Jul 2015 11:40:10 -0500 Subject: [PATCH 7/8] Check deps in matplotlib.test() And print an informative message --- lib/matplotlib/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/matplotlib/__init__.py b/lib/matplotlib/__init__.py index ddd99b01f776..ead63f4f3af6 100644 --- a/lib/matplotlib/__init__.py +++ b/lib/matplotlib/__init__.py @@ -1449,8 +1449,21 @@ def tk_window_focus(): ] +def verify_test_dependencies(): + try: + import nose + try: + from unittest import mock + except: + import mock + except ImportError: + print("matplotlib.test requires nose and mock to run.") + raise + + def test(verbosity=1): """run the matplotlib test suite""" + verify_test_dependencies() try: import faulthandler except ImportError: From f7571664133856cc4f6feca983760b594d253c5f Mon Sep 17 00:00:00 2001 From: Jens Hedegaard Nielsen Date: Sun, 12 Jul 2015 11:44:58 -0500 Subject: [PATCH 8/8] Correct setup.py message for test deps --- setupext.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setupext.py b/setupext.py index 23c4ff899aa1..29e549e4d833 100755 --- a/setupext.py +++ b/setupext.py @@ -651,8 +651,8 @@ def check(self): msgs = [] msg_template = ('{package} is required to run the matplotlib test ' - 'suite. pip/easy_install may attempt to install it ' - 'after matplotlib.') + 'suite. "setup.py test" will automatically download it.' + ' Install {package} to run matplotlib.test()') bad_nose = msg_template.format( package='nose %s or later' % self.nose_min_version