diff --git a/.travis.yml b/.travis.yml index c3ff8cf8b090..6f2c58150f42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,9 +25,9 @@ env: - secure: E7OCdqhZ+PlwJcn+Hd6ns9TDJgEUXiUNEI0wu7xjxB2vBRRIKtZMbuaZjd+iKDqCKuVOJKu0ClBUYxmgmpLicTwi34CfTUYt6D4uhrU+8hBBOn1iiK51cl/aBvlUUrqaRLVhukNEBGZcyqAjXSA/Qsnp2iELEmAfOUa92ZYo1sk= - secure: "dfjNqGKzQG5bu3FnDNwLG8H/C4QoieFo4PfFmZPdM2RY7WIzukwKFNT6kiDfOrpwt+2bR7FhzjOGlDECGtlGOtYPN8XuXGjhcP4a4IfakdbDfF+D3NPIpf5VlE6776k0VpvcZBTMYJKNFIMc7QPkOwjvNJ2aXyfe3hBuGlKJzQU=" - BUILD_DOCS=false - - TEST_ARGS=--no-pep8 - NUMPY=numpy - NPROC=2 + - TEST_ARGS=--omit-pep8 language: python @@ -39,7 +39,7 @@ matrix: - python: 3.3 - python: 3.4 - python: 2.7 - env: TEST_ARGS=--pep8 + env: TEST_ARGS=--pep8-only - python: 2.7 env: BUILD_DOCS=true - python: "nightly" @@ -72,7 +72,7 @@ install: # Neihter is installed as a dependency of IPython since they are not used by the IPython console. - | if [[ $BUILD_DOCS == true ]]; then - pip install $PRE numpydoc ipython jsonschema mistune + pip install $PRE numpydoc ipython jsonschema mistune mock pip install -q $PRE linkchecker wget https://github.com/google/fonts/blob/master/ofl/felipa/Felipa-Regular.ttf?raw=true -O Felipa-Regular.ttf wget http://mirrors.kernel.org/ubuntu/pool/universe/f/fonts-humor-sans/fonts-humor-sans_1.0-1_all.deb @@ -96,9 +96,10 @@ script: - | if [[ $BUILD_DOCS == false ]]; then export MPL_REPO_DIR=$PWD # needed for pep8-conformance test of the examples - mkdir ../tmp_test_dir - cd ../tmp_test_dir - gdb -return-child-result -batch -ex r -ex bt --args python ../matplotlib/tests.py -s --processes=$NPROC --process-timeout=300 $TEST_ARGS + # mkdir ../tmp_test_dir + # cd ../tmp_test_dir + # gdb -return-child-result -batch -ex r -ex bt --args python ../matplotlib/tests.py -s --processes=$NPROC --process-timeout=300 $TEST_ARGS + gdb -return-child-result -batch -ex r -ex bt --args python setup.py test --nocapture --nose-verbose --processes=8 --process-timeout=300 $TEST_ARGS else cd doc python make.py html --small --warningsaserrors diff --git a/Makefile b/Makefile index 626b78d009fa..397e45c61513 100644 --- a/Makefile +++ b/Makefile @@ -45,10 +45,10 @@ jdh_doc_snapshot: test: - ${PYTHON} tests.py + ${PYTHON} setup.py test test-coverage: - ${PYTHON} tests.py --with-coverage --cover-package=matplotlib + ${PYTHON} setup.py test --with-coverage --cover-package=matplotlib diff --git a/README.rst b/README.rst index 6e01ea7275cc..af3dbdd95b17 100644 --- a/README.rst +++ b/README.rst @@ -20,7 +20,7 @@ Testing After installation, you can launch the test suite:: - python tests.py + python setup.py tests Consider reading http://matplotlib.org/devel/coding_guide.html#testing for more information. diff --git a/doc/devel/release_guide.rst b/doc/devel/release_guide.rst index 458b33ab8a6b..14f5420f085e 100644 --- a/doc/devel/release_guide.rst +++ b/doc/devel/release_guide.rst @@ -13,8 +13,8 @@ A guide for developers who are doing a matplotlib release. Testing ======= -* Run all of the regression tests by running the `tests.py` script at - the root of the source tree. +* Run all of the regression tests by running ``python setup.py tests`` script + at the root of the source tree. * Run :file:`unit/memleak_hawaii3.py` and make sure there are no memory leaks diff --git a/doc/devel/testing.rst b/doc/devel/testing.rst index 1a62dab34532..5e74eba380f5 100644 --- a/doc/devel/testing.rst +++ b/doc/devel/testing.rst @@ -37,14 +37,24 @@ Running the tests ----------------- Running the tests is simple. Make sure you have nose installed and run -the script :file:`tests.py` in the root directory of the distribution. -The script can take any of the usual `nosetest arguments`_, such as +the setup script's ``test`` command:: -=================== =========== -``-v`` increase verbosity -``-d`` detailed error messages -``--with-coverage`` enable collecting coverage information -=================== =========== + python setup.py test + +in the root directory of the distribution. The script takes a set of +commands, such as: + +======================== =========== +``--pep8-only`` pep8 checks +``--omit-pep8`` Do not perform pep8 checks +``--nocapture`` do not capture stdout (nosetests) +``--nose-verbose`` be verbose (nosetests) +``--processes`` number of processes (nosetests) +``--process-timeout`` process timeout (nosetests) +``--with-coverage`` with coverage +``--detailed-error-msg`` detailed error message (nosetest) +``--tests`` comma separated selection of tests (nosetest) +======================== =========== Additionally it is possible to run only coding standard test or disable them: @@ -57,28 +67,34 @@ To run a single test from the command line, you can provide a dot-separated path to the module followed by the function separated by a colon, e.g., (this is assuming the test is installed):: - python tests.py matplotlib.tests.test_simplification:test_clipping - -If you want to run the full test suite, but want to save wall time try running the -tests in parallel:: + python setup.py test --tests=matplotlib.tests.test_simplification:test_clipping - python ../matplotlib/tests.py -sv --processes=5 --process-timeout=300 +If you want to run the full test suite, but want to save wall time try +running the tests in parallel:: -as we do on Travis.ci. + python setup.py test --nocapture --nose-verbose --processes=5 --process-timeout=300 An alternative implementation that does not look at command line -arguments works from within Python:: +arguments works from within Python is to run the tests from the +matplotlib library function :func:`matplotlib.test`:: import matplotlib matplotlib.test() +.. hint:: + + You might need to install nose for this:: + + pip install nose + + .. _`nosetest arguments`: http://nose.readthedocs.org/en/latest/usage.html -Running tests by any means other than `matplotlib.test()` -does not load the nose "knownfailureif" (Known failing tests) plugin, -causing known-failing tests to fail for real. +Running tests by any means other than `matplotlib.test()` does not +load the nose "knownfailureif" (Known failing tests) plugin, causing +known-failing tests to fail for real. Writing a simple test --------------------- diff --git a/setup.py b/setup.py index 409a90f60530..735239335efd 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,7 @@ # This needs to be the very first thing to use distribute from distribute_setup import use_setuptools use_setuptools() +from setuptools.command.test import test as TestCommand import sys @@ -121,6 +122,121 @@ 'Topic :: Scientific/Engineering :: Visualization', ] + +class NoseTestCommand(TestCommand): + """Invoke unit tests using nose after an in-place build.""" + + description = "Invoke unit tests using nose after an in-place build." + user_options = [ + ("pep8-only", None, "pep8 checks"), + ("omit-pep8", None, "Do not perform pep8 checks"), + ("nocapture", None, "do not capture stdout (nosetests)"), + ("nose-verbose", None, "be verbose (nosetests)"), + ("processes=", None, "number of processes (nosetests)"), + ("process-timeout=", None, "process timeout (nosetests)"), + ("with-coverage", None, "with coverage"), + ("detailed-error-msg", None, "detailed error message (nosetest)"), + ("tests=", None, "comma separated selection of tests (nosetest)"), + ] + + def initialize_options(self): + self.pep8_only = None + self.omit_pep8 = None + + # parameters passed to nose tests + self.processes = None + self.process_timeout = None + self.nose_verbose = None + self.nocapture = None + self.with_coverage = None + self.detailed_error_msg = None + self.tests = None + + def finalize_options(self): + self.test_args = [] + if self.pep8_only: + self.pep8_only = True + if self.omit_pep8: + self.omit_pep8 = True + + if self.pep8_only and self.omit_pep8: + from distutils.errors import DistutilsOptionError + raise DistutilsOptionError( + "You are using several options for the test command in an " + "incompatible manner. Please use either one of --pep8-only," + "--omit-pep8" + ) + + if self.processes: + self.test_args.append("--processes={prc}".format( + prc=self.processes)) + + if self.process_timeout: + self.test_args.append("--process-timeout={tout}".format( + tout=self.process_timeout)) + + if self.nose_verbose: + self.test_args.append("--verbose") + + if self.nocapture: + self.test_args.append("--nocapture") + + if self.with_coverage: + self.test_args.append("--with-coverage") + + if self.detailed_error_msg: + self.test_args.append("-d") + + if self.tests: + self.test_args.append("--tests={names}".format(names=self.tests)) + + + def run(self): + if self.distribution.install_requires: + self.distribution.fetch_build_eggs( + self.distribution.install_requires) + if self.distribution.tests_require: + self.distribution.fetch_build_eggs(self.distribution.tests_require) + + self.announce('running unittests with nose') + self.with_project_on_sys_path(self.run_tests) + + + def run_tests(self): + try: + import matplotlib + matplotlib.use('agg') + import nose + from matplotlib.testing.noseclasses import KnownFailure + from matplotlib import default_test_modules as testmodules + from matplotlib import font_manager + import time + # Make sure the font caches are created before starting any possibly + # parallel tests + if font_manager._fmcache is not None: + while not os.path.exists(font_manager._fmcache): + time.sleep(0.5) + plugins = [KnownFailure] + + # Nose doesn't automatically instantiate all of the plugins in the + # child processes, so we have to provide the multiprocess plugin + # with a list. + from nose.plugins import multiprocess + multiprocess._instantiate_plugins = plugins + + if self.omit_pep8: + testmodules.remove('matplotlib.tests.test_coding_standards') + elif self.pep8_only: + testmodules = ['matplotlib.tests.test_coding_standards'] + + nose.main(addplugins=[x() for x in plugins], + defaultTest=testmodules, + argv=['nosetests'] + self.test_args, + exit=False) + except ImportError: + sys.exit(-1) + + # One doesn't normally see `if __name__ == '__main__'` blocks in a setup.py, # however, this is needed on Windows to avoid creating infinite subprocesses # when using multiprocessing. @@ -135,6 +251,7 @@ package_dir = {'': 'lib'} install_requires = [] setup_requires = [] + tests_require = [] default_backend = None # Go through all of the packages and figure out which ones we are @@ -195,6 +312,7 @@ package_data[key] = list(set(val + package_data[key])) install_requires.extend(package.get_install_requires()) setup_requires.extend(package.get_setup_requires()) + tests_require.extend(package.get_tests_require()) # Write the default matplotlibrc file if default_backend is None: @@ -254,11 +372,13 @@ # List third-party Python packages that we require install_requires=install_requires, setup_requires=setup_requires, + tests_require=tests_require, # matplotlib has C/C++ extensions, so it's not zip safe. # Telling setuptools this prevents it from doing an automatic # check for zip safety. zip_safe=False, + cmdclass={'test': NoseTestCommand}, **extra_args ) diff --git a/setupext.py b/setupext.py index 80b2a6c4d783..b01f60c5d40e 100755 --- a/setupext.py +++ b/setupext.py @@ -415,6 +415,12 @@ def get_setup_requires(self): """ return [] + def get_tests_require(self): + """ + Get a list of Python packages that we require for executing tests. + """ + return [] + def _check_for_pkg_config(self, package, include_file, min_version=None, version=None): """ @@ -694,7 +700,7 @@ def get_package_data(self): 'sphinxext/tests/tinypages/_static/*', ]} - def get_install_requires(self): + def get_tests_require(self): requires = ['nose>=%s' % self.nose_min_version] if not sys.version_info >= (3, 3): requires += ['mock'] diff --git a/tox.ini b/tox.ini index b34e6ea7af13..296cefb56281 100644 --- a/tox.ini +++ b/tox.ini @@ -13,4 +13,5 @@ commands = {envpython} {toxinidir}/tests.py --processes=-1 --process-timeout=300 deps = nose + mock numpy