diff --git a/appveyor.yml b/appveyor.yml index cf186d552cb5..599fd5819541 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -139,7 +139,7 @@ test_script: - if x%USE_PYTEST% == xyes py.test %PYTEST_ARGS% - if x%USE_PYTEST% == xno python tests.py %PYTEST_ARGS% # Generate a html for visual tests - - python visual_tests.py + - python tools/visualize_tests.py --no-browser - pip install codecov - codecov -e PYTHON_VERSION PLATFORM @@ -180,7 +180,7 @@ artifacts: on_finish: on_failure: - - python visual_tests.py + - python tools/visualize_tests.py --no-browser - echo zipping images after a failure... - 7z a result_images.zip result_images\ |grep -v "Compressing" - appveyor PushArtifact result_images.zip diff --git a/pytest.ini b/pytest.ini index 805e38201e28..7ca06444fcbc 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -norecursedirs = .git build ci dist doc extern lib/mpl_examples release tools unit venv +norecursedirs = .git build ci dist doc extern lib/mpl_examples release unit venv python_files = test_*.py markers = @@ -15,7 +15,10 @@ pep8ignore = setupext.py E301 E302 E501 setup_external_compile.py E302 E501 E711 versioneer.py ALL # External file. - visual_tests.py E302 E501 + + tools/gh_api.py ALL # External file. + tools/github_stats.py ALL # External file. + tools/subset.py E221 E231 E251 E261 E302 E501 E701 E703 matplotlib/backends/qt_editor/formlayout.py E301 E402 E501 matplotlib/backends/backend_agg.py E225 E228 E231 E261 E301 E302 E303 E501 E701 diff --git a/tools/make_icons.py b/tools/make_icons.py index a0f664da16ce..d673d971d041 100755 --- a/tools/make_icons.py +++ b/tools/make_icons.py @@ -11,7 +11,7 @@ """ import matplotlib -matplotlib.use('agg') +matplotlib.use('agg') # noqa import six diff --git a/tools/test_triage.py b/tools/triage_tests.py similarity index 96% rename from tools/test_triage.py rename to tools/triage_tests.py index f378ef371ff8..139116d33d57 100644 --- a/tools/test_triage.py +++ b/tools/triage_tests.py @@ -11,7 +11,7 @@ If you ran the tests from the top-level of a source checkout, simply run: - python tools/test_triage.py + python tools/triage_tests.py Otherwise, you can manually select the location of `result_images` on the commandline. @@ -187,8 +187,8 @@ def set_entry(self, index): def set_large_image(self, index): self.thumbnails[self.current_thumbnail].setFrameShape(0) self.current_thumbnail = index - pixmap = QtGui.QPixmap( - self.entries[self.current_entry].thumbnails[self.current_thumbnail]) + pixmap = QtGui.QPixmap(self.entries[self.current_entry] + .thumbnails[self.current_thumbnail]) self.image_display.setPixmap(pixmap) self.thumbnails[self.current_thumbnail].setFrameShape(1) @@ -212,9 +212,9 @@ def keyPressEvent(self, e): elif e.key() == QtCore.Qt.Key_Right: self.set_large_image((self.current_thumbnail + 1) % 3) elif e.key() == QtCore.Qt.Key_Up: - self.set_entry(max((self.current_entry - 1), 0)) + self.set_entry(max(self.current_entry - 1, 0)) elif e.key() == QtCore.Qt.Key_Down: - self.set_entry(min((self.current_entry + 1), len(self.entries) - 1)) + self.set_entry(min(self.current_entry + 1, len(self.entries) - 1)) elif e.key() == QtCore.Qt.Key_A: self.accept_test() elif e.key() == QtCore.Qt.Key_R: @@ -249,7 +249,8 @@ def __init__(self, path, root, source): self.extension = extension self.generated = basename + '.' + extension self.expected = basename + '-expected.' + extension - self.expected_display = basename + '-expected' + display_extension + '.png' + self.expected_display = (basename + '-expected' + display_extension + + '.png') self.generated_display = basename + display_extension + '.png' self.name = os.path.join(self.reldir, self.basename) self.destdir = self.get_dest_dir(self.reldir) diff --git a/tools/visualize_tests.py b/tools/visualize_tests.py new file mode 100644 index 000000000000..539dd4db2eb4 --- /dev/null +++ b/tools/visualize_tests.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +# +# This builds a html page of all images from the image comparison tests +# and opens that page in the browser. +# +# $ python tools/visualize_tests.py +# + +import argparse +import os +from collections import defaultdict + + +html_template = """ + +{failed} +{body} + +""" + +subdir_template = """

{subdir}

+ +{rows} +
nameactualexpecteddiff
+""" + +failed_template = """

Only Failed

+ +{rows} +
nameactualexpecteddiff
+""" + +row_template = ('' + '{0}{1}' + '{2}' + '' + '{4}' + '') + +linked_image_template = '' + + +def run(show_browser=True): + """ + Build a website for visual comparison + """ + image_dir = "result_images" + _subdirs = (name + for name in os.listdir(image_dir) + if os.path.isdir(os.path.join(image_dir, name))) + + failed_rows = [] + body_sections = [] + for subdir in sorted(_subdirs): + if subdir == "test_compare_images": + # These are the images which test the image comparison functions. + continue + + pictures = defaultdict(dict) + for file in os.listdir(os.path.join(image_dir, subdir)): + if os.path.isdir(os.path.join(image_dir, subdir, file)): + continue + fn, fext = os.path.splitext(file) + if fext != ".png": + continue + # Always use / for URLs. + if "-failed-diff" in fn: + pictures[fn[:-12]]["f"] = "/".join((subdir, file)) + elif "-expected" in fn: + pictures[fn[:-9]]["e"] = "/".join((subdir, file)) + else: + pictures[fn]["c"] = "/".join((subdir, file)) + + subdir_rows = [] + for name, test in sorted(pictures.items()): + expected_image = test.get('e', '') + actual_image = test.get('c', '') + + if 'f' in test: + # A real failure in the image generation, resulting in + # different images. + status = " (failed)" + failed = 'diff'.format(test['f']) + current = linked_image_template.format(actual_image) + failed_rows.append(row_template.format(name, "", current, + expected_image, failed)) + elif 'c' not in test: + # A failure in the test, resulting in no current image + status = " (failed)" + failed = '--' + current = '(Failure in test, no image produced)' + failed_rows.append(row_template.format(name, "", current, + expected_image, failed)) + else: + status = " (passed)" + failed = '--' + current = linked_image_template.format(actual_image) + + subdir_rows.append(row_template.format(name, status, current, + expected_image, failed)) + + body_sections.append( + subdir_template.format(subdir=subdir, rows='\n'.join(subdir_rows))) + + if failed_rows: + failed = failed_template.format(rows='\n'.join(failed_rows)) + else: + failed = '' + body = ''.join(body_sections) + html = html_template.format(failed=failed, body=body) + index = os.path.join(image_dir, "index.html") + with open(index, "w") as f: + f.write(html) + + show_message = not show_browser + if show_browser: + try: + import webbrowser + webbrowser.open(index) + except: + show_message = True + + if show_message: + print("Open {} in a browser for a visual comparison.".format(index)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--no-browser', action='store_true', + help="Don't show browser after creating index page.") + args = parser.parse_args() + run(show_browser=not args.no_browser) diff --git a/visual_tests.py b/visual_tests.py deleted file mode 100644 index b2c37eb78906..000000000000 --- a/visual_tests.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python -# -# This builds a html page of all images from the image comparison tests -# and opens that page in the browser. -# -# $ python visual_tests.py -# - -import os -import time -import six - -from collections import defaultdict - -def run(): - # Build a website for visual comparison - image_dir = "result_images" - # build the website - _html = "" - _html += """ - \n""" - _subdirs = [name for name in os.listdir(image_dir) if os.path.isdir(os.path.join(image_dir, name))] - # loop over all pictures - _row = '{0} {1}{2}{4}\n' - _failed = "" - _failed += "

Only Failed

" - _failed += "\n\n" - _has_failure = False - _body = "" - for subdir in _subdirs: - if subdir == "test_compare_images": - # these are the image which test the image comparison functions... - continue - pictures = defaultdict(dict) - for file in os.listdir(os.path.join(image_dir, subdir)): - if os.path.isdir(os.path.join(image_dir, subdir, file)): - continue - fn, fext = os.path.splitext(file) - if fext != ".png": - continue - # Always use / for URLs. - if "-failed-diff" in fn: - pictures[fn[:-12]]["f"] = "/".join((subdir, file)) - elif "-expected" in fn: - pictures[fn[:-9]]["e"] = "/".join((subdir, file)) - else: - pictures[fn]["c"] = "/".join((subdir, file)) - - _body += "

{0}

".format(subdir) - _body += "
nameactualexpecteddiff
\n\n" - for name, test in six.iteritems(pictures): - if test.get("f", None): - # a real failure in the image generation, resulting in different images - _has_failure = True - s = "(failed)" - failed = 'diff'.format(test.get("f", "")) - current = ''.format(test.get("c", "")) - _failed += _row.format(name, "", current, test.get("e", ""), failed) - elif test.get("c", None) is None: - # A failure in the test, resulting in no current image - _has_failure = True - s = "(failed)" - failed = '--' - current = '(Failure in test, no image produced)' - _failed += _row.format(name, "", current, test.get("e", ""), failed) - else: - s = "(passed)" - failed = '--' - current = ''.format(test.get("c", "")) - _body += _row.format(name, "", current, test.get("e", ""), failed) - _body += "
nameactualexpecteddiff
\n" - _failed += "\n" - if _has_failure: - _html += _failed - _html += _body - _html += "\n" - index = os.path.join(image_dir, "index.html") - with open(index, "w") as f: - f.write(_html) - try: - import webbrowser - webbrowser.open(index) - except: - print("Open {} in a browser for a visual comparison.".format(index)) - -if __name__ == '__main__': - run()