From 50f8c599dce7c9f547f5999690ba401e724ba8a0 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 2 Feb 2017 23:11:57 -0500 Subject: [PATCH 1/8] Template-ize visual_tests.py. --- visual_tests.py | 85 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 59 insertions(+), 26 deletions(-) diff --git a/visual_tests.py b/visual_tests.py index b2c37eb78906..d83fafcc5b65 100644 --- a/visual_tests.py +++ b/visual_tests.py @@ -12,26 +12,50 @@ from collections import defaultdict + +html_template = """ + +{failed} +{body} + +""" + +subdir_template = """

{subdir}

+ +{rows} +

name	actual	expected	diff

+""" + +failed_template = """

Only Failed

+ +{rows} +

name	actual	expected	diff

+""" + +row_template = ('' + '{0} {1}' + '{2}' + '

' + '{4}' + '') + +linked_image_template = '

' + + def run(): # Build a website for visual comparison image_dir = "result_images" # build the website - _html = "" - _html += """ - \n""" _subdirs = [name for name in os.listdir(image_dir) if os.path.isdir(os.path.join(image_dir, name))] # loop over all pictures - _row = '{0} {1}{2}

{4}\n' - _failed = "" - _failed += "

Only Failed

" - _failed += "\n\n" _has_failure = False - _body = "" + failed_rows = [] + body_sections = [] for subdir in _subdirs: if subdir == "test_compare_images": # these are the image which test the image comparison functions... @@ -51,37 +75,46 @@ def run(): else: pictures[fn]["c"] = "/".join((subdir, file)) - _body += "

{0}

".format(subdir) - _body += "

name	actual	expected	diff

\n\n" + subdir_rows = [] for name, test in six.iteritems(pictures): if test.get("f", None): # a real failure in the image generation, resulting in different images _has_failure = True s = "(failed)" failed = 'diff'.format(test.get("f", "")) - current = '

'.format(test.get("c", "")) - _failed += _row.format(name, "", current, test.get("e", ""), failed) + current = linked_image_template.format(test.get("c", "")) + failed_rows.append(row_template.format(name, "", current, + test.get("e", ""), + failed)) elif test.get("c", None) is None: # A failure in the test, resulting in no current image _has_failure = True s = "(failed)" failed = '--' current = '(Failure in test, no image produced)' - _failed += _row.format(name, "", current, test.get("e", ""), failed) + failed_rows.append(row_template.format(name, "", current, + test.get("e", ""), + failed)) else: s = "(passed)" failed = '--' - current = '

'.format(test.get("c", "")) - _body += _row.format(name, "", current, test.get("e", ""), failed) - _body += "

name	actual	expected	diff

\n" - _failed += "\n" + current = linked_image_template.format(test.get("c", "")) + subdir_rows.append(row_template.format(name, "", current, + test.get("e", ""), failed)) + + body_sections.append( + subdir_template.format(subdir=subdir, rows='\n'.join(subdir_rows))) + if _has_failure: - _html += _failed - _html += _body - _html += "\n" + failed = failed_template.format(rows='\n'.join(failed_rows)) + else: + failed = '' + body = ''.join(body_sections) + html = html_template.format(failed=failed, body=body) index = os.path.join(image_dir, "index.html") with open(index, "w") as f: - f.write(_html) + f.write(html) + try: import webbrowser webbrowser.open(index) From 20b018a0e692dbcd50a6077392f4d37c8360f2d7 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 2 Feb 2017 23:20:53 -0500 Subject: [PATCH 2/8] Pass missing variable to format string. --- visual_tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/visual_tests.py b/visual_tests.py index d83fafcc5b65..da5864a3f71c 100644 --- a/visual_tests.py +++ b/visual_tests.py @@ -38,7 +38,7 @@ """ row_template = ('' - '{0} {1}' + '{0}{1}' '{2}' '

' '{4}' @@ -80,7 +80,7 @@ def run(): if test.get("f", None): # a real failure in the image generation, resulting in different images _has_failure = True - s = "(failed)" + status = " (failed)" failed = 'diff'.format(test.get("f", "")) current = linked_image_template.format(test.get("c", "")) failed_rows.append(row_template.format(name, "", current, @@ -89,17 +89,17 @@ def run(): elif test.get("c", None) is None: # A failure in the test, resulting in no current image _has_failure = True - s = "(failed)" + status = " (failed)" failed = '--' current = '(Failure in test, no image produced)' failed_rows.append(row_template.format(name, "", current, test.get("e", ""), failed)) else: - s = "(passed)" + status = " (passed)" failed = '--' current = linked_image_template.format(test.get("c", "")) - subdir_rows.append(row_template.format(name, "", current, + subdir_rows.append(row_template.format(name, status, current, test.get("e", ""), failed)) body_sections.append( From 389156683dc008e396d13ead8a4c524fc4c42dfb Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Fri, 3 Feb 2017 00:05:35 -0500 Subject: [PATCH 3/8] Simplify dictionary access in visual_tests.py. --- visual_tests.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/visual_tests.py b/visual_tests.py index da5864a3f71c..a7969f742c57 100644 --- a/visual_tests.py +++ b/visual_tests.py @@ -77,30 +77,31 @@ def run(): subdir_rows = [] for name, test in six.iteritems(pictures): - if test.get("f", None): + expected_image = test.get('e', '') + actual_image = test.get('c', '') + + if 'f' in test: # a real failure in the image generation, resulting in different images _has_failure = True status = " (failed)" - failed = 'diff'.format(test.get("f", "")) - current = linked_image_template.format(test.get("c", "")) + failed = 'diff'.format(test['f']) + current = linked_image_template.format(actual_image) failed_rows.append(row_template.format(name, "", current, - test.get("e", ""), - failed)) - elif test.get("c", None) is None: + expected_image, failed)) + elif 'c' not in test: # A failure in the test, resulting in no current image _has_failure = True status = " (failed)" failed = '--' current = '(Failure in test, no image produced)' failed_rows.append(row_template.format(name, "", current, - test.get("e", ""), - failed)) + expected_image, failed)) else: status = " (passed)" failed = '--' - current = linked_image_template.format(test.get("c", "")) + current = linked_image_template.format(actual_image) subdir_rows.append(row_template.format(name, status, current, - test.get("e", ""), failed)) + expected_image, failed)) body_sections.append( subdir_template.format(subdir=subdir, rows='\n'.join(subdir_rows))) From 7cd4cc95b6dba230b67c242389d5249621f25a13 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Fri, 3 Feb 2017 00:14:39 -0500 Subject: [PATCH 4/8] PEP8 visual_tests.py. --- pytest.ini | 1 - visual_tests.py | 25 ++++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pytest.ini b/pytest.ini index 805e38201e28..9c8b4bad38c2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -15,7 +15,6 @@ pep8ignore = setupext.py E301 E302 E501 setup_external_compile.py E302 E501 E711 versioneer.py ALL # External file. - visual_tests.py E302 E501 matplotlib/backends/qt_editor/formlayout.py E301 E402 E501 matplotlib/backends/backend_agg.py E225 E228 E231 E261 E301 E302 E303 E501 E701 diff --git a/visual_tests.py b/visual_tests.py index a7969f742c57..7726210f18c6 100644 --- a/visual_tests.py +++ b/visual_tests.py @@ -7,7 +7,6 @@ # import os -import time import six from collections import defaultdict @@ -48,18 +47,21 @@ def run(): - # Build a website for visual comparison + """ + Build a website for visual comparison + """ image_dir = "result_images" - # build the website - _subdirs = [name for name in os.listdir(image_dir) if os.path.isdir(os.path.join(image_dir, name))] - # loop over all pictures - _has_failure = False + _subdirs = (name + for name in os.listdir(image_dir) + if os.path.isdir(os.path.join(image_dir, name))) + failed_rows = [] body_sections = [] for subdir in _subdirs: if subdir == "test_compare_images": - # these are the image which test the image comparison functions... + # These are the images which test the image comparison functions. continue + pictures = defaultdict(dict) for file in os.listdir(os.path.join(image_dir, subdir)): if os.path.isdir(os.path.join(image_dir, subdir, file)): @@ -81,8 +83,8 @@ def run(): actual_image = test.get('c', '') if 'f' in test: - # a real failure in the image generation, resulting in different images - _has_failure = True + # A real failure in the image generation, resulting in + # different images. status = " (failed)" failed = 'diff'.format(test['f']) current = linked_image_template.format(actual_image) @@ -90,7 +92,6 @@ def run(): expected_image, failed)) elif 'c' not in test: # A failure in the test, resulting in no current image - _has_failure = True status = " (failed)" failed = '--' current = '(Failure in test, no image produced)' @@ -100,13 +101,14 @@ def run(): status = " (passed)" failed = '--' current = linked_image_template.format(actual_image) + subdir_rows.append(row_template.format(name, status, current, expected_image, failed)) body_sections.append( subdir_template.format(subdir=subdir, rows='\n'.join(subdir_rows))) - if _has_failure: + if failed_rows: failed = failed_template.format(rows='\n'.join(failed_rows)) else: failed = '' @@ -122,5 +124,6 @@ def run(): except: print("Open {} in a browser for a visual comparison.".format(index)) + if __name__ == '__main__': run() From ccca1f3efa1e702604cf188c484f14d94f8c90b2 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Fri, 3 Feb 2017 00:18:04 -0500 Subject: [PATCH 5/8] Sort results in visual test HTML. This makes the result a bit more reproducable and comparable. --- visual_tests.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/visual_tests.py b/visual_tests.py index 7726210f18c6..74b8313a6a7d 100644 --- a/visual_tests.py +++ b/visual_tests.py @@ -7,8 +7,6 @@ # import os -import six - from collections import defaultdict @@ -57,7 +55,7 @@ def run(): failed_rows = [] body_sections = [] - for subdir in _subdirs: + for subdir in sorted(_subdirs): if subdir == "test_compare_images": # These are the images which test the image comparison functions. continue @@ -78,7 +76,7 @@ def run(): pictures[fn]["c"] = "/".join((subdir, file)) subdir_rows = [] - for name, test in six.iteritems(pictures): + for name, test in sorted(pictures.items()): expected_image = test.get('e', '') actual_image = test.get('c', '') From a89d7d2178bd943aea0bea14c60d667e5ca1085a Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Fri, 3 Feb 2017 00:25:12 -0500 Subject: [PATCH 6/8] Don't start a browser for visual tests on CI. Travis may or may not have a browser installed, but AppVeyor definitely does, so this saves a tiny bit of processing power per run. --- appveyor.yml | 4 ++-- visual_tests.py | 22 ++++++++++++++++------ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index cf186d552cb5..14dadf9c1631 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -139,7 +139,7 @@ test_script: - if x%USE_PYTEST% == xyes py.test %PYTEST_ARGS% - if x%USE_PYTEST% == xno python tests.py %PYTEST_ARGS% # Generate a html for visual tests - - python visual_tests.py + - python visual_tests.py --no-browser - pip install codecov - codecov -e PYTHON_VERSION PLATFORM @@ -180,7 +180,7 @@ artifacts: on_finish: on_failure: - - python visual_tests.py + - python visual_tests.py --no-browser - echo zipping images after a failure... - 7z a result_images.zip result_images\ |grep -v "Compressing" - appveyor PushArtifact result_images.zip diff --git a/visual_tests.py b/visual_tests.py index 74b8313a6a7d..015abc4ed3d3 100644 --- a/visual_tests.py +++ b/visual_tests.py @@ -6,6 +6,7 @@ # $ python visual_tests.py # +import argparse import os from collections import defaultdict @@ -44,7 +45,7 @@ linked_image_template = '

' -def run(): +def run(show_browser=True): """ Build a website for visual comparison """ @@ -116,12 +117,21 @@ def run(): with open(index, "w") as f: f.write(html) - try: - import webbrowser - webbrowser.open(index) - except: + show_message = not show_browser + if show_browser: + try: + import webbrowser + webbrowser.open(index) + except: + show_message = True + + if show_message: print("Open {} in a browser for a visual comparison.".format(index)) if __name__ == '__main__': - run() + parser = argparse.ArgumentParser() + parser.add_argument('--no-browser', action='store_true', + help="Don't show browser after creating index page.") + args = parser.parse_args() + run(show_browser=not args.no_browser) From ef20a3bd9a94f17f8a0f5179300e815eeb2bc9a3 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Mon, 6 Feb 2017 17:12:48 -0500 Subject: [PATCH 7/8] Move visual_tests into tools. --- appveyor.yml | 4 ++-- tools/{test_triage.py => triage_tests.py} | 2 +- visual_tests.py => tools/visualize_tests.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename tools/{test_triage.py => triage_tests.py} (99%) rename visual_tests.py => tools/visualize_tests.py (99%) diff --git a/appveyor.yml b/appveyor.yml index 14dadf9c1631..599fd5819541 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -139,7 +139,7 @@ test_script: - if x%USE_PYTEST% == xyes py.test %PYTEST_ARGS% - if x%USE_PYTEST% == xno python tests.py %PYTEST_ARGS% # Generate a html for visual tests - - python visual_tests.py --no-browser + - python tools/visualize_tests.py --no-browser - pip install codecov - codecov -e PYTHON_VERSION PLATFORM @@ -180,7 +180,7 @@ artifacts: on_finish: on_failure: - - python visual_tests.py --no-browser + - python tools/visualize_tests.py --no-browser - echo zipping images after a failure... - 7z a result_images.zip result_images\ |grep -v "Compressing" - appveyor PushArtifact result_images.zip diff --git a/tools/test_triage.py b/tools/triage_tests.py similarity index 99% rename from tools/test_triage.py rename to tools/triage_tests.py index f378ef371ff8..d591f6c34b20 100644 --- a/tools/test_triage.py +++ b/tools/triage_tests.py @@ -11,7 +11,7 @@ If you ran the tests from the top-level of a source checkout, simply run: - python tools/test_triage.py + python tools/triage_tests.py Otherwise, you can manually select the location of `result_images` on the commandline. diff --git a/visual_tests.py b/tools/visualize_tests.py similarity index 99% rename from visual_tests.py rename to tools/visualize_tests.py index 015abc4ed3d3..539dd4db2eb4 100644 --- a/visual_tests.py +++ b/tools/visualize_tests.py @@ -3,7 +3,7 @@ # This builds a html page of all images from the image comparison tests # and opens that page in the browser. # -# $ python visual_tests.py +# $ python tools/visualize_tests.py # import argparse From a4dcd547751f8180a8e42130f2227eca5a41e295 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Mon, 6 Feb 2017 17:16:38 -0500 Subject: [PATCH 8/8] Add tools directory to pep8. --- pytest.ini | 6 +++++- tools/make_icons.py | 2 +- tools/triage_tests.py | 11 ++++++----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pytest.ini b/pytest.ini index 9c8b4bad38c2..7ca06444fcbc 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -norecursedirs = .git build ci dist doc extern lib/mpl_examples release tools unit venv +norecursedirs = .git build ci dist doc extern lib/mpl_examples release unit venv python_files = test_*.py markers = @@ -16,6 +16,10 @@ pep8ignore = setup_external_compile.py E302 E501 E711 versioneer.py ALL # External file. + tools/gh_api.py ALL # External file. + tools/github_stats.py ALL # External file. + tools/subset.py E221 E231 E251 E261 E302 E501 E701 E703 + matplotlib/backends/qt_editor/formlayout.py E301 E402 E501 matplotlib/backends/backend_agg.py E225 E228 E231 E261 E301 E302 E303 E501 E701 matplotlib/backends/backend_cairo.py E203 E211 E221 E231 E261 E272 E302 E303 E401 E402 E501 E701 E711 diff --git a/tools/make_icons.py b/tools/make_icons.py index a0f664da16ce..d673d971d041 100755 --- a/tools/make_icons.py +++ b/tools/make_icons.py @@ -11,7 +11,7 @@ """ import matplotlib -matplotlib.use('agg') +matplotlib.use('agg') # noqa import six diff --git a/tools/triage_tests.py b/tools/triage_tests.py index d591f6c34b20..139116d33d57 100644 --- a/tools/triage_tests.py +++ b/tools/triage_tests.py @@ -187,8 +187,8 @@ def set_entry(self, index): def set_large_image(self, index): self.thumbnails[self.current_thumbnail].setFrameShape(0) self.current_thumbnail = index - pixmap = QtGui.QPixmap( - self.entries[self.current_entry].thumbnails[self.current_thumbnail]) + pixmap = QtGui.QPixmap(self.entries[self.current_entry] + .thumbnails[self.current_thumbnail]) self.image_display.setPixmap(pixmap) self.thumbnails[self.current_thumbnail].setFrameShape(1) @@ -212,9 +212,9 @@ def keyPressEvent(self, e): elif e.key() == QtCore.Qt.Key_Right: self.set_large_image((self.current_thumbnail + 1) % 3) elif e.key() == QtCore.Qt.Key_Up: - self.set_entry(max((self.current_entry - 1), 0)) + self.set_entry(max(self.current_entry - 1, 0)) elif e.key() == QtCore.Qt.Key_Down: - self.set_entry(min((self.current_entry + 1), len(self.entries) - 1)) + self.set_entry(min(self.current_entry + 1, len(self.entries) - 1)) elif e.key() == QtCore.Qt.Key_A: self.accept_test() elif e.key() == QtCore.Qt.Key_R: @@ -249,7 +249,8 @@ def __init__(self, path, root, source): self.extension = extension self.generated = basename + '.' + extension self.expected = basename + '-expected.' + extension - self.expected_display = basename + '-expected' + display_extension + '.png' + self.expected_display = (basename + '-expected' + display_extension + + '.png') self.generated_display = basename + display_extension + '.png' self.name = os.path.join(self.reldir, self.basename) self.destdir = self.get_dest_dir(self.reldir)