From 9cd3fbeda67639ecffbb5de70b858e51dc2f89e5 Mon Sep 17 00:00:00 2001 From: Antony Lee Date: Sun, 10 Jun 2018 11:01:36 +0200 Subject: [PATCH] Figure equality-based tests. Implement a `check_figures_equal` decorator, which allows tests where both the reference and the test image are generated. The idea is to allow tests of the form "this feature should be equivalent to that (usually more complex) way of achieving the same thing" without further bloating the test image directory. The saved images are properly created in the `result_images` folder, but cannot be "accepted" or "rejected" using the triage_tests UI (as there is indeed no reference image to be saved in the repo). Includes an example use case. --- lib/matplotlib/testing/decorators.py | 40 ++++++++++++++++++++++++++++ lib/matplotlib/tests/test_axes.py | 19 +++++-------- tools/triage_tests.py | 36 ++++++++++++++++--------- 3 files changed, 70 insertions(+), 25 deletions(-) diff --git a/lib/matplotlib/testing/decorators.py b/lib/matplotlib/testing/decorators.py index 97fcf23a4cb7..85b8d5e87d6b 100644 --- a/lib/matplotlib/testing/decorators.py +++ b/lib/matplotlib/testing/decorators.py @@ -407,6 +407,46 @@ def image_comparison(baseline_images, extensions=None, tol=0, savefig_kwargs=savefig_kwarg, style=style) +def check_figures_equal(*, extensions=("png", "pdf", "svg"), tol=0): + """ + Decorator for test cases that generate and compare two figures. + + The decorated function must take two arguments, *fig_test* and *fig_ref*, + and draw the test and reference images on them. After the function + returns, the figures are saved and compared. + + Arguments + --------- + extensions : list, default: ["png", "pdf", "svg"] + The extensions to test. + tol : float + The RMS threshold above which the test is considered failed. + """ + + def decorator(func): + import pytest + + _, result_dir = map(Path, _image_directories(func)) + + @pytest.mark.parametrize("ext", extensions) + def wrapper(ext): + fig_test = plt.figure("test") + fig_ref = plt.figure("reference") + func(fig_test, fig_ref) + test_image_path = str( + result_dir / (func.__name__ + "." + ext)) + ref_image_path = str( + result_dir / (func.__name__ + "-expected." + ext)) + fig_test.savefig(test_image_path) + fig_ref.savefig(ref_image_path) + _raise_on_image_difference( + ref_image_path, test_image_path, tol=tol) + + return wrapper + + return decorator + + def _image_directories(func): """ Compute the baseline and result image directories for testing *func*. diff --git a/lib/matplotlib/tests/test_axes.py b/lib/matplotlib/tests/test_axes.py index 83a3973f032c..fda0fe0f362b 100644 --- a/lib/matplotlib/tests/test_axes.py +++ b/lib/matplotlib/tests/test_axes.py @@ -14,7 +14,7 @@ import warnings import matplotlib -from matplotlib.testing.decorators import image_comparison +from matplotlib.testing.decorators import image_comparison, check_figures_equal import matplotlib.pyplot as plt import matplotlib.markers as mmarkers import matplotlib.patches as mpatches @@ -5702,18 +5702,11 @@ def test_plot_columns_cycle_deprecation(): plt.plot(np.zeros((2, 2)), np.zeros((2, 3))) -def test_markerfacecolor_none_alpha(): - fig1, ax1 = plt.subplots() - ax1.plot(0, "o", mfc="none", alpha=.5) - buf1 = io.BytesIO() - fig1.savefig(buf1) - - fig2, ax2 = plt.subplots() - ax2.plot(0, "o", mfc="w", alpha=.5) - buf2 = io.BytesIO() - fig2.savefig(buf2) - - assert buf1.getvalue() == buf2.getvalue() +# pdf and svg tests fail using travis' old versions of gs and inkscape. +@check_figures_equal(extensions=["png"]) +def test_markerfacecolor_none_alpha(fig_test, fig_ref): + fig_test.subplots().plot(0, "o", mfc="none", alpha=.5) + fig_ref.subplots().plot(0, "o", mfc="w", alpha=.5) def test_tick_padding_tightbbox(): diff --git a/tools/triage_tests.py b/tools/triage_tests.py index c8c60f8142b2..cac1bc38b660 100644 --- a/tools/triage_tests.py +++ b/tools/triage_tests.py @@ -192,14 +192,22 @@ def set_large_image(self, index): self.thumbnails[self.current_thumbnail].setFrameShape(1) def accept_test(self): - self.entries[self.current_entry].accept() + entry = self.entries[self.current_entry] + if entry.status == 'autogen': + print('Cannot accept autogenerated test cases.') + return + entry.accept() self.filelist.currentItem().setText( self.entries[self.current_entry].display) # Auto-move to the next entry self.set_entry(min((self.current_entry + 1), len(self.entries) - 1)) def reject_test(self): - self.entries[self.current_entry].reject() + entry = self.entries[self.current_entry] + if entry.status == 'autogen': + print('Cannot reject autogenerated test cases.') + return + entry.reject() self.filelist.currentItem().setText( self.entries[self.current_entry].display) # Auto-move to the next entry @@ -261,11 +269,14 @@ def __init__(self, path, root, source): ] self.thumbnails = [os.path.join(self.dir, x) for x in self.thumbnails] - self.status = 'unknown' - - if self.same(os.path.join(self.dir, self.generated), + if not Path(self.destdir, self.generated).exists(): + # This case arises from a check_figures_equal test. + self.status = 'autogen' + elif self.same(os.path.join(self.dir, self.generated), os.path.join(self.destdir, self.generated)): self.status = 'accept' + else: + self.status = 'unknown' def same(self, a, b): """ @@ -297,16 +308,18 @@ def display(self): Get the display string for this entry. This is the text that appears in the list widget. """ - status_map = {'unknown': '\N{BALLOT BOX}', - 'accept': '\N{BALLOT BOX WITH CHECK}', - 'reject': '\N{BALLOT BOX WITH X}'} + status_map = { + 'unknown': '\N{BALLOT BOX}', + 'accept': '\N{BALLOT BOX WITH CHECK}', + 'reject': '\N{BALLOT BOX WITH X}', + 'autogen': '\N{WHITE SQUARE CONTAINING BLACK SMALL SQUARE}', + } box = status_map[self.status] return '{} {} [{}]'.format(box, self.name, self.extension) def accept(self): """ - Accept this test by copying the generated result to the - source tree. + Accept this test by copying the generated result to the source tree. """ a = os.path.join(self.dir, self.generated) b = os.path.join(self.destdir, self.generated) @@ -315,8 +328,7 @@ def accept(self): def reject(self): """ - Reject this test by copying the expected result to the - source tree. + Reject this test by copying the expected result to the source tree. """ a = os.path.join(self.dir, self.expected) b = os.path.join(self.destdir, self.generated)