From b11b65f1a549f8eeb27798f6c493d4a2fd0070fd Mon Sep 17 00:00:00 2001
From: Thomas A Caswell <tcaswell@gmail.com>
Date: Tue, 14 Jan 2014 16:22:36 -0600
Subject: [PATCH] ENH : added test on number of large differences between
 images

Added a test in compare_images to count the number of pixel which
are different by some threshold.  If the number of pixels which
have a difference which exceeds  this threshold is greater than
some maximum, the test fails.

This is to address the observation that some of the image tests
lost half of their axes labels, but were still passing the RMS
test.
---
 lib/matplotlib/testing/compare.py    | 25 +++++++++++++++++++++----
 lib/matplotlib/testing/decorators.py | 13 +++++++++----
 2 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/lib/matplotlib/testing/compare.py b/lib/matplotlib/testing/compare.py
index 5769cb7123c2..3c2c1c7209ec 100644
--- a/lib/matplotlib/testing/compare.py
+++ b/lib/matplotlib/testing/compare.py
@@ -260,7 +260,16 @@ def calculate_rms(expectedImage, actualImage):
     return rms
 
 
-def compare_images(expected, actual, tol, in_decorator=False):
+def count_large_differences(expectedImage, actualImage, threshold):
+    """
+    Returns the number of pixel which differ by greater than `threshold`
+    """
+    abs_diff_image = abs(expectedImage - actualImage)
+    return np.sum(abs_diff_image > threshold)
+
+
+def compare_images(expected, actual, tol, in_decorator=False,
+                   excursion_threshold=None, max_excursion_count=None):
     """
     Compare two "image" files checking differences within a tolerance.
 
@@ -324,17 +333,25 @@ def compare_images(expected, actual, tol, in_decorator=False):
     actualImage = actualImage.astype(np.int16)
 
     rms = calculate_rms(expectedImage, actualImage)
+    excursion_pass = True  # default to true for back
+    results = {}
+    if excursion_threshold is not None and max_excursion_count is not None:
+        excursion_count = count_large_differences(expectedImage,
+                                                  actualImage,
+                                                  excursion_threshold)
+        excursion_pass = excursion_count < excursion_threshold
+        results['excursion_count'] = excursion_count
 
     diff_image = make_test_filename(actual, 'failed-diff')
-
-    if rms <= tol:
+    if rms <= tol and excursion_pass:
         if os.path.exists(diff_image):
             os.unlink(diff_image)
+
         return None
 
     save_diff_image(expected, actual, diff_image)
 
-    results = dict(rms=rms, expected=str(expected),
+    results.update(rms=rms, expected=str(expected),
                    actual=str(actual), diff=str(diff_image))
 
     if not in_decorator:
diff --git a/lib/matplotlib/testing/decorators.py b/lib/matplotlib/testing/decorators.py
index 655fba1b9987..be92d2078a9b 100644
--- a/lib/matplotlib/testing/decorators.py
+++ b/lib/matplotlib/testing/decorators.py
@@ -176,7 +176,9 @@ def do_test():
                     figure.savefig(actual_fname, **self._savefig_kwarg)
 
                     err = compare_images(expected_fname, actual_fname,
-                                         self._tol, in_decorator=True)
+                                self._tol, in_decorator=True,
+                                excursion_threshold=self._excursion_threshold,
+                                max_excursion_count=self._max_excursion_count)
 
                     try:
                         if not os.path.exists(expected_fname):
@@ -186,7 +188,7 @@ def do_test():
                         if err:
                             raise ImageComparisonFailure(
                                 'images not close: %(actual)s vs. %(expected)s '
-                                '(RMS %(rms).3f)'%err)
+                                '(RMS %(rms).3f) (excursions %(excursion_count)d)'%err)
                     except ImageComparisonFailure:
                         if not check_freetype_version(self._freetype_version):
                             raise KnownFailureTest(
@@ -198,7 +200,8 @@ def do_test():
 
 def image_comparison(baseline_images=None, extensions=None, tol=13,
                      freetype_version=None, remove_text=False,
-                     savefig_kwarg=None):
+                     savefig_kwarg=None, excursion_threshold=128,
+                     max_excursion_count=100):
     """
     call signature::
 
@@ -272,7 +275,9 @@ def compare_images_decorator(func):
              '_tol': tol,
              '_freetype_version': freetype_version,
              '_remove_text': remove_text,
-             '_savefig_kwarg': savefig_kwarg})
+             '_savefig_kwarg': savefig_kwarg,
+             '_max_excursion_count': max_excursion_count,
+             '_excursion_threshold': excursion_threshold})
 
         return new_class
     return compare_images_decorator