Skip to content

Faster image comparison decorator #97

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 27, 2011
Merged

Faster image comparison decorator #97

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
27d4c0c
Updates to the image comparison testing decorator:
mdboom Apr 26, 2011
354dee8
Fix memory leak in pyplot.close('all')
mdboom Apr 27, 2011
a074823
Run inkscape for SVG conversion as a server -- prevents starting inks…
mdboom Apr 27, 2011
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion lib/matplotlib/_pylab_helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,12 @@ def destroy_fig(fig):
@staticmethod
def destroy_all():
for manager in Gcf.figs.values():
Gcf.destroy(manager.num)
manager.canvas.mpl_disconnect(manager._cidgcf)
manager.destroy()

Gcf._activeQue = []
Gcf.figs.clear()
gc.collect()

@staticmethod
def has_fignum(num):
Expand Down
61 changes: 45 additions & 16 deletions lib/matplotlib/testing/compare.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,22 @@ def compare_float( expected, actual, relTol = None, absTol = None ):
# convert files with that extension to png format.
converter = { }

def make_external_conversion_command(cmd):
def convert(*args):
cmdline = cmd(*args)
oldname, newname = args
pipe = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = pipe.communicate()
errcode = pipe.wait()
if not os.path.exists(newname) or errcode:
msg = "Conversion command failed:\n%s\n" % ' '.join(cmd)
if stdout:
msg += "Standard output:\n%s\n" % stdout
if stderr:
msg += "Standard error:\n%s\n" % stderr
raise IOError, msg
return convert

if matplotlib.checkdep_ghostscript() is not None:
# FIXME: make checkdep_ghostscript return the command
if sys.platform == 'win32':
Expand All @@ -92,13 +108,36 @@ def compare_float( expected, actual, relTol = None, absTol = None ):
cmd = lambda old, new: \
[gs, '-q', '-sDEVICE=png16m', '-dNOPAUSE', '-dBATCH',
'-sOutputFile=' + new, old]
converter['pdf'] = cmd
converter['eps'] = cmd
converter['pdf'] = make_external_conversion_command(cmd)
converter['eps'] = make_external_conversion_command(cmd)

if matplotlib.checkdep_inkscape() is not None:
cmd = lambda old, new: \
['inkscape', old, '--export-png=' + new]
converter['svg'] = cmd
def make_svg_converter():
def read_to_end(buf):
ret = ''
lastchar = ''
while True:
char = buf.readline(1)
if char == '>' and lastchar == '\n':
break
ret += char
lastchar = char
return ret

p = subprocess.Popen(['inkscape', '--shell'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
read_to_end(p.stdout)

def convert_svg(old, new):
p.stdin.write('%s --export-png=%s\n' % (old, new))
p.stdin.flush()
read_to_end(p.stdout)

return convert_svg

converter['svg'] = make_svg_converter()

def comparable_formats():
'''Returns the list of file formats that compare_images can compare
Expand All @@ -116,17 +155,7 @@ def convert(filename):
newname = base + '_' + extension + '.png'
if not os.path.exists(filename):
raise IOError, "'%s' does not exist" % filename
cmd = converter[extension](filename, newname)
pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = pipe.communicate()
errcode = pipe.wait()
if not os.path.exists(newname) or errcode:
msg = "Conversion command failed:\n%s\n" % ' '.join(cmd)
if stdout:
msg += "Standard output:\n%s\n" % stdout
if stderr:
msg += "Standard error:\n%s\n" % stderr
raise IOError, msg
converter[extension](filename, newname)
return newname

verifiers = { }
Expand Down
156 changes: 100 additions & 56 deletions lib/matplotlib/testing/decorators.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
from matplotlib.testing.noseclasses import KnownFailureTest, \
KnownFailureDidNotFailTest, ImageComparisonFailure
import os, sys, shutil
import os, sys, shutil, new
import nose
import matplotlib
import matplotlib.tests
import matplotlib.units
from matplotlib import pyplot as plt
import numpy as np
from matplotlib.testing.compare import comparable_formats, compare_images

Expand Down Expand Up @@ -46,7 +48,83 @@ def failer(*args, **kwargs):
return nose.tools.make_decorator(f)(failer)
return known_fail_decorator

def image_comparison(baseline_images=None,extensions=None,tol=1e-3):
class CleanupTest:
@classmethod
def setup_class(cls):
cls.original_rcParams = {}
cls.original_rcParams.update(matplotlib.rcParams)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shouldn't this be rcParamsDefault - since test may be launched after the user starts up e.g. ipython with a profile which modifies the rcParams - making test incompatible?

also, is there a reason to not just do it in one line, instead of two?
cls.original_rcParams = matplotlib.rcParamsDefault.copy()

same for the orignal_units_registry below

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good points on both. I think I was confused by the restoration phase -- which does need the .clear and .update because we can't replace the underlying rcParams object which is referenced all over the matplotlib source code.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah -- on your first point: it turns out we can't use rcParamsDefault since the test infrastructure overrides certain things in a setup() function. See lib/matplotlib/tests/init.py.

def setup():
    use('Agg', warn=False) # use Agg backend for these tests

    # These settings *must* be hardcoded for running the comparison
    # tests and are not necessarily the default values as specified in
    # rcsetup.py
    rcdefaults() # Start with all defaults
    rcParams['font.family'] = 'Bitstream Vera Sans'
    rcParams['text.hinting'] = False

Maybe we should call this function from the teardown part of the decorator? Then everything gets reset to this.


cls.original_units_registry = {}
cls.original_units_registry.update(matplotlib.units.registry)

@classmethod
def teardown_class(cls):
plt.close('all')

matplotlib.rcParams.clear()
matplotlib.rcParams.update(cls.original_rcParams)

matplotlib.units.registry.clear()
matplotlib.units.registry.update(cls.original_units_registry)

def test(self):
self.func()

def cleanup(func):
new_class = new.classobj(
func.__name__,
(CleanupTest,),
{'func': staticmethod(func)})
return new_class

class ImageComparisonTest(CleanupTest):
@classmethod
def setup_class(cls):
CleanupTest.setup_class()

cls.func()

def test(self):
baseline_dir, result_dir = _image_directories(self.func)

for fignum, baseline in zip(plt.get_fignums(), self.baseline_images):
figure = plt.figure(fignum)

for extension in self.extensions:
will_fail = not extension in comparable_formats()
if will_fail:
fail_msg = 'Cannot compare %s files on this system' % extension
else:
fail_msg = 'No failure expected'

orig_expected_fname = os.path.join(baseline_dir, baseline) + '.' + extension
expected_fname = os.path.join(result_dir, 'expected-' + baseline) + '.' + extension
actual_fname = os.path.join(result_dir, baseline) + '.' + extension
if os.path.exists(orig_expected_fname):
shutil.copyfile(orig_expected_fname, expected_fname)
else:
will_fail = True
fail_msg = 'Do not have baseline image %s' % expected_fname

@knownfailureif(
will_fail, fail_msg,
known_exception_class=ImageComparisonFailure)
def do_test():
figure.savefig(actual_fname)

if not os.path.exists(expected_fname):
raise ImageComparisonFailure(
'image does not exist: %s' % expected_fname)

err = compare_images(expected_fname, actual_fname, self.tol, in_decorator=True)
if err:
raise ImageComparisonFailure(
'images not close: %(actual)s vs. %(expected)s '
'(RMS %(rms).3f)'%err)

yield (do_test,)

def image_comparison(baseline_images=None, extensions=None, tol=1e-3):
"""
call signature::

Expand Down Expand Up @@ -76,56 +154,22 @@ def image_comparison(baseline_images=None,extensions=None,tol=1e-3):
# default extensions to test
extensions = ['png', 'pdf', 'svg']

# The multiple layers of defs are required because of how
# parameterized decorators work, and because we want to turn the
# single test_foo function to a generator that generates a
# separate test case for each file format.
def compare_images_decorator(func):
baseline_dir, result_dir = _image_directories(func)

def compare_images_generator():
for extension in extensions:
orig_expected_fnames = [os.path.join(baseline_dir,fname) + '.' + extension for fname in baseline_images]
expected_fnames = [os.path.join(result_dir,'expected-'+fname) + '.' + extension for fname in baseline_images]
actual_fnames = [os.path.join(result_dir, fname) + '.' + extension for fname in baseline_images]
have_baseline_images = [os.path.exists(expected) for expected in orig_expected_fnames]
have_baseline_image = np.all(have_baseline_images)
is_comparable = extension in comparable_formats()
if not is_comparable:
fail_msg = 'Cannot compare %s files on this system' % extension
elif not have_baseline_image:
fail_msg = 'Do not have baseline images %s' % expected_fnames
else:
fail_msg = 'No failure expected'
will_fail = not (is_comparable and have_baseline_image)
@knownfailureif(will_fail, fail_msg,
known_exception_class=ImageComparisonFailure )
def decorated_compare_images():
# set the default format of savefig
matplotlib.rc('savefig', extension=extension)
# change to the result directory for the duration of the test
old_dir = os.getcwd()
os.chdir(result_dir)
try:
result = func() # actually call the test function
finally:
os.chdir(old_dir)
for original, expected in zip(orig_expected_fnames, expected_fnames):
if not os.path.exists(original):
raise ImageComparisonFailure(
'image does not exist: %s'%original)
shutil.copyfile(original, expected)
for actual,expected in zip(actual_fnames,expected_fnames):
# compare the images
err = compare_images( expected, actual, tol,
in_decorator=True )
if err:
raise ImageComparisonFailure(
'images not close: %(actual)s vs. %(expected)s '
'(RMS %(rms).3f)'%err)
return result
yield (decorated_compare_images,)
return nose.tools.make_decorator(func)(compare_images_generator)
# We want to run the setup function (the actual test function
# that generates the figure objects) only once for each type
# of output file. The only way to achieve this with nose
# appears to be to create a test class with "setup_class" and
# "teardown_class" methods. Creating a class instance doesn't
# work, so we use new.classobj to actually create a class and
# fill it with the appropriate methods.
new_class = new.classobj(
func.__name__,
(ImageComparisonTest,),
{'func': staticmethod(func),
'baseline_images': baseline_images,
'extensions': extensions,
'tol': tol})
return new_class
return compare_images_decorator

def _image_directories(func):
Expand All @@ -134,7 +178,7 @@ def _image_directories(func):
Create the result directory if it doesn't exist.
"""
module_name = func.__module__
if module_name=='__main__':
if module_name == '__main__':
# FIXME: this won't work for nested packages in matplotlib.tests
import warnings
warnings.warn('test module run as script. guessing baseline image locations')
Expand All @@ -143,13 +187,13 @@ def _image_directories(func):
subdir = os.path.splitext(os.path.split(script_name)[1])[0]
else:
mods = module_name.split('.')
assert mods.pop(0)=='matplotlib'
assert mods.pop(0)=='tests'
assert mods.pop(0) == 'matplotlib'
assert mods.pop(0) == 'tests'
subdir = os.path.join(*mods)
basedir = os.path.dirname(matplotlib.tests.__file__)

baseline_dir = os.path.join(basedir,'baseline_images',subdir)
result_dir = os.path.abspath(os.path.join('result_images',subdir))
baseline_dir = os.path.join(basedir, 'baseline_images', subdir)
result_dir = os.path.abspath(os.path.join('result_images', subdir))

if not os.path.exists(result_dir):
os.makedirs(result_dir)
Expand Down
Binary file added lib/matplotlib/tests/baseline_images/test_axes/polar_units_2.pdf
View file
Open in desktop
Binary file not shown.
Binary file added lib/matplotlib/tests/baseline_images/test_axes/polar_units_2.png
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading