From f38046b445df61d8a4747ddaaccab23006bbab91 Mon Sep 17 00:00:00 2001 From: pwuertz Date: Tue, 21 Aug 2012 13:58:49 +0200 Subject: [PATCH 1/6] backend_pgf: rework of the latex communication process for python3 --- lib/matplotlib/backends/backend_pgf.py | 92 +++++++++++++------------- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/lib/matplotlib/backends/backend_pgf.py b/lib/matplotlib/backends/backend_pgf.py index 5684522efdc6..74fffa0ae926 100644 --- a/lib/matplotlib/backends/backend_pgf.py +++ b/lib/matplotlib/backends/backend_pgf.py @@ -220,15 +220,33 @@ def _build_latex_header(): # Create LaTeX header with some content, else LaTeX will load some # math fonts later when we don't expect the additional output on stdout. # TODO: is this sufficient? - latex_header = u"""\\documentclass{minimal} -%s -%s -\\begin{document} -text $math \mu$ %% force latex to load fonts now -\\typeout{pgf_backend_query_start} -""" % (latex_preamble, latex_fontspec) + latex_header = [r"\documentclass{minimal}", + latex_preamble, + latex_fontspec, + r"\begin{document}", + r"text $math \mu$", # force latex to load fonts now + r"\typeout{pgf_backend_query_start}"] + return "\n".join(latex_header) + + def _stdin_writeln(self, s): + self.latex_stdin_utf8.write(s) + self.latex_stdin_utf8.write("\n") + self.latex_stdin_utf8.flush() + + def _expect(self, s): + exp = s.encode("utf8") + buf = bytearray() + while True: + b = self.latex.stdout.read(1) + buf += b + if buf[-len(exp):] == exp: + break + if not len(b): + raise LatexError("LaTeX process halted", buf.decode("utf8")) + return buf.decode("utf8") - return latex_header + def _expect_prompt(self): + return self._expect("\n*") def __init__(self): self.texcommand = get_texcommand() @@ -238,27 +256,23 @@ def __init__(self): # test the LaTeX setup to ensure a clean startup of the subprocess latex = subprocess.Popen([self.texcommand, "-halt-on-error"], stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - universal_newlines=True) - stdout, stderr = latex.communicate(self.latex_header + latex_end) + stdout=subprocess.PIPE) + test_input = self.latex_header + latex_end + stdout, stderr = latex.communicate(test_input.encode("utf-8")) if latex.returncode != 0: raise LatexError("LaTeX returned an error, probably missing font or error in preamble:\n%s" % stdout) - # open LaTeX process + # open LaTeX process for real work latex = subprocess.Popen([self.texcommand, "-halt-on-error"], stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - universal_newlines=True) - latex.stdin.write(self.latex_header) - latex.stdin.flush() - # read all lines until our 'pgf_backend_query_start' token appears - while not latex.stdout.readline().startswith("*pgf_backend_query_start"): - pass - while latex.stdout.read(1) != '*': - pass + stdout=subprocess.PIPE) self.latex = latex - self.latex_stdin = codecs.getwriter("utf-8")(latex.stdin) - self.latex_stdout = codecs.getreader("utf-8")(latex.stdout) + self.latex_stdin_utf8 = codecs.getwriter("utf8")(self.latex.stdin) + # write header with 'pgf_backend_query_start' token + self._stdin_writeln(self._build_latex_header()) + # read all lines until our 'pgf_backend_query_start' token appears + self._expect("*pgf_backend_query_start") + self._expect_prompt() # cache for strings already processed self.str_cache = {} @@ -277,19 +291,6 @@ def __del__(self): except: pass - def _wait_for_prompt(self): - """ - Read all bytes from LaTeX stdout until a new line starts with a *. - """ - buf = [""] - while True: - buf.append(self.latex_stdout.read(1)) - if buf[-1] == "*" and buf[-2] == "\n": - break - if buf[-1] == "": - raise LatexError("LaTeX process halted", u"".join(buf)) - return "".join(buf) - def get_width_height_descent(self, text, prop): """ Get the width, total height and descent for a text typesetted by the @@ -298,30 +299,27 @@ def get_width_height_descent(self, text, prop): # apply font properties and define textbox prop_cmds = _font_properties_str(prop) - textbox = u"\\sbox0{%s %s}\n" % (prop_cmds, text) + textbox = "\\sbox0{%s %s}" % (prop_cmds, text) # check cache if textbox in self.str_cache: return self.str_cache[textbox] # send textbox to LaTeX and wait for prompt - self.latex_stdin.write(unicode(textbox)) - self.latex_stdin.flush() + self._stdin_writeln(textbox) try: - self._wait_for_prompt() + self._expect_prompt() except LatexError as e: - msg = u"Error processing '%s'\nLaTeX Output:\n%s" % (text, e.latex_output) + msg = "Error processing '%s'\nLaTeX Output:\n%s" % (text, e.latex_output) raise ValueError(msg) # typeout width, height and text offset of the last textbox - query = "\\typeout{\\the\\wd0,\\the\\ht0,\\the\\dp0}\n" - self.latex_stdin.write(query) - self.latex_stdin.flush() + self._stdin_writeln(r"\typeout{\the\wd0,\the\ht0,\the\dp0}") # read answer from latex and advance to the next prompt try: - answer = self._wait_for_prompt() + answer = self._expect_prompt() except LatexError as e: - msg = u"Error processing '%s'\nLaTeX Output:\n%s" % (text, e.latex_output) + msg = "Error processing '%s'\nLaTeX Output:\n%s" % (text, e.latex_output) raise ValueError(msg) # parse metrics from the answer string @@ -712,7 +710,7 @@ def print_pdf(self, filename, *args, **kwargs): texcommand = get_texcommand() cmdargs = [texcommand, "-interaction=nonstopmode", "-halt-on-error", "figure.tex"] try: - stdout = subprocess.check_output(cmdargs, universal_newlines=True, stderr=subprocess.STDOUT) + stdout = subprocess.check_output(cmdargs, stderr=subprocess.STDOUT) except: raise RuntimeError("%s was not able to process your file.\n\nFull log:\n%s" % (texcommand, stdout)) shutil.copyfile("figure.pdf", target) From 6b571899eac97a9b9da4cdf4b90cf9901a7f098a Mon Sep 17 00:00:00 2001 From: pwuertz Date: Tue, 21 Aug 2012 14:28:32 +0200 Subject: [PATCH 2/6] backend_pgf: fix error in exception handling --- lib/matplotlib/backends/backend_pgf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/matplotlib/backends/backend_pgf.py b/lib/matplotlib/backends/backend_pgf.py index 74fffa0ae926..415926b26441 100644 --- a/lib/matplotlib/backends/backend_pgf.py +++ b/lib/matplotlib/backends/backend_pgf.py @@ -711,8 +711,8 @@ def print_pdf(self, filename, *args, **kwargs): cmdargs = [texcommand, "-interaction=nonstopmode", "-halt-on-error", "figure.tex"] try: stdout = subprocess.check_output(cmdargs, stderr=subprocess.STDOUT) - except: - raise RuntimeError("%s was not able to process your file.\n\nFull log:\n%s" % (texcommand, stdout)) + except subprocess.CalledProcessError as e: + raise RuntimeError("%s was not able to process your file.\n\nFull log:\n%s" % (texcommand, e.output)) shutil.copyfile("figure.pdf", target) finally: os.chdir(cwd) From bf725a0ea75f8a16b00116dbfd4823e84e6406da Mon Sep 17 00:00:00 2001 From: pwuertz Date: Tue, 21 Aug 2012 18:10:36 +0200 Subject: [PATCH 3/6] backend_pgf: check latex+pgf environment when running tests --- lib/matplotlib/tests/test_backend_pgf.py | 29 ++++++++++++++++-------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/lib/matplotlib/tests/test_backend_pgf.py b/lib/matplotlib/tests/test_backend_pgf.py index 10993927fe5b..16693ef53b73 100644 --- a/lib/matplotlib/tests/test_backend_pgf.py +++ b/lib/matplotlib/tests/test_backend_pgf.py @@ -11,12 +11,21 @@ baseline_dir, result_dir = _image_directories(lambda: 'dummy func') -def run(*args): - try: - subprocess.check_output(args) - return True - except: - return False +def check_for(texsystem): + header = r""" + \documentclass{minimal} + \usepackage{pgf} + \begin{document} + \typeout{pgfversion=\pgfversion} + \makeatletter + \@@end + """ + latex = subprocess.Popen(["xelatex", "-halt-on-error"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + stdout, stderr = latex.communicate(header.encode("utf8")) + + return latex.returncode == 0 def switch_backend(backend): import nose @@ -58,7 +67,7 @@ def create_figure(): # test compiling a figure to pdf with xelatex -@knownfailureif(not run('xelatex', '-v'), msg="xelatex is required for this test") +@knownfailureif(not check_for('xelatex'), msg='xelatex + pgf is required') @switch_backend('pgf') def test_xelatex(): rc_xelatex = {'font.family': 'serif', @@ -69,7 +78,7 @@ def test_xelatex(): # test compiling a figure to pdf with pdflatex -@knownfailureif(not run('pdflatex', '-v'), msg="pdflatex is required for this test") +@knownfailureif(not check_for('pdflatex'), msg='pdflatex + pgf is required') @switch_backend('pgf') def test_pdflatex(): rc_pdflatex = {'font.family': 'serif', @@ -83,8 +92,8 @@ def test_pdflatex(): # test updating the rc parameters for each figure -@knownfailureif(not run('pdflatex', '-v') or not run('xelatex', '-v'), - msg="xelatex and pdflatex are required for this test") +@knownfailureif(not check_for('xelatex') or not check_for('pdflatex'), + msg="xelatex and pdflatex + pgf required") @switch_backend('pgf') def test_rcupdate(): rc_sets = [] From d1d39676920115ed7be3a1b88e5b82478d5a02e6 Mon Sep 17 00:00:00 2001 From: pwuertz Date: Tue, 21 Aug 2012 22:06:04 +0200 Subject: [PATCH 4/6] backend_pgf: support writing to file-like objects --- lib/matplotlib/backends/backend_pgf.py | 110 ++++++++++++++++--------- 1 file changed, 73 insertions(+), 37 deletions(-) diff --git a/lib/matplotlib/backends/backend_pgf.py b/lib/matplotlib/backends/backend_pgf.py index 415926b26441..41a331dd53f0 100644 --- a/lib/matplotlib/backends/backend_pgf.py +++ b/lib/matplotlib/backends/backend_pgf.py @@ -18,6 +18,7 @@ from matplotlib import _png, rcParams from matplotlib import font_manager from matplotlib.ft2font import FT2Font +from matplotlib.cbook import is_string_like, is_writable_file_like ############################################################################### @@ -623,12 +624,7 @@ def __init__(self, *args): def get_default_filetype(self): return 'pdf' - def print_pgf(self, filename, *args, **kwargs): - """ - Output pgf commands for drawing the figure so it can be included and - rendered in latex documents. - """ - + def _print_pgf_to_fh(self, fh): header_text = r"""%% Creator: Matplotlib, PGF backend %% %% To include the figure in your LaTeX document, write @@ -658,37 +654,50 @@ def print_pgf(self, filename, *args, **kwargs): # get figure size in inch w, h = self.figure.get_figwidth(), self.figure.get_figheight() - # start a pgfpicture environment and set a bounding box - with codecs.open(filename, "w", encoding="utf-8") as fh: - fh.write(header_text) - fh.write(header_info_preamble) - fh.write("\n") - writeln(fh, r"\begingroup") - writeln(fh, r"\makeatletter") - writeln(fh, r"\begin{pgfpicture}") - writeln(fh, r"\pgfpathrectangle{\pgfpointorigin}{\pgfqpoint{%fin}{%fin}}" % (w,h)) - writeln(fh, r"\pgfusepath{use as bounding box}") - - renderer = RendererPgf(self.figure, fh) - self.figure.draw(renderer) - - # end the pgfpicture environment - writeln(fh, r"\end{pgfpicture}") - writeln(fh, r"\makeatother") - writeln(fh, r"\endgroup") - - def print_pdf(self, filename, *args, **kwargs): + # create pgfpicture environment and write the pgf code + fh.write(header_text) + fh.write(header_info_preamble) + fh.write("\n") + writeln(fh, r"\begingroup") + writeln(fh, r"\makeatletter") + writeln(fh, r"\begin{pgfpicture}") + writeln(fh, r"\pgfpathrectangle{\pgfpointorigin}{\pgfqpoint{%fin}{%fin}}" % (w,h)) + writeln(fh, r"\pgfusepath{use as bounding box}") + renderer = RendererPgf(self.figure, fh) + self.figure.draw(renderer) + + # end the pgfpicture environment + writeln(fh, r"\end{pgfpicture}") + writeln(fh, r"\makeatother") + writeln(fh, r"\endgroup") + + def print_pgf(self, fname_or_fh, *args, **kwargs): """ - Use LaTeX to compile a Pgf generated figure to PDF. + Output pgf commands for drawing the figure so it can be included and + rendered in latex documents. """ - w, h = self.figure.get_figwidth(), self.figure.get_figheight() + if kwargs.get("dryrun", False): return + + # figure out where the pgf is to be written to + if is_string_like(fname_or_fh): + with codecs.open(fname_or_fh, "w", encoding="utf-8") as fh: + self._print_pgf_to_fh(fh) + elif is_writable_file_like(fname_or_fh): + raise ValueError("saving pgf to a stream is not supported, " + \ + "consider using the pdf option of the pgf-backend") + else: + raise ValueError("filename must be a path") - target = os.path.abspath(filename) + def _print_pdf_to_fh(self, fh): + w, h = self.figure.get_figwidth(), self.figure.get_figheight() try: + # create and switch to temporary directory tmpdir = tempfile.mkdtemp() cwd = os.getcwd() os.chdir(tmpdir) + + # print figure to pgf and compile it with latex self.print_pgf("figure.pgf") latex_preamble = get_preamble() @@ -704,16 +713,19 @@ def print_pdf(self, filename, *args, **kwargs): \centering \input{figure.pgf} \end{document}""" % (w, h, latex_preamble, latex_fontspec) - with codecs.open("figure.tex", "w", "utf-8") as fh: - fh.write(latexcode) + with codecs.open("figure.tex", "w", "utf-8") as fh_tex: + fh_tex.write(latexcode) texcommand = get_texcommand() cmdargs = [texcommand, "-interaction=nonstopmode", "-halt-on-error", "figure.tex"] try: - stdout = subprocess.check_output(cmdargs, stderr=subprocess.STDOUT) + subprocess.check_output(cmdargs, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: raise RuntimeError("%s was not able to process your file.\n\nFull log:\n%s" % (texcommand, e.output)) - shutil.copyfile("figure.pdf", target) + + # copy file contents to target + with open("figure.pdf", "rb") as fh_src: + shutil.copyfileobj(fh_src, fh) finally: os.chdir(cwd) try: @@ -721,21 +733,33 @@ def print_pdf(self, filename, *args, **kwargs): except: sys.stderr.write("could not delete tmp directory %s\n" % tmpdir) - def print_png(self, filename, *args, **kwargs): + def print_pdf(self, fname_or_fh, *args, **kwargs): """ - Use LaTeX to compile a pgf figure to pdf and convert it to png. + Use LaTeX to compile a Pgf generated figure to PDF. """ + # figure out where the pdf is to be written to + if is_string_like(fname_or_fh): + with open(fname_or_fh, "wb") as fh: + self._print_pdf_to_fh(fh) + elif is_writable_file_like(fname_or_fh): + self._print_pdf_to_fh(fname_or_fh) + else: + raise ValueError("filename must be a path or a file-like object") + def _print_png_to_fh(self, fh): converter = make_pdf_to_png_converter() - target = os.path.abspath(filename) try: + # create and switch to temporary directory tmpdir = tempfile.mkdtemp() cwd = os.getcwd() os.chdir(tmpdir) + # create pdf and try to convert it to png self.print_pdf("figure.pdf") converter("figure.pdf", "figure.png", dpi=self.figure.dpi) - shutil.copyfile("figure.png", target) + # copy file contents to target + with open("figure.png", "rb") as fh_src: + shutil.copyfileobj(fh_src, fh) finally: os.chdir(cwd) try: @@ -743,6 +767,18 @@ def print_png(self, filename, *args, **kwargs): except: sys.stderr.write("could not delete tmp directory %s\n" % tmpdir) + def print_png(self, fname_or_fh, *args, **kwargs): + """ + Use LaTeX to compile a pgf figure to pdf and convert it to png. + """ + if is_string_like(fname_or_fh): + with open(fname_or_fh, "wb") as fh: + self._print_png_to_fh(fh) + elif is_writable_file_like(fname_or_fh): + self._print_png_to_fh(fname_or_fh) + else: + raise ValueError("filename must be a path or a file-like object") + def _render_texts_pgf(self, fh): # TODO: currently unused code path From 834faf7be1e95b4fcce30521e5de981d32b372e3 Mon Sep 17 00:00:00 2001 From: pwuertz Date: Wed, 22 Aug 2012 15:57:24 +0200 Subject: [PATCH 5/6] backend_pgf: use SkipTest instead knownfailureif, raise comparison tolerance --- lib/matplotlib/tests/test_backend_pgf.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/matplotlib/tests/test_backend_pgf.py b/lib/matplotlib/tests/test_backend_pgf.py index 16693ef53b73..ed0fbc0386c6 100644 --- a/lib/matplotlib/tests/test_backend_pgf.py +++ b/lib/matplotlib/tests/test_backend_pgf.py @@ -4,10 +4,12 @@ import shutil import subprocess import numpy as np +import nose +from nose.plugins.skip import SkipTest import matplotlib as mpl import matplotlib.pyplot as plt from matplotlib.testing.compare import compare_images, ImageComparisonFailure -from matplotlib.testing.decorators import _image_directories, knownfailureif +from matplotlib.testing.decorators import _image_directories baseline_dir, result_dir = _image_directories(lambda: 'dummy func') @@ -28,7 +30,6 @@ def check_for(texsystem): return latex.returncode == 0 def switch_backend(backend): - import nose def switch_backend_decorator(func): def backend_switcher(*args, **kwargs): @@ -50,7 +51,7 @@ def compare_figure(fname): expected = os.path.join(result_dir, "expected_%s" % fname) shutil.copyfile(os.path.join(baseline_dir, fname), expected) - err = compare_images(expected, actual, tol=1e-4) + err = compare_images(expected, actual, tol=5e-3) if err: raise ImageComparisonFailure('images not close: %s vs. %s' % (actual, expected)) @@ -67,9 +68,11 @@ def create_figure(): # test compiling a figure to pdf with xelatex -@knownfailureif(not check_for('xelatex'), msg='xelatex + pgf is required') @switch_backend('pgf') def test_xelatex(): + if not check_for('xelatex'): + raise SkipTest('xelatex + pgf is required') + rc_xelatex = {'font.family': 'serif', 'pgf.rcfonts': False,} mpl.rcParams.update(rc_xelatex) @@ -78,9 +81,11 @@ def test_xelatex(): # test compiling a figure to pdf with pdflatex -@knownfailureif(not check_for('pdflatex'), msg='pdflatex + pgf is required') @switch_backend('pgf') def test_pdflatex(): + if not check_for('pdflatex'): + raise SkipTest('pdflatex + pgf is required') + rc_pdflatex = {'font.family': 'serif', 'pgf.rcfonts': False, 'pgf.texsystem': 'pdflatex', @@ -92,10 +97,11 @@ def test_pdflatex(): # test updating the rc parameters for each figure -@knownfailureif(not check_for('xelatex') or not check_for('pdflatex'), - msg="xelatex and pdflatex + pgf required") @switch_backend('pgf') def test_rcupdate(): + if not check_for('xelatex') or not check_for('pdflatex'): + raise SkipTest('xelatex and pdflatex + pgf required') + rc_sets = [] rc_sets.append({'font.family': 'sans-serif', 'font.size': 30, From 138d55dad6e99799481f202f4ba993b301c0e628 Mon Sep 17 00:00:00 2001 From: pwuertz Date: Wed, 22 Aug 2012 20:36:41 +0200 Subject: [PATCH 6/6] backend_pgf: closing subprocess differently, eliminates some py3 warnings --- lib/matplotlib/backends/backend_pgf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/matplotlib/backends/backend_pgf.py b/lib/matplotlib/backends/backend_pgf.py index 41a331dd53f0..22e900b8a98b 100644 --- a/lib/matplotlib/backends/backend_pgf.py +++ b/lib/matplotlib/backends/backend_pgf.py @@ -282,8 +282,8 @@ def __del__(self): if rcParams.get("pgf.debug", False): print "deleting LatexManager" try: - self.latex.terminate() - self.latex.wait() + self.latex_stdin_utf8.close() + self.latex.communicate() except: pass try: