From 76eb88165b6faa0f06115f751f5c3b4864a8d955 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 20 Mar 2017 19:28:18 -0700 Subject: [PATCH 1/9] [ENH] Sphinx extension to plot workflows Close #1873 --- doc/conf.py | 1 + nipype/sphinxext/__init__.py | 5 + nipype/sphinxext/plot_workflow.py | 670 ++++++++++++++++++++++++++++++ 3 files changed, 676 insertions(+) create mode 100644 nipype/sphinxext/__init__.py create mode 100644 nipype/sphinxext/plot_workflow.py diff --git a/doc/conf.py b/doc/conf.py index 9ed5c87da9..b69379d9a0 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -52,6 +52,7 @@ 'numpy_ext.numpydoc', 'matplotlib.sphinxext.plot_directive', 'matplotlib.sphinxext.only_directives', + 'nipype.sphinxext.plot_workflow', #'IPython.sphinxext.ipython_directive', #'IPython.sphinxext.ipython_console_highlighting' ] diff --git a/nipype/sphinxext/__init__.py b/nipype/sphinxext/__init__.py new file mode 100644 index 0000000000..b2033960f3 --- /dev/null +++ b/nipype/sphinxext/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +from __future__ import print_function, division, absolute_import, unicode_literals diff --git a/nipype/sphinxext/plot_workflow.py b/nipype/sphinxext/plot_workflow.py new file mode 100644 index 0000000000..11428858da --- /dev/null +++ b/nipype/sphinxext/plot_workflow.py @@ -0,0 +1,670 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +""" +A directive for including a nipype workflow graph in a Sphinx document. + +By default, in HTML output, `workflow` will include a .png file with a +link to a high-res .png. In LaTeX output, it will include a +.pdf. +The source code for the workflow may be included as **inline content** to +the directive:: + .. workflow:: + from mriqc.workflows.anatomical import airmsk_wf + wf = airmsk_wf() + + +Options +------- + +The ``workflow`` directive supports the following options: + format : {'python', 'doctest'} + Specify the format of the input + include-source : bool + Whether to display the source code. The default can be changed + using the `workflow_include_source` variable in conf.py + encoding : str + If this source file is in a non-UTF8 or non-ASCII encoding, + the encoding must be specified using the `:encoding:` option. + The encoding will not be inferred using the ``-*- coding -*-`` + metacomment. + +Additionally, this directive supports all of the options of the +`image` directive, except for `target` (since workflow will add its own +target). These include `alt`, `height`, `width`, `scale`, `align` and +`class`. + +Configuration options +--------------------- +The workflow directive has the following configuration options: + wf_include_source + Default value for the include-source option + wf_html_show_source_link + Whether to show a link to the source in HTML. + wf_pre_code + Code that should be executed before each workflow. + wf_basedir + Base directory, to which ``workflow::`` file names are relative + to. (If None or empty, file names are relative to the + directory where the file containing the directive is.) + wf_formats + File formats to generate. List of tuples or strings:: + [(suffix, dpi), suffix, ...] + that determine the file format and the DPI. For entries whose + DPI was omitted, sensible defaults are chosen. When passing from + the command line through sphinx_build the list should be passed as + suffix:dpi,suffix:dpi, .... + wf_html_show_formats + Whether to show links to the files in HTML. + wf_rcparams + A dictionary containing any non-standard rcParams that should + be applied before each workflow. + wf_apply_rcparams + By default, rcParams are applied when `context` option is not used in + a workflow directive. This configuration option overrides this behavior + and applies rcParams before each workflow. + wf_working_directory + By default, the working directory will be changed to the directory of + the example, so the code can get at its data files, if any. Also its + path will be added to `sys.path` so it can import any helper modules + sitting beside it. This configuration option can be used to specify + a central directory (also added to `sys.path`) where data files and + helper modules for all code are located. + wf_template + Provide a customized template for preparing restructured text. + +""" +from __future__ import print_function, division, absolute_import, unicode_literals + +import sys, os, shutil, io, re, textwrap +from os.path import relpath +import traceback + +from docutils.parsers.rst import directives +from docutils.parsers.rst.directives.images import Image + +from mriqc.utils.misc import check_folder as mkdirs + + + +try: + # Sphinx depends on either Jinja or Jinja2 + import jinja2 + def format_template(template, **kw): + return jinja2.Template(template).render(**kw) +except ImportError: + import jinja + def format_template(template, **kw): + return jinja.from_string(template, **kw) + +from builtins import str, bytes +align = Image.align + +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 + + +def wf_directive(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + return run(arguments, content, options, state_machine, state, lineno) +wf_directive.__doc__ = __doc__ + +def _option_boolean(arg): + if not arg or not arg.strip(): + # no argument given, assume used as a flag + return True + elif arg.strip().lower() in ('no', '0', 'false'): + return False + elif arg.strip().lower() in ('yes', '1', 'true'): + return True + else: + raise ValueError('"%s" unknown boolean' % arg) + + +def _option_context(arg): + if arg in [None, 'reset', 'close-figs']: + return arg + raise ValueError("argument should be None or 'reset' or 'close-figs'") + + +def _option_format(arg): + return directives.choice(arg, ('python', 'doctest')) + + +def _option_align(arg): + return directives.choice(arg, ("top", "middle", "bottom", "left", "center", + "right")) + + +def mark_wf_labels(app, document): + """ + To make graphs referenceable, we need to move the reference from + the "htmlonly" (or "latexonly") node to the actual figure node + itself. + """ + for name, explicit in list(document.nametypes.items()): + if not explicit: + continue + labelid = document.nameids[name] + if labelid is None: + continue + node = document.ids[labelid] + if node.tagname in ('html_only', 'latex_only'): + for n in node: + if n.tagname == 'figure': + sectname = name + for c in n: + if c.tagname == 'caption': + sectname = c.astext() + break + + node['ids'].remove(labelid) + node['names'].remove(name) + n['ids'].append(labelid) + n['names'].append(name) + document.settings.env.labels[name] = \ + document.settings.env.docname, labelid, sectname + break + + +def setup(app): + setup.app = app + setup.config = app.config + setup.confdir = app.confdir + + options = {'alt': directives.unchanged, + 'height': directives.length_or_unitless, + 'width': directives.length_or_percentage_or_unitless, + 'scale': directives.nonnegative_int, + 'align': _option_align, + 'class': directives.class_option, + 'include-source': _option_boolean, + 'format': _option_format, + 'context': _option_context, + 'nofigs': directives.flag, + 'encoding': directives.encoding + } + + app.add_directive('workflow', wf_directive, True, (0, 2, False), **options) + app.add_config_value('wf_pre_code', None, True) + app.add_config_value('wf_include_source', False, True) + app.add_config_value('wf_html_show_source_link', True, True) + app.add_config_value('wf_formats', ['png', 'hires.png', 'pdf'], True) + app.add_config_value('wf_basedir', None, True) + app.add_config_value('wf_html_show_formats', True, True) + app.add_config_value('wf_rcparams', {}, True) + app.add_config_value('wf_apply_rcparams', False, True) + app.add_config_value('wf_working_directory', None, True) + app.add_config_value('wf_template', None, True) + + app.connect('doctree-read'.encode() if PY2 else 'doctree-read', mark_wf_labels) + + metadata = {'parallel_read_safe': True, 'parallel_write_safe': True} + return metadata + + +#------------------------------------------------------------------------------ +# Doctest handling +#------------------------------------------------------------------------------ + +def contains_doctest(text): + try: + # check if it's valid Python as-is + compile(text, '', 'exec') + return False + except SyntaxError: + pass + r = re.compile(r'^\s*>>>', re.M) + m = r.search(text) + return bool(m) + + +def unescape_doctest(text): + """ + Extract code from a piece of text, which contains either Python code + or doctests. + """ + if not contains_doctest(text): + return text + + code = "" + for line in text.split("\n"): + m = re.match(r'^\s*(>>>|\.\.\.) (.*)$', line) + if m: + code += m.group(2) + "\n" + elif line.strip(): + code += "# " + line.strip() + "\n" + else: + code += "\n" + return code + + +def remove_coding(text): + """ + Remove the coding comment, which exec doesn't like. + """ + sub_re = re.compile("^#\s*-\*-\s*coding:\s*.*-\*-$", flags=re.MULTILINE) + return sub_re.sub("", text) + +#------------------------------------------------------------------------------ +# Template +#------------------------------------------------------------------------------ + + +TEMPLATE = """ +{{ source_code }} +{{ only_html }} + {% if source_link or (html_show_formats and not multi_image) %} + ( + {%- if source_link -%} + `Source code <{{ source_link }}>`__ + {%- endif -%} + {%- if html_show_formats and not multi_image -%} + {%- for img in images -%} + {%- for fmt in img.formats -%} + {%- if source_link or not loop.first -%}, {% endif -%} + `{{ fmt }} <{{ dest_dir }}/{{ img.basename }}.{{ fmt }}>`__ + {%- endfor -%} + {%- endfor -%} + {%- endif -%} + ) + {% endif %} + {% for img in images %} + .. figure:: {{ build_dir }}/{{ img.basename }}.{{ default_fmt }} + {% for option in options -%} + {{ option }} + {% endfor %} + {% if html_show_formats and multi_image -%} + ( + {%- for fmt in img.formats -%} + {%- if not loop.first -%}, {% endif -%} + `{{ fmt }} <{{ dest_dir }}/{{ img.basename }}.{{ fmt }}>`__ + {%- endfor -%} + ) + {%- endif -%} + {{ caption }} + {% endfor %} +{{ only_latex }} + {% for img in images %} + {% if 'pdf' in img.formats -%} + .. figure:: {{ build_dir }}/{{ img.basename }}.pdf + {% for option in options -%} + {{ option }} + {% endfor %} + {{ caption }} + {% endif -%} + {% endfor %} +{{ only_texinfo }} + {% for img in images %} + .. image:: {{ build_dir }}/{{ img.basename }}.png + {% for option in options -%} + {{ option }} + {% endfor %} + {% endfor %} +""" + +exception_template = """ +.. htmlonly:: + [`source code <%(linkdir)s/%(basename)s.py>`__] +Exception occurred rendering plot. +""" + +# the context of the plot for all directives specified with the +# :context: option +wf_context = dict() + +class ImageFile(object): + def __init__(self, basename, dirname): + self.basename = basename + self.dirname = dirname + self.formats = [] + + def filename(self, fmt): + return os.path.join(self.dirname, "%s.%s" % (self.basename, fmt)) + + def filenames(self): + return [self.filename(fmt) for fmt in self.formats] + + +def out_of_date(original, derived): + """ + Returns True if derivative is out-of-date wrt original, + both of which are full file paths. + """ + return (not os.path.exists(derived) or + (os.path.exists(original) and + os.stat(derived).st_mtime < os.stat(original).st_mtime)) + + +class GraphError(RuntimeError): + pass + + +def run_code(code, code_path, ns=None, function_name=None): + """ + Import a Python module from a path, and run the function given by + name, if function_name is not None. + """ + + # Change the working directory to the directory of the example, so + # it can get at its data files, if any. Add its path to sys.path + # so it can import any helper modules sitting beside it. + pwd = str(os.getcwd()) + old_sys_path = list(sys.path) + if setup.config.wf_working_directory is not None: + try: + os.chdir(setup.config.wf_working_directory) + except OSError as err: + raise OSError(str(err) + '\n`wf_working_directory` option in' + 'Sphinx configuration file must be a valid ' + 'directory path') + except TypeError as err: + raise TypeError(str(err) + '\n`wf_working_directory` option in ' + 'Sphinx configuration file must be a string or ' + 'None') + sys.path.insert(0, setup.config.wf_working_directory) + elif code_path is not None: + dirname = os.path.abspath(os.path.dirname(code_path)) + os.chdir(dirname) + sys.path.insert(0, dirname) + + # Reset sys.argv + old_sys_argv = sys.argv + sys.argv = [code_path] + + # Redirect stdout + stdout = sys.stdout + if PY3: + sys.stdout = io.StringIO() + else: + from cStringIO import StringIO + sys.stdout = StringIO() + + # Assign a do-nothing print function to the namespace. There + # doesn't seem to be any other way to provide a way to (not) print + # that works correctly across Python 2 and 3. + def _dummy_print(*arg, **kwarg): + pass + + try: + try: + code = unescape_doctest(code) + if ns is None: + ns = {} + if not ns: + if setup.config.wf_pre_code is not None: + exec(str(setup.config.wf_pre_code), ns) + ns['print'] = _dummy_print + if "__main__" in code: + exec("__name__ = '__main__'", ns) + code = remove_coding(code) + exec(code, ns) + if function_name is not None: + exec(function_name + "()", ns) + except (Exception, SystemExit) as err: + raise GraphError(traceback.format_exc()) + finally: + os.chdir(pwd) + sys.argv = old_sys_argv + sys.path[:] = old_sys_path + sys.stdout = stdout + return ns + + +def get_wf_formats(config): + default_dpi = {'png': 80, 'hires.png': 200, 'pdf': 200} + formats = [] + wf_formats = config.wf_formats + if isinstance(wf_formats, (str, bytes)): + # String Sphinx < 1.3, Split on , to mimic + # Sphinx 1.3 and later. Sphinx 1.3 always + # returns a list. + wf_formats = wf_formats.split(',') + for fmt in wf_formats: + if isinstance(fmt, (str, bytes)): + if ':' in fmt: + suffix, dpi = fmt.split(':') + formats.append((str(suffix), int(dpi))) + else: + formats.append((fmt, default_dpi.get(fmt, 80))) + elif isinstance(fmt, (tuple, list)) and len(fmt) == 2: + formats.append((str(fmt[0]), int(fmt[1]))) + else: + raise GraphError('invalid image format "%r" in wf_formats' % fmt) + return formats + + +def render_figures(code, code_path, output_dir, output_base, context, + function_name, config, context_reset=False, + close_figs=False): + """ + Run a nipype workflow creation script and save the graph in *output_dir*. + Save the images under *output_dir* with file names derived from + *output_base* + """ + formats = get_wf_formats(config) + + ns = wf_context if context else {} + if context_reset: + wf_context.clear() + + run_code(code, code_path, ns, function_name) + img = ImageFile(output_base, output_dir) + + for fmt, dpi in formats: + try: + img_path = img.filename(fmt) + imgname, ext = os.path.splitext(os.path.basename(img_path)) + ns['wf'].base_dir = output_dir + ns['wf'].write_graph(imgname, format=ext[1:]) + + src = os.path.join(os.path.dirname(img_path), ns['wf'].name, + os.path.basename(img_path)) + print(src, img_path) + shutil.move(src, img_path) + except Exception as err: + raise GraphError(traceback.format_exc()) + + img.formats.append(fmt) + + return [(code, [img])] + + +def run(arguments, content, options, state_machine, state, lineno): + document = state_machine.document + config = document.settings.env.config + nofigs = 'nofigs' in options + + formats = get_wf_formats(config) + default_fmt = formats[0][0] + + options.setdefault('include-source', config.wf_include_source) + keep_context = 'context' in options + context_opt = None if not keep_context else options['context'] + + rst_file = document.attributes['source'] + rst_dir = os.path.dirname(rst_file) + + if len(arguments): + if not config.wf_basedir: + source_file_name = os.path.join(setup.app.builder.srcdir, + directives.uri(arguments[0])) + else: + source_file_name = os.path.join(setup.confdir, config.wf_basedir, + directives.uri(arguments[0])) + + # If there is content, it will be passed as a caption. + caption = '\n'.join(content) + + # If the optional function name is provided, use it + if len(arguments) == 2: + function_name = arguments[1] + else: + function_name = None + + with io.open(source_file_name, 'r', encoding='utf-8') as fd: + code = fd.read() + output_base = os.path.basename(source_file_name) + else: + source_file_name = rst_file + code = textwrap.dedent("\n".join([str(c) for c in content])) + counter = document.attributes.get('_wf_counter', 0) + 1 + document.attributes['_wf_counter'] = counter + base, _ = os.path.splitext(os.path.basename(source_file_name)) + output_base = '%s-%d.py' % (base, counter) + function_name = None + caption = '' + + base, source_ext = os.path.splitext(output_base) + if source_ext in ('.py', '.rst', '.txt'): + output_base = base + else: + source_ext = '' + + # ensure that LaTeX includegraphics doesn't choke in foo.bar.pdf filenames + output_base = output_base.replace('.', '-') + + # is it in doctest format? + is_doctest = contains_doctest(code) + if 'format' in options: + if options['format'] == 'python': + is_doctest = False + else: + is_doctest = True + + # determine output directory name fragment + source_rel_name = relpath(source_file_name, setup.confdir) + source_rel_dir = os.path.dirname(source_rel_name) + while source_rel_dir.startswith(os.path.sep): + source_rel_dir = source_rel_dir[1:] + + # build_dir: where to place output files (temporarily) + build_dir = os.path.join(os.path.dirname(setup.app.doctreedir), + 'wf_directive', + source_rel_dir) + # get rid of .. in paths, also changes pathsep + # see note in Python docs for warning about symbolic links on Windows. + # need to compare source and dest paths at end + build_dir = os.path.normpath(build_dir) + + if not os.path.exists(build_dir): + os.makedirs(build_dir) + + # output_dir: final location in the builder's directory + dest_dir = os.path.abspath(os.path.join(setup.app.builder.outdir, + source_rel_dir)) + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) # no problem here for me, but just use built-ins + + # how to link to files from the RST file + dest_dir_link = os.path.join(relpath(setup.confdir, rst_dir), + source_rel_dir).replace(os.path.sep, '/') + try: + build_dir_link = relpath(build_dir, rst_dir).replace(os.path.sep, '/') + except ValueError: + # on Windows, relpath raises ValueError when path and start are on + # different mounts/drives + build_dir_link = build_dir + source_link = dest_dir_link + '/' + output_base + source_ext + + # make figures + try: + results = render_figures(code, + source_file_name, + build_dir, + output_base, + keep_context, + function_name, + config, + context_reset=context_opt == 'reset', + close_figs=context_opt == 'close-figs') + errors = [] + except GraphError as err: + reporter = state.memo.reporter + sm = reporter.system_message( + 2, "Exception occurred in plotting %s\n from %s:\n%s" % (output_base, + source_file_name, err), + line=lineno) + results = [(code, [])] + errors = [sm] + + # Properly indent the caption + caption = '\n'.join(' ' + line.strip() + for line in caption.split('\n')) + + # generate output restructuredtext + total_lines = [] + for j, (code_piece, images) in enumerate(results): + if options['include-source']: + if is_doctest: + lines = [''] + lines += [row.rstrip() for row in code_piece.split('\n')] + else: + lines = ['.. code-block:: python', ''] + lines += [' %s' % row.rstrip() + for row in code_piece.split('\n')] + source_code = "\n".join(lines) + else: + source_code = "" + + if nofigs: + images = [] + + opts = [':%s: %s' % (key, val) for key, val in list(options.items()) + if key in ('alt', 'height', 'width', 'scale', 'align', 'class')] + + only_html = ".. only:: html" + only_latex = ".. only:: latex" + only_texinfo = ".. only:: texinfo" + + # Not-None src_link signals the need for a source link in the generated + # html + if j == 0 and config.wf_html_show_source_link: + src_link = source_link + else: + src_link = None + + result = format_template( + config.wf_template or TEMPLATE, + default_fmt=default_fmt, + dest_dir=dest_dir_link, + build_dir=build_dir_link, + source_link=src_link, + multi_image=len(images) > 1, + only_html=only_html, + only_latex=only_latex, + only_texinfo=only_texinfo, + options=opts, + images=images, + source_code=source_code, + html_show_formats=config.wf_html_show_formats and len(images), + caption=caption) + + total_lines.extend(result.split("\n")) + total_lines.extend("\n") + + if total_lines: + state_machine.insert_input(total_lines, source=source_file_name) + + # copy image files to builder's output directory, if necessary + if not os.path.exists(dest_dir): + mkdirs(dest_dir) + + for code_piece, images in results: + for img in images: + for fn in img.filenames(): + destimg = os.path.join(dest_dir, os.path.basename(fn)) + if fn != destimg: + shutil.copyfile(fn, destimg) + + # copy script (if necessary) + target_name = os.path.join(dest_dir, output_base + source_ext) + with io.open(target_name, 'w', encoding="utf-8") as f: + if source_file_name == rst_file: + code_escaped = unescape_doctest(code) + else: + code_escaped = code + f.write(code_escaped) + + return errors From 805ba528614273201a8084f878007b9c4d7b63c4 Mon Sep 17 00:00:00 2001 From: oesteban Date: Mon, 20 Mar 2017 23:38:22 -0700 Subject: [PATCH 2/9] support graph2html and simple_form options --- doc/conf.py | 3 +- nipype/info.py | 2 +- nipype/pipeline/engine/utils.py | 24 ++++++----- nipype/pipeline/engine/workflows.py | 10 +++-- nipype/sphinxext/plot_workflow.py | 65 +++++++++++++++++------------ 5 files changed, 62 insertions(+), 42 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index b69379d9a0..116ff3f36e 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -42,12 +42,11 @@ # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.todo', - 'sphinx.ext.pngmath', + 'sphinx.ext.imgmath', 'sphinx.ext.inheritance_diagram', 'sphinx.ext.graphviz', 'sphinx.ext.autodoc', 'sphinx.ext.doctest', - 'sphinx.ext.pngmath', 'sphinx.ext.autosummary', 'numpy_ext.numpydoc', 'matplotlib.sphinxext.plot_directive', diff --git a/nipype/info.py b/nipype/info.py index 81a8adce0a..c70b555ee3 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -152,7 +152,7 @@ def get_nipype_gitversion(): ] EXTRA_REQUIRES = { - 'doc': ['Sphinx>=0.3', 'matplotlib', 'pydotplus'], + 'doc': ['Sphinx>=1.4', 'matplotlib', 'pydotplus'], 'tests': TESTS_REQUIRES, 'fmri': ['nitime', 'nilearn', 'dipy', 'nipy', 'matplotlib'], 'profiler': ['psutil'], diff --git a/nipype/pipeline/engine/utils.py b/nipype/pipeline/engine/utils.py index ce2f927b15..0211de16f6 100644 --- a/nipype/pipeline/engine/utils.py +++ b/nipype/pipeline/engine/utils.py @@ -1018,20 +1018,18 @@ def export_graph(graph_in, base_dir=None, show=False, use_execgraph=False, suffix='_detailed.dot', use_ext=False, newpath=base_dir) - logger.info('Creating detailed dot file: %s' % outfname) _write_detailed_dot(graph, outfname) cmd = 'dot -T%s -O %s' % (format, outfname) res = CommandLine(cmd, terminal_output='allatonce').run() if res.runtime.returncode: logger.warn('dot2png: %s', res.runtime.stderr) pklgraph = _create_dot_graph(graph, show_connectinfo, simple_form) - outfname = fname_presuffix(dotfilename, - suffix='.dot', - use_ext=False, - newpath=base_dir) - nx.drawing.nx_pydot.write_dot(pklgraph, outfname) - logger.info('Creating dot file: %s' % outfname) - cmd = 'dot -T%s -O %s' % (format, outfname) + simplefname = fname_presuffix(dotfilename, + suffix='.dot', + use_ext=False, + newpath=base_dir) + nx.drawing.nx_pydot.write_dot(pklgraph, simplefname) + cmd = 'dot -T%s -O %s' % (format, simplefname) res = CommandLine(cmd, terminal_output='allatonce').run() if res.runtime.returncode: logger.warn('dot2png: %s', res.runtime.stderr) @@ -1041,6 +1039,10 @@ def export_graph(graph_in, base_dir=None, show=False, use_execgraph=False, if show_connectinfo: nx.draw_networkx_edge_labels(pklgraph, pos) + if format != 'dot': + outfname += '.%s' % format + return outfname + def format_dot(dotfilename, format=None): """Dump a directed graph (Linux only; install via `brew` on OSX)""" @@ -1052,8 +1054,10 @@ def format_dot(dotfilename, format=None): raise IOError("Cannot draw directed graph; executable 'dot' is unavailable") else: raise ioe - else: - logger.info('Converting dotfile: %s to %s format' % (dotfilename, format)) + + if format != 'dot': + dotfilename += '.%s' % format + return dotfilename def make_output_dir(outdir): diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py index 5a8c5eed56..50b47c63a9 100644 --- a/nipype/pipeline/engine/workflows.py +++ b/nipype/pipeline/engine/workflows.py @@ -429,15 +429,19 @@ def write_graph(self, dotfilename='graph.dot', graph2use='hierarchical', self.write_hierarchical_dotfile(dotfilename=dotfilename, colored=graph2use == "colored", simple_form=simple_form) - format_dot(dotfilename, format=format) + outfname = format_dot(dotfilename, format=format) else: graph = self._graph if graph2use in ['flat', 'exec']: graph = self._create_flat_graph() if graph2use == 'exec': graph = generate_expanded_graph(deepcopy(graph)) - export_graph(graph, base_dir, dotfilename=dotfilename, - format=format, simple_form=simple_form) + outfname = export_graph(graph, base_dir, dotfilename=dotfilename, + format=format, simple_form=simple_form) + + logger.info('Generated workflow graph: %s (graph2use=%s, simple_form=%s).' % ( + outfname, graph2use, simple_form)) + return outfname def write_hierarchical_dotfile(self, dotfilename=None, colored=False, simple_form=True): diff --git a/nipype/sphinxext/plot_workflow.py b/nipype/sphinxext/plot_workflow.py index 11428858da..1d7aed5082 100644 --- a/nipype/sphinxext/plot_workflow.py +++ b/nipype/sphinxext/plot_workflow.py @@ -10,6 +10,7 @@ .pdf. The source code for the workflow may be included as **inline content** to the directive:: + .. workflow:: from mriqc.workflows.anatomical import airmsk_wf wf = airmsk_wf() @@ -38,6 +39,11 @@ Configuration options --------------------- The workflow directive has the following configuration options: + graph2use + Select a graph type to use + simple_form + determines if the node name shown in the visualization is either of the form nodename + (package) when set to True or nodename.Class.package when set to False. wf_include_source Default value for the include-source option wf_html_show_source_link @@ -121,6 +127,8 @@ def _option_boolean(arg): else: raise ValueError('"%s" unknown boolean' % arg) +def _option_graph2use(arg): + return directives.choice(arg, ('hierarchical', 'colored', 'flat', 'orig', 'exec')) def _option_context(arg): if arg in [None, 'reset', 'close-figs']: @@ -183,14 +191,18 @@ def setup(app): 'format': _option_format, 'context': _option_context, 'nofigs': directives.flag, - 'encoding': directives.encoding + 'encoding': directives.encoding, + 'graph2use': _option_graph2use, + 'simple_form': _option_boolean } app.add_directive('workflow', wf_directive, True, (0, 2, False), **options) + app.add_config_value('graph2use', 'hierarchical', 'html') + app.add_config_value('simple_form', True, 'html') app.add_config_value('wf_pre_code', None, True) app.add_config_value('wf_include_source', False, True) app.add_config_value('wf_html_show_source_link', True, True) - app.add_config_value('wf_formats', ['png', 'hires.png', 'pdf'], True) + app.add_config_value('wf_formats', ['png', 'svg', 'pdf'], True) app.add_config_value('wf_basedir', None, True) app.add_config_value('wf_html_show_formats', True, True) app.add_config_value('wf_rcparams', {}, True) @@ -255,21 +267,6 @@ def remove_coding(text): TEMPLATE = """ {{ source_code }} {{ only_html }} - {% if source_link or (html_show_formats and not multi_image) %} - ( - {%- if source_link -%} - `Source code <{{ source_link }}>`__ - {%- endif -%} - {%- if html_show_formats and not multi_image -%} - {%- for img in images -%} - {%- for fmt in img.formats -%} - {%- if source_link or not loop.first -%}, {% endif -%} - `{{ fmt }} <{{ dest_dir }}/{{ img.basename }}.{{ fmt }}>`__ - {%- endfor -%} - {%- endfor -%} - {%- endif -%} - ) - {% endif %} {% for img in images %} .. figure:: {{ build_dir }}/{{ img.basename }}.{{ default_fmt }} {% for option in options -%} @@ -285,6 +282,21 @@ def remove_coding(text): {%- endif -%} {{ caption }} {% endfor %} + {% if source_link or (html_show_formats and not multi_image) %} + ( + {%- if source_link -%} + `Source code <{{ source_link }}>`__ + {%- endif -%} + {%- if html_show_formats and not multi_image -%} + {%- for img in images -%} + {%- for fmt in img.formats -%} + {%- if source_link or not loop.first -%}, {% endif -%} + `{{ fmt }} <{{ dest_dir }}/{{ img.basename }}.{{ fmt }}>`__ + {%- endfor -%} + {%- endfor -%} + {%- endif -%} + ) + {% endif %} {{ only_latex }} {% for img in images %} {% if 'pdf' in img.formats -%} @@ -411,7 +423,6 @@ def _dummy_print(*arg, **kwarg): sys.stdout = stdout return ns - def get_wf_formats(config): default_dpi = {'png': 80, 'hires.png': 200, 'pdf': 200} formats = [] @@ -436,15 +447,14 @@ def get_wf_formats(config): def render_figures(code, code_path, output_dir, output_base, context, - function_name, config, context_reset=False, - close_figs=False): + function_name, config, graph2use, simple_form, + context_reset=False, close_figs=False): """ Run a nipype workflow creation script and save the graph in *output_dir*. Save the images under *output_dir* with file names derived from *output_base* """ formats = get_wf_formats(config) - ns = wf_context if context else {} if context_reset: wf_context.clear() @@ -457,11 +467,9 @@ def render_figures(code, code_path, output_dir, output_base, context, img_path = img.filename(fmt) imgname, ext = os.path.splitext(os.path.basename(img_path)) ns['wf'].base_dir = output_dir - ns['wf'].write_graph(imgname, format=ext[1:]) - - src = os.path.join(os.path.dirname(img_path), ns['wf'].name, - os.path.basename(img_path)) - print(src, img_path) + src = ns['wf'].write_graph(imgname, format=ext[1:], + graph2use=graph2use, + simple_form=simple_form) shutil.move(src, img_path) except Exception as err: raise GraphError(traceback.format_exc()) @@ -479,6 +487,9 @@ def run(arguments, content, options, state_machine, state, lineno): formats = get_wf_formats(config) default_fmt = formats[0][0] + graph2use = options.get('graph2use', 'hierarchical') + simple_form = options.get('simple_form', True) + options.setdefault('include-source', config.wf_include_source) keep_context = 'context' in options context_opt = None if not keep_context else options['context'] @@ -577,6 +588,8 @@ def run(arguments, content, options, state_machine, state, lineno): keep_context, function_name, config, + graph2use, + simple_form, context_reset=context_opt == 'reset', close_figs=context_opt == 'close-figs') errors = [] From b0b9d84b692a91df70a4c011a69e4eeb6a3f3e5f Mon Sep 17 00:00:00 2001 From: oesteban Date: Mon, 20 Mar 2017 23:52:00 -0700 Subject: [PATCH 3/9] update CHANGES --- CHANGES | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 970422b3b4..891df66bb6 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,7 @@ Upcoming Release ===================== +* ENH: Sphinx extension to plot workflows (https://github.com/nipy/nipype/pull/1896) * ENH: Added non-steady state detector for EPI data (https://github.com/nipy/nipype/pull/1839) * ENH: Enable new BBRegister init options for FSv6+ (https://github.com/nipy/nipype/pull/1811) * REF: Splits nipype.interfaces.utility into base, csv, and wrappers (https://github.com/nipy/nipype/pull/1828) @@ -22,7 +23,7 @@ Upcoming Release 0.13.0-rc1 (January 4, 2017) =============================== - + * FIX: Compatibility with traits 4.6 (https://github.com/nipy/nipype/pull/1770) * FIX: Multiproc deadlock (https://github.com/nipy/nipype/pull/1756) * TST: Replace nose and unittest with pytest (https://github.com/nipy/nipype/pull/1722, https://github.com/nipy/nipype/pull/1751) From 35c2e5db82c04491c7b5741995de31a914c6f051 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 21 Mar 2017 09:21:10 -0700 Subject: [PATCH 4/9] remove mriqc leftovers --- nipype/sphinxext/plot_workflow.py | 41 ++++++++++++++++++++++++------- nipype/utils/filemanip.py | 13 ++++++++++ 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/nipype/sphinxext/plot_workflow.py b/nipype/sphinxext/plot_workflow.py index 1d7aed5082..0aea845f01 100644 --- a/nipype/sphinxext/plot_workflow.py +++ b/nipype/sphinxext/plot_workflow.py @@ -3,23 +3,49 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: """ + + +:mod:`nipype.sphinxext.plot_workflow` -- Workflow plotting extension +==================================================================== + + A directive for including a nipype workflow graph in a Sphinx document. +This code is forked from the plot_figure sphinx extension of matplotlib. + By default, in HTML output, `workflow` will include a .png file with a link to a high-res .png. In LaTeX output, it will include a .pdf. The source code for the workflow may be included as **inline content** to -the directive:: +the directive `workflow`:: + + .. workflow: + :graph2use: flat + :simple_form: no + + from nipype.workflows.dmri.camino.connectivity_mapping import create_connectivity_pipeline + wf = create_connectivity_pipeline() - .. workflow:: - from mriqc.workflows.anatomical import airmsk_wf - wf = airmsk_wf() + +For example, the following graph has been generated inserting the previous +code block in this documentation: + +.. workflow: + :graph2use: flat + :simple_form: no + + from nipype.workflows.dmri.camino.connectivity_mapping import create_connectivity_pipeline + wf = create_connectivity_pipeline() Options ------- The ``workflow`` directive supports the following options: + graph2use : {'hierarchical', 'colored', 'flat', 'orig', 'exec'} + Specify the type of graph to be generated. + simple_form: bool + Whether the graph will be in detailed or simple form. format : {'python', 'doctest'} Specify the format of the input include-source : bool @@ -90,8 +116,7 @@ from docutils.parsers.rst import directives from docutils.parsers.rst.directives.images import Image -from mriqc.utils.misc import check_folder as mkdirs - +from nipype.utils.filemanip import mkdirp try: @@ -661,9 +686,7 @@ def run(arguments, content, options, state_machine, state, lineno): state_machine.insert_input(total_lines, source=source_file_name) # copy image files to builder's output directory, if necessary - if not os.path.exists(dest_dir): - mkdirs(dest_dir) - + mkdirp(dest_dir) for code_piece, images in results: for img in images: for fn in img.filenames(): diff --git a/nipype/utils/filemanip.py b/nipype/utils/filemanip.py index cbf32392eb..8ac4b5dda0 100644 --- a/nipype/utils/filemanip.py +++ b/nipype/utils/filemanip.py @@ -16,6 +16,7 @@ import hashlib from hashlib import md5 import os +from errno import EEXIST import re import shutil import posixpath @@ -425,6 +426,18 @@ def copyfiles(filelist, dest, copy=False, create_new=False): newfiles.insert(i, destfile) return newfiles +def mkdirp(folder): + """ + Equivalent to bash's mkdir -p + """ + if not os.path.exists(folder): + try: + os.makedirs(folder) + except OSError as exc: + if not exc.errno == EEXIST: + raise + return folder + def filename_to_list(filename): """Returns a list given either a string or a list From 5ed4683635a6bc04b8ea59100bd4e44379feacd4 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 21 Mar 2017 09:21:26 -0700 Subject: [PATCH 5/9] add documentation --- doc/documentation.rst | 2 ++ doc/users/index.rst | 2 ++ doc/users/sphinx_ext.rst | 15 +++++++++++++++ 3 files changed, 19 insertions(+) create mode 100644 doc/users/sphinx_ext.rst diff --git a/doc/documentation.rst b/doc/documentation.rst index 8ca2e503c8..3468525492 100644 --- a/doc/documentation.rst +++ b/doc/documentation.rst @@ -24,10 +24,12 @@ Previous versions: `0.12.0 `_ `0.11.0 :maxdepth: 2 users/index + .. toctree:: :maxdepth: 1 changes + * Developer .. toctree:: diff --git a/doc/users/index.rst b/doc/users/index.rst index c9cbcd961b..3c39ce08b2 100644 --- a/doc/users/index.rst +++ b/doc/users/index.rst @@ -39,6 +39,8 @@ mipav nipypecmd aws + resource_sched_profiler + sphinx_ext diff --git a/doc/users/sphinx_ext.rst b/doc/users/sphinx_ext.rst new file mode 100644 index 0000000000..11748287d0 --- /dev/null +++ b/doc/users/sphinx_ext.rst @@ -0,0 +1,15 @@ + +.. _sphinx_ext: + +Sphinx extensions +----------------- + + +To help users document their *Nipype*-based code, the software is shipped +with a set of extensions (currently only one) to customize the appearance +and simplify the generation process. + +.. automodule:: nipype.sphinxext.plot_workflow + :members: + :undoc-members: + :show-inheritance: From 88b3ec4bcf9ec117787cfc7437ae9b4385a3d697 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 21 Mar 2017 09:34:47 -0700 Subject: [PATCH 6/9] added resource profiling to user docs, example not working close #1526 --- doc/users/sphinx_ext.rst | 3 +-- nipype/sphinxext/plot_workflow.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/users/sphinx_ext.rst b/doc/users/sphinx_ext.rst index 11748287d0..6326a6041a 100644 --- a/doc/users/sphinx_ext.rst +++ b/doc/users/sphinx_ext.rst @@ -10,6 +10,5 @@ with a set of extensions (currently only one) to customize the appearance and simplify the generation process. .. automodule:: nipype.sphinxext.plot_workflow - :members: :undoc-members: - :show-inheritance: + :noindex: \ No newline at end of file diff --git a/nipype/sphinxext/plot_workflow.py b/nipype/sphinxext/plot_workflow.py index 0aea845f01..e50a4dfdd8 100644 --- a/nipype/sphinxext/plot_workflow.py +++ b/nipype/sphinxext/plot_workflow.py @@ -3,8 +3,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: """ - - :mod:`nipype.sphinxext.plot_workflow` -- Workflow plotting extension ==================================================================== @@ -64,6 +62,7 @@ Configuration options --------------------- + The workflow directive has the following configuration options: graph2use Select a graph type to use From 5e7fac9424e1af3562463da251ced0945585cdbf Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 21 Mar 2017 12:39:01 -0700 Subject: [PATCH 7/9] fix example of use of the new workflow directive --- nipype/sphinxext/plot_workflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nipype/sphinxext/plot_workflow.py b/nipype/sphinxext/plot_workflow.py index e50a4dfdd8..726c5df88c 100644 --- a/nipype/sphinxext/plot_workflow.py +++ b/nipype/sphinxext/plot_workflow.py @@ -17,7 +17,7 @@ The source code for the workflow may be included as **inline content** to the directive `workflow`:: - .. workflow: + .. workflow :: :graph2use: flat :simple_form: no @@ -28,7 +28,7 @@ For example, the following graph has been generated inserting the previous code block in this documentation: -.. workflow: +.. workflow :: :graph2use: flat :simple_form: no From e173327b256dcdd5293e64129920b394f749d4c3 Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 22 Mar 2017 10:44:25 -0700 Subject: [PATCH 8/9] check that folder is a folder in mkdirp --- nipype/utils/filemanip.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nipype/utils/filemanip.py b/nipype/utils/filemanip.py index 8ac4b5dda0..3d31ac66fd 100644 --- a/nipype/utils/filemanip.py +++ b/nipype/utils/filemanip.py @@ -430,12 +430,12 @@ def mkdirp(folder): """ Equivalent to bash's mkdir -p """ - if not os.path.exists(folder): - try: - os.makedirs(folder) - except OSError as exc: - if not exc.errno == EEXIST: - raise + try: + os.makedirs(folder) + except OSError as exc: + if exc.errno != EEXIST or not os.path.isdir(folder): + raise + return folder From 1f6335bdcc6ced28593ad4c4b2d823453d062662 Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 22 Mar 2017 19:13:45 -0700 Subject: [PATCH 9/9] make mkdirp private --- nipype/sphinxext/plot_workflow.py | 21 ++++++++++++++++++--- nipype/utils/filemanip.py | 13 ------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/nipype/sphinxext/plot_workflow.py b/nipype/sphinxext/plot_workflow.py index 726c5df88c..1ed10aef42 100644 --- a/nipype/sphinxext/plot_workflow.py +++ b/nipype/sphinxext/plot_workflow.py @@ -110,13 +110,12 @@ import sys, os, shutil, io, re, textwrap from os.path import relpath +from errno import EEXIST import traceback from docutils.parsers.rst import directives from docutils.parsers.rst.directives.images import Image -from nipype.utils.filemanip import mkdirp - try: # Sphinx depends on either Jinja or Jinja2 @@ -134,6 +133,22 @@ def format_template(template, **kw): PY2 = sys.version_info[0] == 2 PY3 = sys.version_info[0] == 3 +def _mkdirp(folder): + """ + Equivalent to bash's mkdir -p + """ + if sys.version_info > (3, 4, 1): + os.makedirs(folder, exist_ok=True) + return folder + + try: + os.makedirs(folder) + except OSError as exc: + if exc.errno != EEXIST or not os.path.isdir(folder): + raise + + return folder + def wf_directive(name, arguments, options, content, lineno, content_offset, block_text, state, state_machine): @@ -685,7 +700,7 @@ def run(arguments, content, options, state_machine, state, lineno): state_machine.insert_input(total_lines, source=source_file_name) # copy image files to builder's output directory, if necessary - mkdirp(dest_dir) + _mkdirp(dest_dir) for code_piece, images in results: for img in images: for fn in img.filenames(): diff --git a/nipype/utils/filemanip.py b/nipype/utils/filemanip.py index 3d31ac66fd..cbf32392eb 100644 --- a/nipype/utils/filemanip.py +++ b/nipype/utils/filemanip.py @@ -16,7 +16,6 @@ import hashlib from hashlib import md5 import os -from errno import EEXIST import re import shutil import posixpath @@ -426,18 +425,6 @@ def copyfiles(filelist, dest, copy=False, create_new=False): newfiles.insert(i, destfile) return newfiles -def mkdirp(folder): - """ - Equivalent to bash's mkdir -p - """ - try: - os.makedirs(folder) - except OSError as exc: - if exc.errno != EEXIST or not os.path.isdir(folder): - raise - - return folder - def filename_to_list(filename): """Returns a list given either a string or a list