Merge pull request #451 from satra/enh/provenance

WIP: Enh/provenance

Author: satra

.travis.yml

@@ -20,5 +20,12 @@ before_install:
     - source /etc/fsl/fsl.sh
     - virtualenv --system-site-packages ~/virtualenv/this
     - source ~/virtualenv/this/bin/activate
-install: python setup.py build_ext --inplace
-script: make test
+    - pip install https://github.com/RDFLib/rdflib/archive/master.zip
+    - pip install https://github.com/satra/prov/archive/enh/rdf.zip
+install:
+    - source ~/virtualenv/this/bin/activate
+    - python setup.py build_ext --inplace
+script:
+    - source ~/virtualenv/this/bin/activate
+    - make test
+

CHANGES

@@ -3,6 +3,7 @@ Next release
 
 * ENH: SelectFiles: a streamlined version of DataGrabber
 * ENH: New interfaces: spm.ResliceToReference
+* ENH: W3C PROV support with optional RDF export built into Nipype
 
 * FIX: Deals properly with 3d files in SPM Realign
 

doc/devel/index.rst

@@ -17,6 +17,7 @@ conventions documented in the `NIPY Developers Guide
    writing_custom_interfaces
    gitwash/index   
    architecture
+   provenance
 
 
 .. include:: ../links_names.txt

doc/devel/provenance.rst

@@ -0,0 +1,21 @@
+================
+W3C PROV support
+================
+
+Overview
+--------
+
+We're using the the `W3C PROV data model <http://www.w3.org/TR/prov-dm/`_ to
+capture and represent provenance in Nipype.
+
+For an overview see:
+
+`PROV-DM overview <http://slideviewer.herokuapp.com/url/raw.github.com/ni-/notebooks/master/NIDMIntro.ipynb>`_
+
+Each interface writes out a provenance.json (currently prov-json) or
+provenance.rdf (if rdflib is available) file. The workflow engine can also
+write out a provenance of the workflow if instructed.
+
+This is very much an experimental feature as we continue to refine how exactly
+the provenance should be stored and how such information can be used for
+reporting or reconstituting workflows.

doc/sphinxext/autosummary_generate.py

@@ -54,7 +54,7 @@ def main():
     for name, path in sorted(names.items()):
         if options.output_dir is not None:
             path = options.output_dir
-        
+
         if not os.path.isdir(path):
             os.makedirs(path)
 
@@ -121,7 +121,7 @@ def get_documented_in_docstring(name, module=None, filename=None):
     """
     Find out what items are documented in the given object's docstring.
     See `get_documented_in_lines`.
-    
+
     """
     try:
         obj, real_name = import_by_name(name)
@@ -136,7 +136,7 @@ def get_documented_in_docstring(name, module=None, filename=None):
 def get_documented_in_lines(lines, module=None, filename=None):
     """
     Find out what items are documented in the given lines
-    
+
     Returns
     -------
     documented : dict of list of (filename, title, keyword, toctree)
@@ -153,15 +153,15 @@ def get_documented_in_lines(lines, module=None, filename=None):
     module_re = re.compile(r'^\.\.\s+(current)?module::\s*([a-zA-Z0-9_.]+)\s*$')
     autosummary_item_re = re.compile(r'^\s+([_a-zA-Z][a-zA-Z0-9_.]*)\s*.*?')
     toctree_arg_re = re.compile(r'^\s+:toctree:\s*(.*?)\s*$')
-    
+
     documented = {}
-   
+
     current_title = []
     last_line = None
     toctree = None
     current_module = module
     in_autosummary = False
-    
+
     for line in lines:
         try:
             if in_autosummary:

doc/sphinxext/inheritance_diagram.py

@@ -48,7 +48,7 @@ def my_import(name):
     """Module importer - taken from the python documentation.
 
     This function allows importing names with dots in them."""
-    
+
     mod = __import__(name)
     components = name.split('.')
     for comp in components[1:]:
@@ -103,7 +103,7 @@ def _import_class_or_module(self, name):
             # second call will force the equivalent of 'import a.b' to happen
             # after the top-level import above.
             my_import(fullname)
-            
+
         except ImportError:
             raise ValueError(
                 "Could not import class or module '%s' specified for inheritance diagram" % name)

doc/sphinxext/ipython_console_highlighting.py

@@ -11,7 +11,7 @@
 
 # Third party
 from pygments.lexer import Lexer, do_insertions
-from pygments.lexers.agile import (PythonConsoleLexer, PythonLexer, 
+from pygments.lexers.agile import (PythonConsoleLexer, PythonLexer,
                                    PythonTracebackLexer)
 from pygments.token import Comment, Generic
 
@@ -47,7 +47,7 @@ class IPythonConsoleLexer(Lexer):
 
       - It assumes the default IPython prompts, not customized ones.
     """
-    
+
     name = 'IPython console session'
     aliases = ['ipython']
     mimetypes = ['text/x-ipython-console']

doc/users/install.rst

@@ -112,20 +112,20 @@ recommendations.
 Must Have
 ~~~~~~~~~
 
-Python_ 2.6 -2.7
+Python_ 2.6 - 2.7
 
-Nibabel_ 1.0 - 1.2
+Nibabel_ 1.0 - 1.4
   Neuroimaging file i/o library
 
-NetworkX_ 1.0 - 1.4
+NetworkX_ 1.0 - 1.8
   Python package for working with complex networks.
 
-NumPy_ 1.3 - 1.6
+NumPy_ 1.3 - 1.7
 
-SciPy_ 0.7 - 0.10
+SciPy_ 0.7 - 0.12
   Numpy and Scipy are high-level, optimized scientific computing libraries.
 
-Enthought_ Traits_ 4.0.0
+Enthought_ Traits_ 4.0.0 - 4.3.0
 
 .. note::
 
@@ -135,14 +135,17 @@ Enthought_ Traits_ 4.0.0
 Strong Recommendations
 ~~~~~~~~~~~~~~~~~~~~~~
 
-IPython_ 0.10.2 - 0.13
+IPython_ 0.10.2 - 1.0.0
   Interactive python environment. This is necessary for some parallel
   components of the pipeline engine.
 
-Matplotlib_ 1.0
+Matplotlib_ 1.0 - 1.2
   Plotting library
 
-Sphinx_
+`RDFLib <http://rdflib.readthedocs.org/en/latest/>`_ 4.1
+RDFLibrary required for provenance export as RDF
+
+Sphinx_ 1.1
   Required for building the documentation
 
 `Graphviz <http://www.graphviz.org/>`_

examples/fmri_fsl.py

@@ -633,7 +633,7 @@ def subjectinfo(subject_id):
 
 if __name__ == '__main__':
     l1pipeline.write_graph()
-    l1pipeline.run()
+    outgraph = l1pipeline.run()
     #l1pipeline.run(plugin='MultiProc', plugin_args={'n_procs':2})
 
 

examples/fmri_openfmri.py

@@ -387,12 +387,13 @@ def get_subs(subject_id, conds, model_id, task_id):
                                                              'crashdumps'),
                                   stop_on_first_crash=True)
     #wf.run('MultiProc', plugin_args={'n_procs': 4})
-    wf.run('Linear', plugin_args={'n_procs': 4})
+    eg = wf.run('Linear')
     wf.export('openfmri.py')
     wf.write_graph(dotfilename='hgraph.dot', graph2use='hierarchical')
     wf.write_graph(dotfilename='egraph.dot', graph2use='exec')
     wf.write_graph(dotfilename='fgraph.dot', graph2use='flat')
     wf.write_graph(dotfilename='ograph.dot', graph2use='orig')
+    return eg
 
 if __name__ == '__main__':
     import argparse
@@ -403,7 +404,9 @@ def get_subs(subject_id, conds, model_id, task_id):
     parser.add_argument('-m', '--model', default=1)
     parser.add_argument('-t', '--task', default=1)
     args = parser.parse_args()
-    analyze_openfmri_dataset(data_dir=os.path.abspath(args.datasetdir),
+    eg = analyze_openfmri_dataset(data_dir=os.path.abspath(args.datasetdir),
                              subject=args.subject,
                              model_id=int(args.model),
                              task_id=int(args.task))
+    from nipype.pipeline.utils import write_prov
+    g = write_prov(eg, format='turtle')