Merge pull request #1444 from satra/fix/prov

fix: cleaned up provenance generation

Author: satra

doc/_templates/indexsidebar.html

@@ -2,7 +2,7 @@
 <h3>{{ _('Links') }}</h3>
 
 <ul>
-    <li>Docs: <a href="http://nipy.org/nipype">Stable</a> · <a href="http://www.mit.edu/~satra/nipype-nightly/">Nightly</a></li>
+    <li>Docs: <a href="http://nipy.org/nipype">Stable</a> · <a href="http://nipype.readthedocs.org/en/latest/">Dev</a></li>
     <li>Code: <a href="http://github.com/nipy/nipype">Github</a> · <a href="http://github.com/nipy/nipype/issues">Bugs-Requests</a></li>
     <li>Forum: <a href="http://neurostars.org/t/nipype">User</a> · <a href="http://projects.scipy.org/mailman/listinfo/nipy-devel">Developer</a></li>
     <li><a href="http://nipy.org/software/license/index.html"><img src="https://img.shields.io/pypi/l/nipype.svg" alt="License"></a> · <a href="about.html#funding">Funding</a></li>

nipype/pipeline/engine/utils.py

@@ -1269,11 +1269,7 @@ def write_workflow_prov(graph, filename=None, format='all'):
                           starter=processes[nodes.index(edgeinfo[0])])
 
     # write provenance
-    if format in ['provn', 'all']:
-        with open(filename + '.provn', 'wt') as fp:
-            fp.writelines(ps.g.get_provn())
-    if format in ['json', 'all']:
-        ps.g.serialize(filename + '.json', format='json')
+    ps.write_provenance(filename, format=format)
     return ps.g
 
 

nipype/utils/provenance.py

@@ -21,7 +21,7 @@
 
 from .. import get_info
 from .filemanip import (md5, hashlib, hash_infile)
-from .. import logging
+from .. import logging, __version__
 iflogger = logging.getLogger('interface')
 
 foaf = pm.Namespace("foaf", "http://xmlns.com/foaf/0.1/")
@@ -185,7 +185,7 @@ def safe_encode(x, as_literal=True):
             return dumps(x)
         return pm.Literal(dumps(x), nipype_ns['pickle'])
     except TypeError as e:
-        iflogger.info(e)
+        iflogger.debug(e)
         value = "Could not encode: " + str(e)
         if not as_literal:
             return value
@@ -239,7 +239,7 @@ def prov_encode(graph, value, create_container=True):
     return entity
 
 
-def write_provenance(results, filename='provenance', format='turtle'):
+def write_provenance(results, filename='provenance', format='all'):
     ps = ProvStore()
     ps.add_results(results)
     return ps.write_provenance(filename=filename, format=format)
@@ -371,7 +371,8 @@ def add_results(self, results, keep_provenance=False):
         user_agent = self.g.agent(get_attr_id(user_attr), user_attr)
         agent_attr = {pm.PROV["type"]: pm.PROV["SoftwareAgent"],
                       pm.PROV["label"]: "Nipype",
-                      foaf["name"]: safe_encode("Nipype")}
+                      foaf["name"]: safe_encode("Nipype"),
+                      nipype_ns["version"]: __version__}
         for key, value in list(get_info().items()):
             agent_attr.update({nipype_ns[key]: safe_encode(value)})
         software_agent = self.g.agent(get_attr_id(agent_attr), agent_attr)
@@ -384,6 +385,17 @@ def write_provenance(self, filename='provenance', format='all'):
         if format in ['provn', 'all']:
             with open(filename + '.provn', 'wt') as fp:
                 fp.writelines(self.g.get_provn())
-        if format in ['json', 'all']:
-            g.serialize(filename + '.json', format='json')
+        try:
+            if format in ['rdf', 'all']:
+                if len(self.g.bundles) == 0:
+                    rdf_format = 'turtle'
+                    ext = '.ttl'
+                else:
+                    rdf_format = 'trig'
+                    ext = '.trig'
+                self.g.serialize(filename + ext, format='rdf', rdf_format=rdf_format)
+            if format in ['jsonld']:
+                self.g.serialize(filename + '.jsonld', format='rdf', rdf_format='json-ld', indent=4)
+        except pm.serializers.DoNotExist:
+            pass
         return self.g

nipype/utils/tests/test_filemanip.py

@@ -6,8 +6,8 @@
 from tempfile import mkstemp, mkdtemp
 import warnings
 
-from nipype.testing import assert_equal, assert_true, assert_false, TempFATFS
-from nipype.utils.filemanip import (save_json, load_json,
+from ...testing import assert_equal, assert_true, assert_false, TempFATFS
+from ...utils.filemanip import (save_json, load_json,
                                     fname_presuffix, fnames_presuffix,
                                     hash_rename, check_forhash,
                                     copyfile, copyfiles,
@@ -176,7 +176,7 @@ def test_copyfallback():
     pth, hdrname = os.path.split(orig_hdr)
     try:
         fatfs = TempFATFS()
-    except IOError:
+    except (IOError, OSError):
         warnings.warn('Fuse mount failed. copyfile fallback tests skipped.')
     else:
         with fatfs as fatdir:

nipype/utils/tests/test_provenance.py

@@ -4,13 +4,13 @@
 standard_library.install_aliases()
 from builtins import str
 
-import sys
+import os
+from tempfile import mkdtemp
 
 from ...testing import assert_equal, assert_true, assert_false
 
 from ..provenance import ProvStore, safe_encode, text_type
 
-
 def test_provenance():
     ps = ProvStore()
     from ...interfaces.base import CommandLine
@@ -20,6 +20,22 @@ def test_provenance():
     prov_json = ps.g.serialize(format='json')
     yield assert_true, 'echo hello' in provn
 
+def test_provenance_exists():
+    tempdir = mkdtemp()
+    cwd = os.getcwd()
+    os.chdir(tempdir)
+    from ...interfaces.base import CommandLine
+    from ... import config
+    provenance_state = config.get('execution', 'write_provenance')
+    hash_state = config.get('execution', 'hash_method')
+    config.enable_provenance()
+    CommandLine('echo hello').run()
+    config.set('execution', 'write_provenance', provenance_state)
+    config.set('execution', 'hash_method', hash_state)
+    provenance_exists = os.path.exists(os.path.join(tempdir, 'provenance.provn'))
+    os.chdir(cwd)
+    yield assert_true, provenance_exists
+
 def test_safe_encode():
     a = '\xc3\xa9lg'
     out = safe_encode(a)