[WIP,FIX] Bugs found after #1572 and #1591

Found corner cases that failed when calculating the hash of dicts - fixes #1620.

Also found that list of lists of ints do not work in #1591. Fixes also that one.

Author: oesteban

examples/smri_ants_registration.py

@@ -24,6 +24,7 @@
 import urllib.error
 import urllib.parse
 from nipype.interfaces.ants import Registration
+from nipype.testing import example_data
 
 """
 2. Download T1 volumes into home directory
@@ -61,11 +62,12 @@
 
 """
 3. Define the parameters of the registration. Settings are
-saved in ``smri_ants_registration_settings.json``.
+found in the file ``smri_ants_registration_settings.json``
+distributed with the ``example_data`` of `nipype`.
 
 """
 
-reg = Registration(from_file='./smri_ants_registration_settings.json')
+reg = Registration(from_file=example_data('smri_ants_registration_settings.json'))
 reg.inputs.fixed_image = input_images[0]
 reg.inputs.moving_image = input_images[1]
 

nipype/interfaces/base.py

@@ -536,6 +536,14 @@ def _clean_container(self, object, undefinedval=None, skipundefined=False):
                     out = undefinedval
         return out
 
+    def has_metadata(self, name, metadata, value=None, recursive=True):
+        """
+        Return has_metadata for the requested trait name in this
+        interface
+        """
+        return has_metadata(self.trait(name).trait_type, metadata, value,
+                            recursive)
+
     def get_hashval(self, hash_method=None):
         """Return a dictionary of our items with hashes for each file.
 
@@ -559,62 +567,61 @@ def get_hashval(self, hash_method=None):
 
         dict_withhash = []
         dict_nofilename = []
-        for name, val in sorted(self.get().items()):
-            if isdefined(val):
-                trait = self.trait(name)
-                if has_metadata(trait.trait_type, "nohash", True):
-                    continue
-                hash_files = (not has_metadata(trait.trait_type, "hash_files",
-                                               False) and not
-                              has_metadata(trait.trait_type, "name_source"))
-                dict_nofilename.append((name,
-                                        self._get_sorteddict(val, hash_method=hash_method,
-                                                             hash_files=hash_files)))
-                dict_withhash.append((name,
-                                      self._get_sorteddict(val, True, hash_method=hash_method,
-                                                           hash_files=hash_files)))
+
+        for name, val in sorted(self.get_traitsfree().items()):
+            if self.has_metadata(name, "nohash", True):
+                continue
+
+            hash_files = (not has_metadata(name, "hash_files", False) and not
+                          has_metadata(name, "name_source"))
+            dict_nofilename.append((name,
+                                    self._get_sorteddict(val, hash_method=hash_method,
+                                                         hash_files=hash_files)))
+            dict_withhash.append((name,
+                                  self._get_sorteddict(val, True, hash_method=hash_method,
+                                                       hash_files=hash_files)))
         return dict_withhash, md5(encode_dict(dict_nofilename).encode()).hexdigest()
 
-    def _get_sorteddict(self, object, dictwithhash=False, hash_method=None,
+    def _get_sorteddict(self, theobject, dictwithhash=False, hash_method=None,
                         hash_files=True):
-        if isinstance(object, dict):
+        if isinstance(theobject, dict):
             out = []
-            for key, val in sorted(object.items()):
+            for key, val in sorted(theobject.items()):
                 if isdefined(val):
                     out.append((key,
                                 self._get_sorteddict(val, dictwithhash,
                                                      hash_method=hash_method,
                                                      hash_files=hash_files)))
-        elif isinstance(object, (list, tuple)):
+        elif isinstance(theobject, (list, tuple)):
             out = []
-            for val in object:
+            for val in theobject:
                 if isdefined(val):
                     out.append(self._get_sorteddict(val, dictwithhash,
                                                     hash_method=hash_method,
                                                     hash_files=hash_files))
-            if isinstance(object, tuple):
+            if isinstance(theobject, tuple):
                 out = tuple(out)
         else:
-            if isdefined(object):
-                if (hash_files and isinstance(object, (str, bytes)) and
-                        os.path.isfile(object)):
+            if isdefined(theobject):
+                if (hash_files and isinstance(theobject, (str, bytes)) and
+                        os.path.isfile(theobject)):
                     if hash_method is None:
                         hash_method = config.get('execution', 'hash_method')
 
                     if hash_method.lower() == 'timestamp':
-                        hash = hash_timestamp(object)
+                        hash = hash_timestamp(theobject)
                     elif hash_method.lower() == 'content':
-                        hash = hash_infile(object)
+                        hash = hash_infile(theobject)
                     else:
                         raise Exception("Unknown hash method: %s" % hash_method)
                     if dictwithhash:
-                        out = (object, hash)
+                        out = (theobject, hash)
                     else:
                         out = hash
-                elif isinstance(object, float):
-                    out = '%.10f' % object
+                elif isinstance(theobject, float):
+                    out = '%.10f' % theobject
                 else:
-                    out = object
+                    out = theobject
         return out
 
 

nipype/interfaces/tests/test_base.py

@@ -13,7 +13,7 @@
 
 from nipype.testing import (assert_equal, assert_not_equal, assert_raises,
                             assert_true, assert_false, with_setup, package_check,
-                            skipif)
+                            skipif, example_data)
 import nipype.interfaces.base as nib
 from nipype.utils.filemanip import split_filename
 from nipype.interfaces.base import Undefined, config
@@ -497,6 +497,13 @@ def __init__(self, **inputs):
     bif6.load_inputs_from_json(tmp_json)
     yield assert_equal, bif6.inputs.get_traitsfree(), inputs_dict
 
+    # test get hashval in a complex interface
+    from nipype.interfaces.ants import Registration
+    tsthash = Registration(from_file=example_data('smri_ants_registration_settings.json'))
+    hashed_inputs, hashvalue = tsthash.inputs.get_hashval(hash_method='timestamp')
+    # yield assert_equal, hashed_inputs, [('input1', 12), ('input3', True), ('input4', 'some string')]
+    yield assert_equal, hashvalue, '9ab944cbccba61475becb9eac65052af'
+
 def assert_not_raises(fn, *args, **kwargs):
     fn(*args, **kwargs)
     return True

examples/smri_ants_registration_settings.json renamed to nipype/testing/data/smri_ants_registration_settings.json

@@ -94,18 +94,16 @@ def encode_dict(value):
     if isinstance(value, str):
         value = value.encode()
 
-    if isinstance(value, tuple):
-        val0 = encode_dict(value[0])
-        val1 = encode_dict(value[1])
-        return '(' + val0 + ', ' + val1 + ')'
-
-    if isinstance(value, list):
-        retval = '['
+    istuple = isinstance(value, tuple)
+    if isinstance(value, list) or istuple:
+        retval = '(' if istuple else '['
         for i, v in enumerate(value):
             if i > 0:
                 retval += ', '
             retval += encode_dict(v)
-        return retval + ']'
+        retval += ')' if istuple else ']'
+        return retval
+
     return repr(value)
 
 def fname_presuffix(fname, prefix='', suffix='', newpath=None, use_ext=True):

nipype/utils/filemanip.py

@@ -10,11 +10,11 @@
 
 from ...testing import assert_equal, assert_true, assert_false, TempFATFS
 from ...utils.filemanip import (save_json, load_json,
-                                    fname_presuffix, fnames_presuffix,
-                                    hash_rename, check_forhash,
-                                    copyfile, copyfiles,
-                                    filename_to_list, list_to_filename,
-                                    split_filename, get_related_files)
+                                fname_presuffix, fnames_presuffix,
+                                hash_rename, check_forhash,
+                                copyfile, copyfiles,
+                                filename_to_list, list_to_filename,
+                                split_filename, get_related_files)
 
 import numpy as np