update pydra.engine.helpers_file.hash_dir to return single sha string

Author: kaczmarj

pydra/engine/helpers_file.py

@@ -87,27 +87,60 @@ def hash_file(afile, chunk_len=8192, crypto=sha256, raise_notfound=True):
     return crypto_obj.hexdigest()
 
 
-def hash_dir(dirpath, raise_notfound=True):
+def hash_dir(
+    dirpath, ignore_hidden_files=False, ignore_hidden_dirs=False, raise_notfound=True
+):
+    """Compute hash of directory contents.
+
+    This function computes the hash of every file in directory `dirpath` and then
+    computes the hash of that list of hashes to return a single hash value. The
+    directory is traversed recursively.
+
+    Parameters
+    ----------
+    dirpath : :obj:`str`
+        Path to directory.
+    ignore_hidden_files : :obj:`bool`
+        If `True`, ignore filenames that begin with `.`.
+    ignore_hidden_dirs : :obj:`bool`
+        If `True`, ignore files in directories that begin with `.`.
+    raise_notfound : :obj:`bool`
+        If `True` and `dirpath` does not exist, raise `FileNotFound` exception. If
+        `False` and `dirpath` does not exist, return `None`.
+
+    Returns
+    -------
+    hash : :obj:`str`
+        Hash of the directory contents.
+    """
     from .specs import LazyField
 
     if dirpath is None or isinstance(dirpath, LazyField) or isinstance(dirpath, list):
         return None
     if not Path(dirpath).is_dir():
         if raise_notfound:
-            raise RuntimeError(f"Directory {dirpath} not found.")
+            raise FileNotFoundError(f"Directory {dirpath} not found.")
         return None
 
-    def search_dir(path):
-        path = Path(path)
-        file_list = []
-        for el in path.iterdir():
-            if el.is_file():
-                file_list.append(hash_file(el))
-            else:
-                file_list.append(search_dir(path / el))
-        return file_list
+    file_hashes = []
+    for dpath, dirnames, filenames in os.walk(dirpath):
+        # Sort in-place to guarantee order.
+        dirnames.sort()
+        filenames.sort()
+        dpath = Path(dpath)
+        if ignore_hidden_dirs and dpath.name.startswith(".") and str(dpath) != dirpath:
+            continue
+        for filename in filenames:
+            if ignore_hidden_files and filename.startswith("."):
+                continue
+            this_hash = hash_file(dpath / filename)
+            file_hashes.append(this_hash)
+
+    sha = sha256()
+    for h in file_hashes:
+        sha.update(h.encode())
 
-    return search_dir(dirpath)
+    return sha.hexdigest()
 
 
 def _parse_mount_table(exit_code, output):

pydra/engine/tests/test_helpers.py

@@ -1,4 +1,6 @@
+import hashlib
 from pathlib import Path
+import random
 
 import pytest
 import cloudpickle as cp
@@ -135,25 +137,49 @@ def test_hash_value_dir(tmpdir):
     with open(file_2, "w") as f:
         f.write("hi")
 
-    assert sorted(hash_value(tmpdir, tp=Directory)) == sorted(
-        hash_value([file_1, file_2], tp=File)
-    )
-    assert hash_value(tmpdir, tp=Directory) == helpers_file.hash_dir(tmpdir)
+    test_sha = hashlib.sha256()
+    for fx in [file_1, file_2]:
+        test_sha.update(helpers_file.hash_file(fx).encode())
+
+    bad_sha = hashlib.sha256()
+    for fx in [file_2, file_1]:
+        bad_sha.update(helpers_file.hash_file(fx).encode())
+
+    orig_hash = helpers_file.hash_dir(tmpdir)
+
+    assert orig_hash == test_sha.hexdigest()
+    assert orig_hash != bad_sha.hexdigest()
+    assert orig_hash == hash_value(tmpdir, tp=Directory)
 
 
 def test_hash_value_nested(tmpdir):
+    hidden = tmpdir.mkdir(".hidden")
     nested = tmpdir.mkdir("nested")
     file_1 = tmpdir.join("file_1.txt")
-    file_2 = nested.join("file_2.txt")
-    file_3 = nested.join("file_3.txt")
-    with open(file_1, "w") as f:
-        f.write("hello")
-    with open(file_2, "w") as f:
-        f.write("hi")
-    with open(file_3, "w") as f:
-        f.write("hola")
+    file_2 = hidden.join("file_2.txt")
+    file_3 = nested.join(".file_3.txt")
+    file_4 = nested.join("file_4.txt")
 
-    assert hash_value(tmpdir, tp=Directory) == hash_value(
-        [file_1, [file_2, file_3]], tp=File
-    )
-    assert hash_value(tmpdir, tp=Directory) == helpers_file.hash_dir(tmpdir)
+    test_sha = hashlib.sha256()
+    for fx in [file_1, file_2, file_3, file_4]:
+        with open(fx, "w") as f:
+            f.write(str(random.randint(0, 1000)))
+        test_sha.update(helpers_file.hash_file(fx).encode())
+
+    orig_hash = helpers_file.hash_dir(tmpdir)
+
+    assert orig_hash == test_sha.hexdigest()
+    assert orig_hash == hash_value(tmpdir, tp=Directory)
+
+    nohidden_hash = helpers_file.hash_dir(tmpdir, ignore_hidden_dirs=True, ignore_hidden_files=True)
+    nohiddendirs_hash = helpers_file.hash_dir(tmpdir, ignore_hidden_dirs=True)
+    nohiddenfiles_hash = helpers_file.hash_dir(tmpdir, ignore_hidden_files=True)
+
+    assert orig_hash != nohidden_hash
+    assert orig_hash != nohiddendirs_hash
+    assert orig_hash != nohiddenfiles_hash
+
+    file_3.remove()
+    assert helpers_file.hash_dir(tmpdir) == nohiddenfiles_hash
+    hidden.remove()
+    assert helpers_file.hash_dir(tmpdir) == nohidden_hash