added comments and doc strings to explain the use of the persistent cache

Author: tclose

pydra/utils/hash.py

@@ -224,13 +224,32 @@ def hash_single(obj: object, cache: Cache) -> Hash:
         # chunk
 
         def calc_hash(first: ty.Optional[bytes] = None) -> Hash:
+            """
+            Calculate the hash of the object
+
+            Parameters
+            ----------
+            first : ty.Optional[bytes]
+                the first bytes chunk from the bytes_repr iterator, passed if the first
+                chunk wasn't a local cache key
+            """
             h = blake2b(digest_size=16, person=b"pydra-hash")
+            # We want to use the first chunk that was popped to check for a cache-key
+            # if present
             if first is not None:
                 h.update(first)
             for chunk in bytes_it:  # Note that `bytes_it` is in outer scope
                 h.update(chunk)
             return Hash(h.digest())
 
+        # Read the first chunk of the bytes_repr iterator, check to see whether it returns
+        # a "cache-key" tuple instead of a bytes chunk for the type of the object to cache
+        # (i.e. file objects). If it does use that key to check the persistent cache for
+        # a precomputed hash and otherwise calculate the hash and store it in the
+        # persistent cache with that key.
+
+        # If the first chunk is a bytes chunk (i.e. the object type doesn't have an associated
+        # 'cache-key'), then simply calculate the hash of the object.
         first = next(bytes_it)
         if isinstance(first, tuple):
             tp = type(obj)

pydra/utils/tests/test_hash.py

@@ -323,11 +323,18 @@ def text_file(tmp_path):
 
 
 def test_persistent_hash_cache(cache_path, text_file):
+    """
+    Test the persistent hash cache with a text file
+
+    The cache is used to store the hash of the text file, and the hash is
+    retrieved from the cache when the file is unchanged.
+    """
     # Test hash is stable between calls
     hsh = hash_object(text_file, persistent_cache=cache_path)
     assert hsh == hash_object(text_file, persistent_cache=cache_path)
 
-    # Test that cached hash has been used
+    # Test that cached hash has been used by explicitly modifying it and seeing that the
+    # hash is the same as the modified hash
     cache_files = list(cache_path.iterdir())
     assert len(cache_files) == 1
     modified_hash = "modified".encode()
@@ -342,6 +349,10 @@ def test_persistent_hash_cache(cache_path, text_file):
 
 
 def test_persistent_hash_cache_cleanup1(cache_path, text_file):
+    """
+    Test the persistent hash is cleaned up after use if the periods between cleanups
+    is greater than the environment variable PYDRA_HASH_CACHE_CLEANUP_PERIOD
+    """
     with mock.patch.dict(
         os.environ,
         {
@@ -350,21 +361,28 @@ def test_persistent_hash_cache_cleanup1(cache_path, text_file):
         },
     ):
         persistent_cache = PersistentCache()
-    hsh = hash_object(text_file, persistent_cache=persistent_cache)
+    hash_object(text_file, persistent_cache=persistent_cache)
     assert len(list(cache_path.iterdir())) == 1
     persistent_cache.clean_up()
     assert len(list(cache_path.iterdir())) == 0
 
 
 def test_persistent_hash_cache_cleanup2(cache_path, text_file):
+    """
+    Test the persistent hash is cleaned up after use if the periods between cleanups
+    is greater than the explicitly provided cleanup_period
+    """
     persistent_cache = PersistentCache(cache_path, cleanup_period=-100)
-    hsh = hash_object(text_file, persistent_cache=persistent_cache)
+    hash_object(text_file, persistent_cache=persistent_cache)
     assert len(list(cache_path.iterdir())) == 1
     time.sleep(2)
     persistent_cache.clean_up()
     assert len(list(cache_path.iterdir())) == 0
 
 
 def test_persistent_hash_cache_not_dir(text_file):
+    """
+    Test that an error is raised if the provided cache path is not a directory
+    """
     with pytest.raises(ValueError, match="is not a directory"):
         PersistentCache(text_file.fspath)