fix: use cache for snapshots even if refs does not exist (#1306)

* fix: use cache for snapshots even if refs does not exist

If one uses `hf_hub_download` only referencing specific commits the `refs` folder will not be created even though data will be cached via `snapshots` and `blobs`.  Subsequent calls to `try_to_load_from_cache` were returning None even though the desired data was in the cache.

Example:

```python
# download something
hf_hub_download(repo_id=repo, revision=commit_hash, filename=filepath, token=token)
# returns None
try_to_load_from_cache(repo_id=repo, revision=commit_hash, filename=filepath)
```

* FIX in case .not_exists + add tests

Co-authored-by: Lucain Pouget <[email protected]>

Author: brycedrennan

src/huggingface_hub/file_download.py

@@ -1345,25 +1345,32 @@ def try_to_load_from_cache(
     if not os.path.isdir(repo_cache):
         # No cache for this model
         return None
-    for subfolder in ["refs", "snapshots"]:
-        if not os.path.isdir(os.path.join(repo_cache, subfolder)):
-            return None
 
-    # Resolve refs (for instance to convert main to the associated commit sha)
-    cached_refs = os.listdir(os.path.join(repo_cache, "refs"))
-    if revision in cached_refs:
-        with open(os.path.join(repo_cache, "refs", revision)) as f:
-            revision = f.read()
+    refs_dir = os.path.join(repo_cache, "refs")
+    snapshots_dir = os.path.join(repo_cache, "snapshots")
+    no_exists_dir = os.path.join(repo_cache, ".no_exist")
 
-    if os.path.isfile(os.path.join(repo_cache, ".no_exist", revision, filename)):
+    # Resolve refs (for instance to convert main to the associated commit sha)
+    if os.path.isdir(refs_dir):
+        cached_refs = os.listdir(refs_dir)
+        if revision in cached_refs:
+            with open(os.path.join(refs_dir, revision)) as f:
+                revision = f.read()
+
+    # Check if file is cached as "no_exists"
+    if os.path.isfile(os.path.join(no_exists_dir, revision, filename)):
         return _CACHED_NO_EXIST
 
-    cached_shas = os.listdir(os.path.join(repo_cache, "snapshots"))
+    # Check if revision folder exists
+    if not os.path.exists(snapshots_dir):
+        return None
+    cached_shas = os.listdir(snapshots_dir)
     if revision not in cached_shas:
         # No cache for this revision and we won't try to return a random revision
         return None
 
-    cached_file = os.path.join(repo_cache, "snapshots", revision, filename)
+    # Check if file exists in cache
+    cached_file = os.path.join(snapshots_dir, revision, filename)
     return cached_file if os.path.isfile(cached_file) else None
 
 

tests/test_file_download.py

@@ -348,7 +348,7 @@ def test_hf_hub_download_legacy(self):
         metadata = filename_to_url(filepath, legacy_cache_layout=True)
         self.assertEqual(metadata[1], f'"{DUMMY_MODEL_ID_PINNED_SHA1}"')
 
-    def test_try_to_load_from_cache(self):
+    def test_try_to_load_from_cache_exist(self):
         # Make sure the file is cached
         filepath = hf_hub_download(DUMMY_MODEL_ID, filename=CONFIG_NAME)
 
@@ -389,6 +389,53 @@ def test_try_to_load_from_cache_no_exist(self):
         # If file non-existence is not cached, returns None
         self.assertIsNone(try_to_load_from_cache(DUMMY_MODEL_ID, filename="dummy2"))
 
+    def test_try_to_load_from_cache_specific_commit_id_exist(self):
+        """Regression test for #1306.
+
+        See https://github.com/huggingface/huggingface_hub/pull/1306."""
+        with SoftTemporaryDirectory() as cache_dir:
+            # Cache file from specific commit id (no "refs/"" folder)
+            commit_id = HfApi().model_info(DUMMY_MODEL_ID).sha
+            filepath = hf_hub_download(
+                DUMMY_MODEL_ID,
+                filename=CONFIG_NAME,
+                revision=commit_id,
+                cache_dir=cache_dir,
+            )
+
+            # Must be able to retrieve it "offline"
+            attempt = try_to_load_from_cache(
+                DUMMY_MODEL_ID,
+                filename=CONFIG_NAME,
+                revision=commit_id,
+                cache_dir=cache_dir,
+            )
+            self.assertEqual(filepath, attempt)
+
+    def test_try_to_load_from_cache_specific_commit_id_no_exist(self):
+        """Regression test for #1306.
+
+        See https://github.com/huggingface/huggingface_hub/pull/1306."""
+        with SoftTemporaryDirectory() as cache_dir:
+            # Cache file from specific commit id (no "refs/"" folder)
+            commit_id = HfApi().model_info(DUMMY_MODEL_ID).sha
+            with self.assertRaises(EntryNotFoundError):
+                hf_hub_download(
+                    DUMMY_MODEL_ID,
+                    filename="missing_file",
+                    revision=commit_id,
+                    cache_dir=cache_dir,
+                )
+
+            # Must be able to retrieve it "offline"
+            attempt = try_to_load_from_cache(
+                DUMMY_MODEL_ID,
+                filename="missing_file",
+                revision=commit_id,
+                cache_dir=cache_dir,
+            )
+            self.assertEqual(attempt, _CACHED_NO_EXIST)
+
     def test_get_hf_file_metadata_basic(self) -> None:
         """Test getting metadata from a file on the Hub."""
         url = hf_hub_url(