Handle hf:// urls + raise ValueError if repo type is unknown (#1298)

Author: Wauplin

src/huggingface_hub/hf_api.py

@@ -103,6 +103,8 @@ def repo_type_and_id_from_hf_id(
 
             - https://huggingface.co/<repo_type>/<namespace>/<repo_id>
             - https://huggingface.co/<namespace>/<repo_id>
+            - hf://<repo_type>/<namespace>/<repo_id>
+            - hf://<namespace>/<repo_id>
             - <repo_type>/<namespace>/<repo_id>
             - <namespace>/<repo_id>
             - <repo_id>
@@ -112,9 +114,21 @@ def repo_type_and_id_from_hf_id(
     Returns:
         A tuple with three items: repo_type (`str` or `None`), namespace (`str` or
         `None`) and repo_id (`str`).
+
+    Raises:
+        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
+            If URL cannot be parsed.
+        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
+            If `repo_type` is unknown.
     """
+    input_hf_id = hf_id
     hub_url = re.sub(r"https?://", "", hub_url if hub_url is not None else ENDPOINT)
     is_hf_url = hub_url in hf_id and "@" not in hf_id
+
+    HFFS_PREFIX = "hf://"
+    if hf_id.startswith(HFFS_PREFIX):  # Remove "hf://" prefix if exists
+        hf_id = hf_id[len(HFFS_PREFIX) :]
+
     url_segments = hf_id.split("/")
     is_hf_id = len(url_segments) <= 3
 
@@ -144,9 +158,13 @@ def repo_type_and_id_from_hf_id(
             f"Unable to retrieve user and repo ID from the passed HF ID: {hf_id}"
         )
 
+    # Check if repo type is known (mapping "spaces" => "space" + empty value => `None`)
+    if repo_type in REPO_TYPES_MAPPING:
+        repo_type = REPO_TYPES_MAPPING[repo_type]
+    if repo_type == "":
+        repo_type = None
     if repo_type not in REPO_TYPES:
-        assert repo_type is not None, "repo_type `None` do not have mapping"
-        repo_type = REPO_TYPES_MAPPING.get(repo_type)
+        raise ValueError(f"Unknown `repo_type`: '{repo_type}' ('{input_hf_id}')")
 
     return repo_type, namespace, repo_id
 
@@ -234,12 +252,21 @@ class RepoUrl(str):
     >>> RepoUrl('https://huggingface.co/gpt2')
     RepoUrl('https://huggingface.co/gpt2', endpoint='https://huggingface.co', repo_type='model', repo_id='gpt2')
 
-    >>> RepoUrl('https://hub-ci.huggingface.co/dataset/dummy_user/dummy_dataset', endpoint='https://hub-ci.huggingface.co')
-    RepoUrl('https://hub-ci.huggingface.co/dataset/dummy_user/dummy_dataset', endpoint='https://hub-ci.huggingface.co', repo_type='dataset', repo_id='dummy_user/dummy_dataset')
+    >>> RepoUrl('https://hub-ci.huggingface.co/datasets/dummy_user/dummy_dataset', endpoint='https://hub-ci.huggingface.co')
+    RepoUrl('https://hub-ci.huggingface.co/datasets/dummy_user/dummy_dataset', endpoint='https://hub-ci.huggingface.co', repo_type='dataset', repo_id='dummy_user/dummy_dataset')
+
+    >>> RepoUrl('hf://datasets/my-user/my-dataset')
+    RepoUrl('hf://datasets/my-user/my-dataset', endpoint='https://huggingface.co', repo_type='dataset', repo_id='user/dataset')
 
     >>> HfApi.create_repo("dummy_model")
     RepoUrl('https://huggingface.co/Wauplin/dummy_model', endpoint='https://huggingface.co', repo_type='model', repo_id='Wauplin/dummy_model')
     ```
+
+    Raises:
+        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
+            If URL cannot be parsed.
+        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
+            If `repo_type` is unknown.
     """
 
     def __new__(cls, url: Any, endpoint: Optional[str] = None):

tests/test_hf_api.py

@@ -2155,8 +2155,8 @@ def test_end_to_end_thresh_16M(self):
         print("took", time.time() - start_time)
 
 
-class HfApiMiscTest(unittest.TestCase):
-    def test_repo_type_and_id_from_hf_id(self):
+class ParseHFUrlTest(unittest.TestCase):
+    def test_repo_type_and_id_from_hf_id_on_correct_values(self):
         possible_values = {
             "https://huggingface.co/id": [None, None, "id"],
             "https://huggingface.co/user/id": [None, "user", "id"],
@@ -2166,6 +2166,10 @@ def test_repo_type_and_id_from_hf_id(self):
             "dataset/user/id": ["dataset", "user", "id"],
             "space/user/id": ["space", "user", "id"],
             "id": [None, None, "id"],
+            "hf://id": [None, None, "id"],
+            "hf://user/id": [None, "user", "id"],
+            "hf://model/user/name": ["model", "user", "name"],  # 's' is optional
+            "hf://models/user/name": ["model", "user", "name"],
         }
 
         for key, value in possible_values.items():
@@ -2174,6 +2178,17 @@ def test_repo_type_and_id_from_hf_id(self):
                 tuple(value),
             )
 
+    def test_repo_type_and_id_from_hf_id_on_wrong_values(self):
+        for hub_id in [
+            "https://unknown-endpoint.co/id",
+            "https://huggingface.co/datasets/user/id@revision",  # @ forbidden
+            "datasets/user/id/subpath",
+            "hffs://model/user/name",
+            "spaeces/user/id",  # with typo in repo type
+        ]:
+            with self.assertRaises(ValueError):
+                repo_type_and_id_from_hf_id(hub_id, hub_url="https://huggingface.co")
+
 
 class HfApiDiscussionsTest(HfApiCommonTestWithLogin):
     def setUp(self):