Send token implicitly on LFS upload (#1084)

* Send token implicitely on LFS upload

* add regression test

Author: Wauplin

src/huggingface_hub/_commit_api.py

@@ -166,7 +166,7 @@ def upload_lfs_files(
     additions: Iterable[CommitOperationAdd],
     repo_type: str,
     repo_id: str,
-    token: str,
+    token: Optional[str],
     endpoint: Optional[str] = None,
     num_threads: int = 5,
 ):
@@ -184,7 +184,7 @@ def upload_lfs_files(
         repo_id (`str`):
             A namespace (user or an organization) and a repo name separated
             by a `/`.
-        token (`str`):
+        token (`str`, *optional*):
             An authentication token ( See https://huggingface.co/settings/tokens )
         num_threads (`int`, *optional*):
             The number of concurrent threads to use when uploading. Defaults to 5.
@@ -256,7 +256,7 @@ def upload_lfs_files(
 
 
 def _upload_lfs_object(
-    operation: CommitOperationAdd, lfs_batch_action: dict, token: str
+    operation: CommitOperationAdd, lfs_batch_action: dict, token: Optional[str]
 ):
     """
     Handles uploading a given object to the Hub with the LFS protocol.
@@ -272,7 +272,7 @@ def _upload_lfs_object(
         lfs_batch_action (`dict`):
             Upload instructions from the LFS batch endpoint for this object.
             See [`~utils.lfs.post_lfs_batch_info`] for more details.
-        token (`str`):
+        token (`str`, *optional*):
             A [user access token](https://hf.co/settings/tokens) to authenticate requests against the Hub
 
     Raises: `ValueError` if `lfs_batch_action` is improperly formatted
@@ -321,7 +321,7 @@ def fetch_upload_modes(
     additions: Iterable[CommitOperationAdd],
     repo_type: str,
     repo_id: str,
-    token: str,
+    token: Optional[str],
     revision: str,
     endpoint: Optional[str] = None,
     create_pr: Optional[bool] = None,
@@ -339,7 +339,7 @@ def fetch_upload_modes(
         repo_id (`str`):
             A namespace (user or an organization) and a repo name separated
             by a `/`.
-        token (`str`):
+        token (`str`, *optional*):
             An authentication token ( See https://huggingface.co/settings/tokens )
         revision (`str`):
             The git revision to upload the files to. Can be any valid git revision.

src/huggingface_hub/hf_api.py

@@ -2705,7 +2705,7 @@ def create_discussion(
         repo_id: str,
         title: str,
         *,
-        token: str,
+        token: Optional[str],
         description: Optional[str] = None,
         repo_type: Optional[str] = None,
         pull_request: bool = False,
@@ -2724,7 +2724,7 @@ def create_discussion(
                 The title of the discussion. It can be up to 200 characters long,
                 and must be at least 3 characters long. Leading and trailing whitespaces
                 will be stripped.
-            token (`str`):
+            token (`str`, *optional*):
                 An authentication token (See https://huggingface.co/settings/token)
             description (`str`, *optional*):
                 An optional description for the Pull Request.
@@ -2794,7 +2794,7 @@ def create_pull_request(
         repo_id: str,
         title: str,
         *,
-        token: str,
+        token: Optional[str],
         description: Optional[str] = None,
         repo_type: Optional[str] = None,
     ) -> DiscussionWithDetails:
@@ -2812,7 +2812,7 @@ def create_pull_request(
                 The title of the discussion. It can be up to 200 characters long,
                 and must be at least 3 characters long. Leading and trailing whitespaces
                 will be stripped.
-            token (`str`):
+            token (`str`, *optional*):
                 An authentication token (See https://huggingface.co/settings/token)
             description (`str`, *optional*):
                 An optional description for the Pull Request.
@@ -3096,7 +3096,7 @@ def merge_pull_request(
         repo_id: str,
         discussion_num: int,
         *,
-        token: str,
+        token: Optional[str],
         comment: Optional[str] = None,
         repo_type: Optional[str] = None,
     ):
@@ -3207,7 +3207,7 @@ def hide_discussion_comment(
         discussion_num: int,
         comment_id: str,
         *,
-        token: str,
+        token: Optional[str],
         repo_type: Optional[str] = None,
     ) -> DiscussionComment:
         """Hides a comment on a Discussion / Pull Request.

src/huggingface_hub/lfs.py

@@ -26,7 +26,12 @@
 from huggingface_hub.constants import ENDPOINT, REPO_TYPES_URL_PREFIXES
 from requests.auth import HTTPBasicAuth
 
-from .utils import hf_raise_for_status, http_backoff, validate_hf_hub_args
+from .utils import (
+    get_token_to_send,
+    hf_raise_for_status,
+    http_backoff,
+    validate_hf_hub_args,
+)
 from .utils.sha import sha256, sha_fileobj
 
 
@@ -132,7 +137,7 @@ def _validate_batch_error(lfs_batch_error: dict):
 @validate_hf_hub_args
 def post_lfs_batch_info(
     upload_infos: Iterable[UploadInfo],
-    token: str,
+    token: Optional[str],
     repo_type: str,
     repo_id: str,
     endpoint: Optional[str] = None,
@@ -151,7 +156,7 @@ def post_lfs_batch_info(
         repo_id (`str`):
             A namespace (user or an organization) and a repo name separated
             by a `/`.
-        token (`str`):
+        token (`str`, *optional*):
             An authentication token ( See https://huggingface.co/settings/tokens )
 
     Returns:
@@ -187,7 +192,10 @@ def post_lfs_batch_info(
             ],
             "hash_algo": "sha256",
         },
-        auth=HTTPBasicAuth("access_token", token),
+        auth=HTTPBasicAuth(
+            "access_token",
+            get_token_to_send(token or True),  # Token must be provided or retrieved
+        ),
     )
     hf_raise_for_status(resp)
     batch_info = resp.json()
@@ -207,7 +215,7 @@ def lfs_upload(
     upload_info: UploadInfo,
     upload_action: dict,
     verify_action: Optional[dict],
-    token: str,
+    token: Optional[str],
 ):
     """
     Uploads a file using the git lfs protocol and determines automatically whether or not
@@ -225,7 +233,7 @@ def lfs_upload(
             The `verify` action from the LFS Batch endpoint. Must contain
             a `href` field, and optionally a `header` field. The `href` URL will
             be called after a successful upload.
-        token (`str`):
+        token (`str`, *optional*):
             A [user access token](https://hf.co/settings/tokens) to authenticate requests
             against the Hub.
 
@@ -268,7 +276,11 @@ def lfs_upload(
     if verify_action is not None:
         verify_resp = requests.post(
             verify_action["href"],
-            auth=HTTPBasicAuth(username="USER", password=token),
+            auth=HTTPBasicAuth(
+                username="USER",
+                # Token must be provided or retrieved
+                password=get_token_to_send(token or True),
+            ),
             json={"oid": upload_info.sha256.hex(), "size": upload_info.size},
         )
         hf_raise_for_status(verify_resp)

src/huggingface_hub/utils/__init__.py

@@ -36,7 +36,7 @@
     hf_raise_for_status,
 )
 from ._fixes import yaml_dump
-from ._headers import build_hf_headers
+from ._headers import build_hf_headers, get_token_to_send
 from ._hf_folder import HfFolder
 from ._http import http_backoff
 from ._paths import filter_repo_objects

src/huggingface_hub/utils/_headers.py

@@ -112,7 +112,7 @@ def build_hf_headers(
             If `use_auth_token=True` but token is not saved locally.
     """
     # Get auth token to send
-    token_to_send = _get_token_to_send(use_auth_token)
+    token_to_send = get_token_to_send(use_auth_token)
     _validate_token_to_send(token_to_send, is_write_action=is_write_action)
 
     # Combine headers
@@ -128,7 +128,7 @@ def build_hf_headers(
     return headers
 
 
-def _get_token_to_send(use_auth_token: Optional[Union[bool, str]]) -> Optional[str]:
+def get_token_to_send(use_auth_token: Optional[Union[bool, str]]) -> Optional[str]:
     """Select the token to send from either `use_auth_token` or the cache."""
     # Case token is explicitly provided
     if isinstance(use_auth_token, str):

tests/test_hf_api.py

@@ -951,6 +951,54 @@ def test_create_commit_repo_does_not_exist(self) -> None:
 
         self.assertEqual(str(context.exception), expected_message)
 
+    @retry_endpoint
+    def test_create_commit_lfs_file_implicit_token(self):
+        """Test that uploading a file as LFS works with implicit token (from cache).
+
+        Regression test for https://github.com/huggingface/huggingface_hub/pull/1084.
+        """
+        REPO_NAME = repo_name("create_commit_with_lfs")
+        repo_id = f"{USER}/{REPO_NAME}"
+
+        with patch("huggingface_hub.utils.HfFolder.get_token") as mock:
+            mock.return_value = self._token  # Set implicit token
+
+            # Create repo
+            self._api.create_repo(repo_id=REPO_NAME, exist_ok=False)
+
+            # Set repo to track png files as LFS
+            self._api.create_commit(
+                operations=[
+                    CommitOperationAdd(
+                        path_in_repo=".gitattributes",
+                        path_or_fileobj=b"*.png filter=lfs diff=lfs merge=lfs -text",
+                    ),
+                ],
+                commit_message="Update .gitattributes",
+                repo_id=repo_id,
+            )
+
+            # Upload a PNG file
+            self._api.create_commit(
+                operations=[
+                    CommitOperationAdd(
+                        path_in_repo="image.png", path_or_fileobj=b"image data"
+                    ),
+                ],
+                commit_message="Test upload lfs file",
+                repo_id=repo_id,
+            )
+
+            # Check uploaded as LFS
+            info = self._api.model_info(
+                repo_id=repo_id, use_auth_token=self._token, files_metadata=True
+            )
+            siblings = {file.rfilename: file for file in info.siblings}
+            self.assertIsInstance(siblings["image.png"].lfs, dict)  # LFS file
+
+            # Delete repo
+            self._api.delete_repo(repo_id=REPO_NAME, token=self._token)
+
 
 class HfApiPublicTest(unittest.TestCase):
     def test_staging_list_models(self):