Merge branch 'main' into buckets-api

Author: Wauplin

src/huggingface_hub/__init__.py

@@ -46,7 +46,7 @@
 from typing import TYPE_CHECKING
 
 
-__version__ = "1.3.0.dev0"
+__version__ = "1.5.0.dev0"
 
 # Alphabetical order of definitions is ensured in tests
 # WARNING: any comment added in this dictionary definition will be lost when

src/huggingface_hub/cli/hf.py

@@ -14,8 +14,11 @@
 
 import sys
 import traceback
+from typing import Annotated, Optional
 
-from huggingface_hub import constants
+import typer
+
+from huggingface_hub import __version__, constants
 from huggingface_hub.cli._cli_utils import check_cli_update, typer_factory
 from huggingface_hub.cli._errors import format_known_exception
 from huggingface_hub.cli.auth import auth_cli
@@ -43,6 +46,21 @@
 app = typer_factory(help="Hugging Face Hub CLI")
 
 
+def _version_callback(value: bool) -> None:
+    if value:
+        print(__version__)
+        raise typer.Exit()
+
+
+@app.callback(invoke_without_command=True)
+def app_callback(
+    version: Annotated[
+        Optional[bool], typer.Option("--version", callback=_version_callback, is_eager=True, hidden=True)
+    ] = None,
+) -> None:
+    pass
+
+
 # top level single commands (defined in their respective files)
 app.command()(sync)
 app.command(examples=DOWNLOAD_EXAMPLES)(download)

src/huggingface_hub/cli/jobs.py

@@ -62,6 +62,7 @@
 import multiprocessing
 import multiprocessing.pool
 import os
+import shutil
 import time
 from dataclasses import asdict
 from fnmatch import fnmatch
@@ -1010,7 +1011,7 @@ def _tabulate(rows: list[list[Union[str, int]]], headers: list[str]) -> str:
     - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data
     """
     col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)]
-    terminal_width = max(os.get_terminal_size().columns, len(headers) * 12)
+    terminal_width = max(shutil.get_terminal_size().columns, len(headers) * 12)
     while len(headers) + sum(col_widths) > terminal_width:
         col_to_minimize = col_widths.index(max(col_widths))
         col_widths[col_to_minimize] //= 2

src/huggingface_hub/file_download.py

@@ -418,6 +418,14 @@ def http_get(
         retry_on_status_codes=(429,),
     ) as response:
         hf_raise_for_status(response)
+
+        # If we requested a Range but got 200 back, the server ignored our Range header
+        # (e.g. CloudFront with Accept-Encoding: gzip). Reset file to avoid corruption.
+        if resume_size > 0 and response.status_code == 200:
+            temp_file.seek(0)
+            temp_file.truncate()
+            resume_size = 0
+
         total: Optional[int] = _get_file_length_from_http_response(response)
 
         if displayed_filename is None:

src/huggingface_hub/hf_api.py

@@ -673,6 +673,8 @@ class RepoFile:
             The file's git OID.
         lfs (`BlobLfsInfo`, *optional*):
             The file's LFS metadata.
+        xet_hash (`str`, *optional*):
+            The file's Xet hash.
         last_commit (`LastCommitInfo`, *optional*):
             The file's last commit metadata. Only defined if [`list_repo_tree`] and [`get_paths_info`]
             are called with `expand=True`.
@@ -685,6 +687,7 @@ class RepoFile:
     size: int
     blob_id: str
     lfs: Optional[BlobLfsInfo] = None
+    xet_hash: Optional[str] = None
     last_commit: Optional[LastCommitInfo] = None
     security: Optional[BlobSecurityInfo] = None
 
@@ -696,6 +699,7 @@ def __init__(self, **kwargs):
         if lfs is not None:
             lfs = BlobLfsInfo(size=lfs["size"], sha256=lfs["oid"], pointer_size=lfs["pointerSize"])
         self.lfs = lfs
+        self.xet_hash = kwargs.pop("xetHash", None)
         last_commit = kwargs.pop("lastCommit", None) or kwargs.pop("last_commit", None)
         if last_commit is not None:
             last_commit = LastCommitInfo(

src/huggingface_hub/hf_file_system.py

@@ -563,6 +563,7 @@ def _ls_tree(
                         "type": "file",
                         "blob_id": path_info.blob_id,
                         "lfs": path_info.lfs,
+                        "xet_hash": path_info.xet_hash,
                         "last_commit": path_info.last_commit,
                         "security": path_info.security,
                     }
@@ -830,6 +831,7 @@ def info(self, path: str, refresh: bool = False, revision: Optional[str] = None,
                         "type": "file",
                         "blob_id": path_info.blob_id,
                         "lfs": path_info.lfs,
+                        "xet_hash": path_info.xet_hash,
                         "last_commit": path_info.last_commit,
                         "security": path_info.security,
                     }

tests/test_file_download.py

@@ -1152,6 +1152,47 @@ def _iter_content_4() -> Iterable[bytes]:
         for i, expected_range in enumerate(expected_ranges):
             assert mock_stream_backoff.call_args_list[i].kwargs["headers"] == {"Range": expected_range}
 
+    def test_http_get_retry_resets_file_when_range_ignored(self, caplog):
+        """Test that http_get resets the file when the server ignores the Range header.
+
+        When a download is interrupted and retried with a Range header, some servers
+        (e.g. CloudFront with Accept-Encoding: gzip) ignore the Range header and return
+        200 with the full file instead of 206. In that case, the code must truncate
+        the file before writing to avoid appending the full content to partial data.
+        """
+
+        def _iter_content_1() -> Iterable[bytes]:
+            yield b"A" * 30
+            raise httpx.TimeoutException("Fake timeout")
+
+        def _iter_content_2() -> Iterable[bytes]:
+            # Server ignores Range, returns full content
+            yield b"B" * 100
+
+        mock_response_1 = Mock()
+        mock_response_1.status_code = 200
+        mock_response_1.headers = {"Content-Length": "100"}
+        mock_response_1.iter_bytes.return_value = _iter_content_1()
+
+        mock_response_2 = Mock()
+        mock_response_2.status_code = 200  # 200, not 206 — Range was ignored
+        mock_response_2.headers = {"Content-Length": "100"}
+        mock_response_2.iter_bytes.return_value = _iter_content_2()
+
+        mock_responses = iter([mock_response_1, mock_response_2])
+
+        @contextmanager
+        def _mock_stream(*args, **kwargs):
+            yield next(mock_responses)
+
+        with patch("huggingface_hub.file_download.http_stream_backoff", side_effect=_mock_stream):
+            temp_file = io.BytesIO()
+            http_get("fake_url", temp_file=temp_file)
+
+        # File should contain only the full content from retry (100 bytes), not 130
+        assert temp_file.tell() == 100
+        assert temp_file.getvalue() == b"B" * 100
+
 
 class CreateSymlinkTest(unittest.TestCase):
     @unittest.skipIf(os.name == "nt", "No symlinks on Windows")

tests/test_hf_api.py

@@ -1450,12 +1450,12 @@ def tearDownClass(cls):
 
     def test_list_tree(self):
         tree = list(self._api.list_repo_tree(repo_id=self.repo_id))
-        self.assertEqual(len(tree), 6)
-        self.assertEqual({tree_obj.path for tree_obj in tree}, {"file.md", "lfs.bin", "1", "2", "3", ".gitattributes"})
+        assert len(tree) == 6
+        assert {tree_obj.path for tree_obj in tree} == {"file.md", "lfs.bin", "1", "2", "3", ".gitattributes"}
 
         tree = list(self._api.list_repo_tree(repo_id=self.repo_id, path_in_repo="1"))
-        self.assertEqual(len(tree), 2)
-        self.assertEqual({tree_obj.path for tree_obj in tree}, {"1/file_1.md", "1/2"})
+        assert len(tree) == 2
+        assert {tree_obj.path for tree_obj in tree} == {"1/file_1.md", "1/2"}
 
     def test_list_tree_recursively(self):
         tree = list(self._api.list_repo_tree(repo_id=self.repo_id, recursive=True))
@@ -1530,6 +1530,12 @@ def test_list_files_without_expand(self):
         feature_extractor = next(tree_obj for tree_obj in tree if tree_obj.path == "feature_extractor")
         self.assertIsNone(feature_extractor.last_commit)
 
+    @with_production_testing
+    def test_list_tree_with_xethash(self):
+        tree = list(HfApi().list_repo_tree(repo_id="openai-community/gpt2"))
+        model_entry = next(tree_obj for tree_obj in tree if tree_obj.path == "model.safetensors")
+        assert model_entry.xet_hash == "63bed80836ee0758c8fd4f8975d59bb0b864263ee2753547c358e8a37cde8758"
+
 
 class HfApiTagEndpointTest(HfApiCommonTest):
     @use_tmp_repo("model")