Fix bad total size after resuming download (#3234) (#3248)

* Fix bad total size after resuming download (#3234)

* Apply suggestions from code review

Co-authored-by: célina <[email protected]>

---------

Co-authored-by: célina <[email protected]>

Author: DKingAlpha

src/huggingface_hub/file_download.py

@@ -317,9 +317,6 @@ def _get_file_length_from_http_response(response: requests.Response) -> Optional
 
     This function extracts the file size from the HTTP response headers, either from the
     `Content-Range` or `Content-Length` header, if available (in that order).
-        The HTTP response object containing the headers.
-        `int` or `None`: The length of the file in bytes if the information is available,
-        otherwise `None`.
 
     Args:
         response (`requests.Response`):
@@ -329,6 +326,15 @@ def _get_file_length_from_http_response(response: requests.Response) -> Optional
         `int` or `None`: The length of the file in bytes, or None if not available.
     """
 
+    # If HTTP response contains compressed body (e.g. gzip), the `Content-Length` header will
+    # contain the length of the compressed body, not the uncompressed file size.
+    # And at the start of transmission there's no way to know the uncompressed file size for gzip,
+    # thus we return None in that case.
+    content_encoding = response.headers.get("Content-Encoding", "identity").lower()
+    if content_encoding != "identity":
+        # gzip/br/deflate/zstd etc
+        return None
+
     content_range = response.headers.get("Content-Range")
     if content_range is not None:
         return int(content_range.rsplit("/")[-1])
@@ -422,11 +428,7 @@ def http_get(
     )
 
     hf_raise_for_status(r)
-    content_length = _get_file_length_from_http_response(r)
-
-    # NOTE: 'total' is the total number of bytes to download, not the number of bytes in the file.
-    #       If the file is compressed, the number of bytes in the saved file will be higher than 'total'.
-    total = resume_size + int(content_length) if content_length is not None else None
+    total: Optional[int] = _get_file_length_from_http_response(r)
 
     if displayed_filename is None:
         displayed_filename = url

tests/test_file_download.py

@@ -1205,6 +1205,7 @@ def test_etag_timeout_set_as_env_variable_parameter_ignored(self):
 @with_production_testing
 class TestExtraLargeFileDownloadPaths(unittest.TestCase):
     @patch("huggingface_hub.file_download.constants.HF_HUB_ENABLE_HF_TRANSFER", False)
+    @patch("huggingface_hub.file_download.constants.HF_HUB_DISABLE_XET", True)
     def test_large_file_http_path_error(self):
         with SoftTemporaryDirectory() as cache_dir:
             with self.assertRaises(
@@ -1226,6 +1227,7 @@ def test_large_file_http_path_error(self):
         "hf_transfer not installed, so skipping large file download with hf_transfer check.",
     )
     @patch("huggingface_hub.file_download.constants.HF_HUB_ENABLE_HF_TRANSFER", True)
+    @patch("huggingface_hub.file_download.constants.HF_HUB_DISABLE_XET", True)
     @patch("huggingface_hub.file_download.constants.MAX_HTTP_DOWNLOAD_SIZE", 44)
     @patch("huggingface_hub.file_download.constants.DOWNLOAD_CHUNK_SIZE", 2)  # make sure hf_download is used
     def test_large_file_download_with_hf_transfer(self):