Support permanently deleting LFS files (#2954)

* Support permanently deleting LFS files

* update example

* example

Author: Wauplin

docs/source/en/package_reference/hf_api.md

@@ -57,6 +57,10 @@ models = hf_api.list_models()
 
 [[autodoc]] huggingface_hub.hf_api.GitRefs
 
+### LFSFileInfo
+
+[[autodoc]] huggingface_hub.hf_api.LFSFileInfo
+
 ### ModelInfo
 
 [[autodoc]] huggingface_hub.hf_api.ModelInfo

src/huggingface_hub/__init__.py

@@ -208,6 +208,7 @@
         "list_datasets",
         "list_inference_catalog",
         "list_inference_endpoints",
+        "list_lfs_files",
         "list_liked_repos",
         "list_models",
         "list_organization_members",
@@ -230,6 +231,7 @@
         "parse_safetensors_file_metadata",
         "pause_inference_endpoint",
         "pause_space",
+        "permanently_delete_lfs_files",
         "preupload_lfs_files",
         "reject_access_request",
         "rename_discussion",
@@ -830,6 +832,7 @@
     "list_datasets",
     "list_inference_catalog",
     "list_inference_endpoints",
+    "list_lfs_files",
     "list_liked_repos",
     "list_models",
     "list_organization_members",
@@ -862,6 +865,7 @@
     "parse_safetensors_file_metadata",
     "pause_inference_endpoint",
     "pause_space",
+    "permanently_delete_lfs_files",
     "preupload_lfs_files",
     "push_to_hub_fastai",
     "push_to_hub_keras",
@@ -1156,6 +1160,7 @@ def __dir__():
         list_datasets,  # noqa: F401
         list_inference_catalog,  # noqa: F401
         list_inference_endpoints,  # noqa: F401
+        list_lfs_files,  # noqa: F401
         list_liked_repos,  # noqa: F401
         list_models,  # noqa: F401
         list_organization_members,  # noqa: F401
@@ -1178,6 +1183,7 @@ def __dir__():
         parse_safetensors_file_metadata,  # noqa: F401
         pause_inference_endpoint,  # noqa: F401
         pause_space,  # noqa: F401
+        permanently_delete_lfs_files,  # noqa: F401
         preupload_lfs_files,  # noqa: F401
         reject_access_request,  # noqa: F401
         rename_discussion,  # noqa: F401

src/huggingface_hub/hf_api.py

@@ -112,6 +112,7 @@
     SafetensorsRepoMetadata,
     TensorInfo,
     build_hf_headers,
+    chunk_iterable,
     experimental,
     filter_repo_objects,
     fix_hf_endpoint_in_url,
@@ -1529,6 +1530,67 @@ def __init__(self, **kwargs) -> None:
         self.__dict__.update(**kwargs)
 
 
+@dataclass
+class LFSFileInfo:
+    """
+    Contains information about a file stored as LFS on a repo on the Hub.
+
+    Used in the context of listing and permanently deleting LFS files from a repo to free-up space.
+    See [`list_lfs_files`] and [`permanently_delete_lfs_files`] for more details.
+
+    Git LFS files are tracked using SHA-256 object IDs, rather than file paths, to optimize performance
+    This approach is necessary because a single object can be referenced by multiple paths across different commits,
+    making it impractical to search and resolve these connections. Check out [our documentation](https://huggingface.co/docs/hub/storage-limits#advanced-track-lfs-file-references)
+    to learn how to know which filename(s) is(are) associated with each SHA.
+
+    Attributes:
+        file_oid (`str`):
+            SHA-256 object ID of the file. This is the identifier to pass when permanently deleting the file.
+        filename (`str`):
+            Possible filename for the LFS object. See the note above for more information.
+        oid (`str`):
+            OID of the LFS object.
+        pushed_at (`datetime`):
+            Date the LFS object was pushed to the repo.
+        ref (`str`, *optional*):
+            Ref where the LFS object has been pushed (if any).
+        size (`int`):
+            Size of the LFS object.
+
+    Example:
+        ```py
+        >>> from huggingface_hub import HfApi
+        >>> api = HfApi()
+        >>> lfs_files = api.list_lfs_files("username/my-cool-repo")
+
+        # Filter files files to delete based on a combination of `filename`, `pushed_at`, `ref` or `size`.
+        # e.g. select only LFS files in the "checkpoints" folder
+        >>> lfs_files_to_delete = (lfs_file for lfs_file in lfs_files if lfs_file.filename.startswith("checkpoints/"))
+
+        # Permanently delete LFS files
+        >>> api.permanently_delete_lfs_files("username/my-cool-repo", lfs_files_to_delete)
+        ```
+    """
+
+    file_oid: str
+    filename: str
+    oid: str
+    pushed_at: datetime
+    ref: Optional[str]
+    size: int
+
+    def __init__(self, **kwargs) -> None:
+        self.file_oid = kwargs.pop("fileOid")
+        self.filename = kwargs.pop("filename")
+        self.oid = kwargs.pop("oid")
+        self.pushed_at = parse_datetime(kwargs.pop("pushedAt"))
+        self.ref = kwargs.pop("ref", None)
+        self.size = kwargs.pop("size")
+
+        # forward compatibility
+        self.__dict__.update(**kwargs)
+
+
 def future_compatible(fn: CallableT) -> CallableT:
     """Wrap a method of `HfApi` to handle `run_as_future=True`.
 
@@ -3387,6 +3449,131 @@ def super_squash_history(
         response = get_session().post(url=url, headers=headers, json={"message": commit_message})
         hf_raise_for_status(response)
 
+    @validate_hf_hub_args
+    def list_lfs_files(
+        self,
+        repo_id: str,
+        *,
+        repo_type: Optional[str] = None,
+        token: Union[bool, str, None] = None,
+    ) -> Iterable[LFSFileInfo]:
+        """
+        List all LFS files in a repo on the Hub.
+
+        This is primarily useful to count how much storage a repo is using and to eventually clean up large files
+        with [`permanently_delete_lfs_files`]. Note that this would be a permanent action that will affect all commits
+        referencing this deleted files and that cannot be undone.
+
+        Args:
+            repo_id (`str`):
+                The repository for which you are listing LFS files.
+            repo_type (`str`, *optional*):
+                Type of repository. Set to `"dataset"` or `"space"` if listing from a dataset or space, `None` or
+                `"model"` if listing from a model. Default is `None`.
+            token (Union[bool, str, None], optional):
+                A valid user access token (string). Defaults to the locally saved
+                token, which is the recommended method for authentication (see
+                https://huggingface.co/docs/huggingface_hub/quick-start#authentication).
+                To disable authentication, pass `False`.
+
+        Returns:
+            `Iterable[LFSFileInfo]`: An iterator of [`LFSFileInfo`] objects.
+
+        Example:
+            ```py
+            >>> from huggingface_hub import HfApi
+            >>> api = HfApi()
+            >>> lfs_files = api.list_lfs_files("username/my-cool-repo")
+
+            # Filter files files to delete based on a combination of `filename`, `pushed_at`, `ref` or `size`.
+            # e.g. select only LFS files in the "checkpoints" folder
+            >>> lfs_files_to_delete = (lfs_file for lfs_file in lfs_files if lfs_file.filename.startswith("checkpoints/"))
+
+            # Permanently delete LFS files
+            >>> api.permanently_delete_lfs_files("username/my-cool-repo", lfs_files_to_delete)
+            ```
+        """
+        # Prepare request
+        if repo_type is None:
+            repo_type = constants.REPO_TYPE_MODEL
+        url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/lfs-files"
+        headers = self._build_hf_headers(token=token)
+
+        # Paginate over LFS items
+        for item in paginate(url, params={}, headers=headers):
+            yield LFSFileInfo(**item)
+
+    @validate_hf_hub_args
+    def permanently_delete_lfs_files(
+        self,
+        repo_id: str,
+        lfs_files: Iterable[LFSFileInfo],
+        *,
+        rewrite_history: bool = True,
+        repo_type: Optional[str] = None,
+        token: Union[bool, str, None] = None,
+    ) -> None:
+        """
+        Permanently delete LFS files from a repo on the Hub.
+
+        <Tip warning={true}>
+
+        This is a permanent action that will affect all commits referencing the deleted files and might corrupt your
+        repository. This is a non-revertible operation. Use it only if you know what you are doing.
+
+        </Tip>
+
+        Args:
+            repo_id (`str`):
+                The repository for which you are listing LFS files.
+            lfs_files (`Iterable[LFSFileInfo]`):
+                An iterable of [`LFSFileInfo`] items to permanently delete from the repo. Use [`list_lfs_files`] to list
+                all LFS files from a repo.
+            rewrite_history (`bool`, *optional*, default to `True`):
+                Whether to rewrite repository history to remove file pointers referencing the deleted LFS files (recommended).
+            repo_type (`str`, *optional*):
+                Type of repository. Set to `"dataset"` or `"space"` if listing from a dataset or space, `None` or
+                `"model"` if listing from a model. Default is `None`.
+            token (Union[bool, str, None], optional):
+                A valid user access token (string). Defaults to the locally saved
+                token, which is the recommended method for authentication (see
+                https://huggingface.co/docs/huggingface_hub/quick-start#authentication).
+                To disable authentication, pass `False`.
+
+        Example:
+            ```py
+            >>> from huggingface_hub import HfApi
+            >>> api = HfApi()
+            >>> lfs_files = api.list_lfs_files("username/my-cool-repo")
+
+            # Filter files files to delete based on a combination of `filename`, `pushed_at`, `ref` or `size`.
+            # e.g. select only LFS files in the "checkpoints" folder
+            >>> lfs_files_to_delete = (lfs_file for lfs_file in lfs_files if lfs_file.filename.startswith("checkpoints/"))
+
+            # Permanently delete LFS files
+            >>> api.permanently_delete_lfs_files("username/my-cool-repo", lfs_files_to_delete)
+            ```
+        """
+        # Prepare request
+        if repo_type is None:
+            repo_type = constants.REPO_TYPE_MODEL
+        url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/lfs-files/batch"
+        headers = self._build_hf_headers(token=token)
+
+        # Delete LFS items by batches of 1000
+        for batch in chunk_iterable(lfs_files, 1000):
+            shas = [item.file_oid for item in batch]
+            if len(shas) == 0:
+                return
+            payload = {
+                "deletions": {
+                    "sha": shas,
+                    "rewriteHistory": rewrite_history,
+                }
+            }
+            response = get_session().post(url, headers=headers, json=payload)
+            hf_raise_for_status(response)
+
     @validate_hf_hub_args
     def create_repo(
         self,
@@ -9641,7 +9828,6 @@ def _parse_revision_from_pr_url(pr_url: str) -> str:
 delete_repo = api.delete_repo
 update_repo_visibility = api.update_repo_visibility
 update_repo_settings = api.update_repo_settings
-super_squash_history = api.super_squash_history
 move_repo = api.move_repo
 upload_file = api.upload_file
 upload_folder = api.upload_folder
@@ -9656,6 +9842,11 @@ def _parse_revision_from_pr_url(pr_url: str) -> str:
 delete_tag = api.delete_tag
 get_full_repo_name = api.get_full_repo_name
 
+# Danger-zone API
+super_squash_history = api.super_squash_history
+list_lfs_files = api.list_lfs_files
+permanently_delete_lfs_files = api.permanently_delete_lfs_files
+
 # Safetensors helpers
 get_safetensors_metadata = api.get_safetensors_metadata
 parse_safetensors_file_metadata = api.parse_safetensors_file_metadata

tests/test_hf_api.py

@@ -3103,6 +3103,69 @@ def test_super_squash_history_on_special_ref(self, repo_url: RepoUrl) -> None:
         assert len(squashed_branch_commits) == 1
 
 
+class TestListAndPermanentlyDeleteLFSFiles(HfApiCommonTest):
+    @use_tmp_repo()
+    def test_list_and_delete_lfs_files(self, repo_url: RepoUrl) -> None:
+        repo_id = repo_url.repo_id
+
+        # Main files
+        self._api.upload_file(path_or_fileobj=b"LFS content", path_in_repo="lfs_file.bin", repo_id=repo_id)
+        self._api.upload_file(path_or_fileobj=b"TXT content", path_in_repo="txt_file.txt", repo_id=repo_id)
+        self._api.upload_file(path_or_fileobj=b"LFS content 2", path_in_repo="lfs_file_2.bin", repo_id=repo_id)
+        self._api.upload_file(path_or_fileobj=b"TXT content 2", path_in_repo="txt_file_2.txt", repo_id=repo_id)
+
+        # Branch files
+        self._api.create_branch(repo_id=repo_id, branch="my-branch")
+        self._api.upload_file(
+            path_or_fileobj=b"LFS content branch",
+            path_in_repo="lfs_file_branch.bin",
+            repo_id=repo_id,
+            revision="my-branch",
+        )
+        self._api.upload_file(
+            path_or_fileobj=b"TXT content branch",
+            path_in_repo="txt_file_branch.txt",
+            repo_id=repo_id,
+            revision="my-branch",
+        )
+
+        # PR files
+        self._api.upload_file(
+            path_or_fileobj=b"LFS content PR", path_in_repo="lfs_file_PR.bin", repo_id=repo_id, create_pr=True
+        )
+        self._api.upload_file(
+            path_or_fileobj=b"TXT content PR", path_in_repo="txt_file_PR.txt", repo_id=repo_id, create_pr=True
+        )
+
+        # List LFS files
+        lfs_files = [file for file in self._api.list_lfs_files(repo_id=repo_id)]
+        assert len(lfs_files) == 4
+        assert {file.filename for file in lfs_files} == {
+            "lfs_file.bin",
+            "lfs_file_2.bin",
+            "lfs_file_branch.bin",
+            "lfs_file_PR.bin",
+        }
+
+        # Select LFS files that are on main
+        lfs_files_on_main = [file for file in lfs_files if file.ref == "main"]
+        assert len(lfs_files_on_main) == 2
+
+        # Permanently delete LFS files
+        self._api.permanently_delete_lfs_files(repo_id=repo_id, lfs_files=lfs_files_on_main)
+
+        # LFS files from branch and PR remain
+        lfs_files = [file for file in self._api.list_lfs_files(repo_id=repo_id)]
+        assert len(lfs_files) == 2
+        assert {file.filename for file in lfs_files} == {"lfs_file_branch.bin", "lfs_file_PR.bin"}
+
+        # Downloading "lfs_file.bin" fails with EntryNotFoundError
+        files = self._api.list_repo_files(repo_id=repo_id)
+        assert set(files) == {".gitattributes", "txt_file.txt", "txt_file_2.txt"}
+        with pytest.raises(EntryNotFoundError):
+            self._api.hf_hub_download(repo_id=repo_id, filename="lfs_file.bin")
+
+
 @pytest.mark.vcr
 class TestSpaceAPIProduction(unittest.TestCase):
     """