Make tests more robust (no pre-existing repo on staging Hub) (#1302)

* Create gated repo on the fly

* create repo with awful filename on the fly

* Create foreign repo on the fly to test PR creation

* scan cache tests on production repo

* make it work on windows

* final adapt

* make snapshot download tests faster

* Fix test_list_liked_repos_no_auth

Author: Wauplin

tests/test_file_download.py

@@ -20,6 +20,8 @@
 
 import pytest
 
+import requests
+from huggingface_hub import HfApi
 from huggingface_hub.constants import (
     CONFIG_NAME,
     PYTORCH_WEIGHTS_NAME,
@@ -45,6 +47,7 @@
 )
 from tests.testing_constants import TOKEN
 
+from .testing_constants import ENDPOINT_STAGING, OTHER_TOKEN
 from .testing_utils import (
     DUMMY_MODEL_ID,
     DUMMY_MODEL_ID_PINNED_SHA1,
@@ -56,6 +59,7 @@
     SAMPLE_DATASET_IDENTIFIER,
     OfflineSimulationMode,
     offline,
+    repo_name,
     with_production_testing,
     xfail_on_windows,
 )
@@ -436,15 +440,28 @@ def test_download_from_a_gated_repo_with_hf_hub_download(self):
         Regression test for #1121.
         https://github.com/huggingface/huggingface_hub/pull/1121
         """
+        # Create a gated repo on the fly. Repo is created by "other user" so that the
+        # usual CI user don't have access to it.
+        api = HfApi(token=OTHER_TOKEN)
+        repo_url = api.create_repo(
+            repo_id="gated_repo_for_huggingface_hub_ci", exist_ok=True
+        )
+        requests.put(
+            f"{repo_url.endpoint}/api/models/{repo_url.repo_id}/settings",
+            headers=api._build_hf_headers(),
+            json={"gated": True},
+        ).raise_for_status()
+
+        # Cannot download file as repo is gated
         with SoftTemporaryDirectory() as tmpdir:
             with self.assertRaisesRegex(
                 GatedRepoError,
                 "Access to model .* is restricted and you are not in the authorized"
                 " list",
             ):
                 hf_hub_download(
-                    repo_id="datasets_server_org/gated_repo_for_huggingface_hub_ci",
-                    filename="config.json",
+                    repo_id=repo_url.repo_id,
+                    filename=".gitattributes",
                     use_auth_token=TOKEN,
                     cache_dir=tmpdir,
                 )
@@ -460,33 +477,48 @@ class StagingCachedDownloadOnAwfulFilenamesTest(unittest.TestCase):
     """
 
     cache_dir: Path
-    repo_id = "valid_org/repo_with_awful_filename"
     subfolder = "subfolder/to?"
     filename = "awful?filename%you:should,never.give"
-    filepath = "subfolder/to?/awful?filename%you:should,never.give"
-    expected_url = "https://hub-ci.huggingface.co/valid_org/repo_with_awful_filename/resolve/main/subfolder/to%3F/awful%3Ffilename%25you%3Ashould%2Cnever.give"
+    filepath = f"subfolder/to?/{filename}"
+
+    @classmethod
+    def setUpClass(cls):
+        cls.api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN)
+        cls.repo_url = cls.api.create_repo(repo_id=repo_name("awful_filename"))
+        cls.expected_resolve_url = f"{cls.repo_url}/resolve/main/subfolder/to%3F/awful%3Ffilename%25you%3Ashould%2Cnever.give"
+        cls.api.upload_file(
+            path_or_fileobj=b"content",
+            path_in_repo=cls.filepath,
+            repo_id=cls.repo_url.repo_id,
+        )
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cls.api.delete_repo(repo_id=cls.repo_url.repo_id)
 
     def test_hf_hub_url_on_awful_filepath(self):
-        self.assertEqual(hf_hub_url(self.repo_id, self.filepath), self.expected_url)
+        self.assertEqual(
+            hf_hub_url(self.repo_url.repo_id, self.filepath), self.expected_resolve_url
+        )
 
     def test_hf_hub_url_on_awful_subfolder_and_filename(self):
         self.assertEqual(
-            hf_hub_url(self.repo_id, self.filename, subfolder=self.subfolder),
-            self.expected_url,
+            hf_hub_url(self.repo_url.repo_id, self.filename, subfolder=self.subfolder),
+            self.expected_resolve_url,
         )
 
     @xfail_on_windows(reason="Windows paths cannot contain a '?'.")
     def test_hf_hub_download_on_awful_filepath(self):
         local_path = hf_hub_download(
-            self.repo_id, self.filepath, cache_dir=self.cache_dir
+            self.repo_url.repo_id, self.filepath, cache_dir=self.cache_dir
         )
         # Local path is not url-encoded
         self.assertTrue(local_path.endswith(self.filepath))
 
     @xfail_on_windows(reason="Windows paths cannot contain a '?'.")
     def test_hf_hub_download_on_awful_subfolder_and_filename(self):
         local_path = hf_hub_download(
-            self.repo_id,
+            self.repo_url.repo_id,
             self.filename,
             subfolder=self.subfolder,
             cache_dir=self.cache_dir,

tests/test_hf_api.py

@@ -82,6 +82,7 @@
     ENDPOINT_STAGING,
     ENDPOINT_STAGING_BASIC_AUTH,
     FULL_NAME,
+    OTHER_TOKEN,
     TOKEN,
     USER,
 )
@@ -833,8 +834,11 @@ def test_create_commit_create_pr_against_branch(self):
 
     @retry_endpoint
     def test_create_commit_create_pr_on_foreign_repo(self):
-        # Repo on which we don't have right
+        # Create a repo with another user. The normal CI user don't have rights on it.
         # We must be able to create a PR on it
+        foreign_api = HfApi(token=OTHER_TOKEN)
+        foreign_repo_url = foreign_api.create_repo(repo_id=repo_name("repo-for-hfh-ci"))
+
         self._api.create_commit(
             operations=[
                 CommitOperationAdd(
@@ -845,10 +849,12 @@ def test_create_commit_create_pr_on_foreign_repo(self):
                 ),
             ],
             commit_message="PR on foreign repo",
-            repo_id="datasets_server_org/repo_for_huggingface_hub_ci_with_prs",
+            repo_id=foreign_repo_url.repo_id,
             create_pr=True,
         )
 
+        foreign_api.delete_repo(repo_id=foreign_repo_url.repo_id)
+
     @retry_endpoint
     def test_create_commit(self):
         for private in (False, True):
@@ -2399,12 +2405,18 @@ def test_like_twice(self) -> None:
         self.api.delete_repo(repo_id, token=TOKEN)
 
     def test_list_liked_repos_no_auth(self) -> None:
+        # Create a list 1 liked repo
+        liked_repo_name = "repo-that-is-liked-public"
+        repo_url = self.api.create_repo(liked_repo_name, exist_ok=True, token=TOKEN)
+        self.api.like(repo_url.repo_id, token=TOKEN)
+
+        # Fetch liked repos without auth
         likes = self.api.list_liked_repos(USER)
         self.assertEqual(likes.user, USER)
         self.assertGreater(
             len(likes.models) + len(likes.datasets) + len(likes.spaces), 0
         )
-        self.assertIn(f"{USER}/repo-that-is-liked-public", likes.models)
+        self.assertIn(repo_url.repo_id, likes.models)
 
     def test_list_likes_repos_auth_and_implicit_user(self) -> None:
         # User is implicit

tests/test_snapshot_download.py

@@ -28,6 +28,7 @@ class SnapshotDownloadTests(unittest.TestCase):
     _api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN)
 
     @classmethod
+    @retry_endpoint
     @expect_deprecation("set_access_token")
     def setUpClass(cls):
         """
@@ -36,22 +37,20 @@ def setUpClass(cls):
         cls._token = TOKEN
         cls._api.set_access_token(TOKEN)
 
-    @retry_endpoint
-    def setUp(self) -> None:
         if os.path.exists(REPO_NAME):
             rmtree_with_retry(REPO_NAME)
         logger.info(f"Does {REPO_NAME} exist: {os.path.exists(REPO_NAME)}")
 
         try:
-            self._api.delete_repo(repo_id=REPO_NAME)
+            cls._api.delete_repo(repo_id=REPO_NAME)
         except RepositoryNotFoundError:
             pass
-        self._api.create_repo(f"{USER}/{REPO_NAME}")
+        cls._api.create_repo(f"{USER}/{REPO_NAME}")
 
         repo = Repository(
             REPO_NAME,
             clone_from=f"{USER}/{REPO_NAME}",
-            use_auth_token=self._token,
+            use_auth_token=cls._token,
             git_user="ci",
             git_email="[email protected]",
         )
@@ -60,24 +59,25 @@ def setUp(self) -> None:
             with open("dummy_file.txt", "w+") as f:
                 f.write("v1")
 
-        self.first_commit_hash = repo.git_head_hash()
+        cls.first_commit_hash = repo.git_head_hash()
 
         with repo.commit("Add file to main branch"):
             with open("dummy_file.txt", "w+") as f:
                 f.write("v2")
             with open("dummy_file_2.txt", "w+") as f:
                 f.write("v3")
 
-        self.second_commit_hash = repo.git_head_hash()
+        cls.second_commit_hash = repo.git_head_hash()
 
         with repo.commit("Add file to other branch", branch="other"):
             with open("dummy_file_2.txt", "w+") as f:
                 f.write("v4")
 
-        self.third_commit_hash = repo.git_head_hash()
+        cls.third_commit_hash = repo.git_head_hash()
 
-    def tearDown(self) -> None:
-        self._api.delete_repo(repo_id=REPO_NAME)
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cls._api.delete_repo(repo_id=REPO_NAME)
         rmtree_with_retry(REPO_NAME)
 
     def test_download_model(self):