[compat] Expand test suite to full transformers v5 (#3615)

* Expand test suite to full transformers v5

* Specify _endpoint=None in tests as required by newer huggingface_hub

* Ensure test_push_to_hub works for older and newer hf_hub

Author: tomaarsen

.github/workflows/tests.yml

@@ -75,7 +75,7 @@ jobs:
           # The --break-system-packages flag is used to allow pip to upgrade or install packages
           # even in environments where system-managed packages might otherwise block such operations.
           python -m pip install --upgrade pip --break-system-packages
-          python -m pip install '.[train, dev]' 'transformers>=5.0.0rc0'
+          python -m pip install '.[train, dev]' 'transformers>=5.0.0'
 
       - name: Install model2vec
         run: python -m pip install model2vec

tests/cross_encoder/test_cross_encoder.py

@@ -275,17 +275,19 @@ def mock_create_repo(self, repo_id, **kwargs):
     def mock_upload_folder(self, **kwargs):
         nonlocal mock_upload_folder_kwargs
         mock_upload_folder_kwargs = kwargs
-        if kwargs.get("revision") is None:
-            revision = "123456"
-        else:
-            revision = "678901"
-        return CommitInfo(
-            commit_url=f"https://huggingface.co/{kwargs.get('repo_id')}/commit/{revision}",
-            commit_message="commit_message",
-            commit_description="commit_description",
-            oid="oid",
-            pr_url=f"https://huggingface.co/{kwargs.get('repo_id')}/discussions/123",
-        )
+        commit_hash = "123456" if kwargs.get("revision") is None else "678901"
+        commit_info_kwargs = {
+            "commit_url": f"https://huggingface.co/{kwargs.get('repo_id')}/commit/{commit_hash}",
+            "commit_message": "commit_message",
+            "commit_description": "commit_description",
+            "oid": "oid",
+            "pr_url": f"https://huggingface.co/{kwargs.get('repo_id')}/discussions/123",
+        }
+        try:
+            return CommitInfo(**commit_info_kwargs)
+        except TypeError:
+            # Required as of https://github.com/huggingface/huggingface_hub/pull/3679
+            return CommitInfo(**commit_info_kwargs, _endpoint=None)
 
     def mock_create_branch(self, repo_id, branch, revision=None, **kwargs):
         return None

tests/test_sentence_transformer.py

@@ -120,20 +120,18 @@ def mock_create_repo(self, repo_id, **kwargs):
     def mock_upload_folder(self, **kwargs):
         nonlocal mock_upload_folder_kwargs
         mock_upload_folder_kwargs = kwargs
-        if kwargs.get("revision") is None:
-            return CommitInfo(
-                commit_url=f"https://huggingface.co/{kwargs.get('repo_id')}/commit/123456",
-                commit_message="commit_message",
-                commit_description="commit_description",
-                oid="oid",
-            )
-        else:
-            return CommitInfo(
-                commit_url=f"https://huggingface.co/{kwargs.get('repo_id')}/commit/678901",
-                commit_message="commit_message",
-                commit_description="commit_description",
-                oid="oid",
-            )
+        commit_hash = "123456" if kwargs.get("revision") is None else "678901"
+        commit_info_kwargs = {
+            "commit_url": f"https://huggingface.co/{kwargs.get('repo_id')}/commit/{commit_hash}",
+            "commit_message": "commit_message",
+            "commit_description": "commit_description",
+            "oid": "oid",
+        }
+        try:
+            return CommitInfo(**commit_info_kwargs)
+        except TypeError:
+            # Required as of https://github.com/huggingface/huggingface_hub/pull/3679
+            return CommitInfo(**commit_info_kwargs, _endpoint=None)
 
     def mock_create_branch(self, repo_id, branch, revision=None, **kwargs):
         return None
@@ -473,10 +471,6 @@ def test_prompt_length_calculation(
     assert model._prompt_length_mapping == {("Prompt: ", "query"): only_prompt_length}
 
 
-@pytest.mark.skipif(
-    parse(transformers_version) == Version("5.0.0rc01"),
-    reason="Transformers 5.0.0rc01 has a bug with saving models modified with model.half().",
-)
 @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA must be available to test float16 support.")
 def test_load_with_torch_dtype(stsb_bert_tiny_model: SentenceTransformer) -> None:
     model = stsb_bert_tiny_model