Update to transfomers an add tests for Gemmma 3 and Qwen3 (#1965)

- Updates transformers dependency for tests to 4.57 version (which enables newer models)
- Add conversion tests Gemma 3 and Qwen3

Author: jordimas

python/ctranslate2/converters/fairseq.py

@@ -146,7 +146,9 @@ def _load(self):
             import_user_module(argparse.Namespace(user_dir=self._user_dir))
 
         with torch.no_grad():
-            checkpoint = checkpoint_utils.load_checkpoint_to_cpu(self._model_path)
+            checkpoint = torch.load(
+                self._model_path, map_location=torch.device("cpu"), weights_only=False
+            )
             args = checkpoint["args"] or checkpoint["cfg"]["model"]
 
             args.data = self._data_dir

python/ctranslate2/converters/opennmt_py.py

@@ -174,7 +174,9 @@ def __init__(self, model_path: str):
     def _load(self):
         import torch
 
-        checkpoint = torch.load(self._model_path, map_location="cpu")
+        checkpoint = torch.load(
+            self._model_path, map_location="cpu", weights_only=False
+        )
 
         src_vocabs, tgt_vocabs = get_vocabs(checkpoint["vocab"])
 

python/ctranslate2/specs/transformer_spec.py

@@ -109,7 +109,7 @@ def __init__(
         quant_type: Optional[common_spec.Quantization] = None,
         quant_group_size: Optional[int] = None,
         quant_bits: Optional[int] = None,
-        qk_norm: Optional[bool] = False,
+        qk_norm: bool = False,
     ):
         """Initializes a Transformer decoder specification.
 
@@ -562,7 +562,7 @@ def from_config(
         quant_type: Optional[common_spec.Quantization] = None,
         quant_group_size: Optional[int] = None,
         quant_bits: Optional[int] = None,
-        qk_norm: Optional[bool] = False,
+        qk_norm: bool = False,
     ):
         """Creates a Transformer decoder model specification.
 

python/tests/requirements.txt

@@ -1,6 +1,6 @@
-transformers==4.41.*;platform_system=='Linux'
-fairseq==0.12.2;platform_system=='Linux' or platform_system=='Darwin'
-OpenNMT-py==2.2.*;platform_system=='Linux' or platform_system=='Darwin'
+transformers==4.57.*;platform_system=='Linux'
+git+https://github.com/facebookresearch/fairseq.git@main; sys_platform == 'linux' or sys_platform == 'darwin'
+OpenNMT-py==2.2.*;sys_platform=='linux' or sys_platform=='darwin'
 pytest
-wurlitzer==3.0.*;platform_system=='Linux'
-torch==2.4.1
+wurlitzer==3.0.*;sys_platform=='linux'
+torch==2.6

python/tests/test_transformers.py

@@ -21,7 +21,7 @@ def clear_transformers_cache_in_ci():
     import transformers
 
     if os.environ.get("CI") == "true":
-        shutil.rmtree(transformers.utils.default_cache_path)
+        shutil.rmtree(transformers.utils.default_cache_path, ignore_errors=True)
 
 
 _TRANSFORMERS_TRANSLATION_TESTS = [
@@ -194,6 +194,21 @@ def test_transformers_translation(
         "Hello , ĠI Ġam Ġa Ġnew bie Ġin Ġthe Ġworld Ġof Ġweb Ġdesign Ġand ĠI Ġam "
         "Ġlooking Ġfor Ġa Ġweb Ġdeveloper",
     ),
+    (
+        "jordimas/gemma-3-1b-it",
+        "<bos> Which ▁city ▁hosted ▁the ▁Olympic ▁Games ▁in ▁ 1 9 9 2 ?",
+        50,
+        "Which ▁city ▁hosted ▁the ▁Olympic ▁Games ▁in ▁ 1 9 9 2 ? \n\n"
+        " The ▁answer ▁is ▁** Barcelona **. \n",
+    ),
+    (
+        "Qwen/Qwen3-0.6B",
+        "<|im_start|> user Ċ What Ġis Ġthe Ġcapital Ġof ĠPortugal ? Ġ/ no _th ink Ċ <|im_end|> Ċ "
+        "<|im_start|> assistant Ċ",
+        50,
+        "<|im_start|> user Ċ What Ġis Ġthe Ġcapital Ġof ĠPortugal ? Ġ/ no _th ink Ċ <|im_end|> Ċ "
+        "<|im_start|> assistant Ċ <think> ĊĊ </think> ĊĊ The Ġcapital Ġof ĠPortugal Ġis ĠLisbon .",
+    ),
 ]