Fix GIL=0 segfault and Add GIL=0 compat for regex paths

src/transformers/models/bart/tokenization_bart.py

@@ -18,10 +18,9 @@
 from functools import lru_cache
 from typing import Optional
 
-import regex as re
-
 from ...tokenization_utils import AddedToken, PreTrainedTokenizer
 from ...utils import logging
+from ...utils.safe import regex as re
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/bertweet/tokenization_bertweet.py

@@ -21,10 +21,9 @@
 from shutil import copyfile
 from typing import Optional
 
-import regex
-
 from ...tokenization_utils import PreTrainedTokenizer
 from ...utils import logging
+from ...utils.safe import regex
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/blenderbot/tokenization_blenderbot.py

@@ -19,10 +19,9 @@
 from functools import lru_cache
 from typing import Optional
 
-import regex as re
-
 from ...tokenization_utils import AddedToken, PreTrainedTokenizer
 from ...utils import logging
+from ...utils.safe import regex as re
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/blenderbot_small/tokenization_blenderbot_small.py

@@ -18,10 +18,9 @@
 import os
 from typing import Optional
 
-import regex as re
-
 from ...tokenization_utils import PreTrainedTokenizer
 from ...utils import logging
+from ...utils.safe import regex as re
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/clip/tokenization_clip.py

@@ -20,10 +20,9 @@
 from functools import lru_cache
 from typing import Optional
 
-import regex as re
-
 from ...tokenization_utils import AddedToken, PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
 from ...utils import logging
+from ...utils.safe import regex as re
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/clvp/tokenization_clvp.py

@@ -19,10 +19,9 @@
 from functools import lru_cache
 from typing import Optional
 
-import regex as re
-
 from ...tokenization_utils import AddedToken, PreTrainedTokenizer
 from ...utils import logging
+from ...utils.safe import regex as re
 from .number_normalizer import EnglishNormalizer
 
 

src/transformers/models/codegen/tokenization_codegen.py

@@ -20,9 +20,9 @@
 from typing import TYPE_CHECKING, Optional, Union
 
 import numpy as np
-import regex as re
 
 from ...utils import logging, to_py_obj
+from ...utils.safe import regex as re
 
 
 if TYPE_CHECKING:

src/transformers/models/ctrl/tokenization_ctrl.py

@@ -18,10 +18,9 @@
 import os
 from typing import Optional
 
-import regex as re
-
 from ...tokenization_utils import PreTrainedTokenizer
 from ...utils import logging
+from ...utils.safe import regex as re
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/deberta/tokenization_deberta.py

@@ -18,10 +18,9 @@
 import os
 from typing import Optional
 
-import regex as re
-
 from ...tokenization_utils import AddedToken, PreTrainedTokenizer
 from ...utils import logging
+from ...utils.safe import regex as re
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/deepseek_vl/convert_deepseek_vl_weights_to_hf.py

@@ -18,7 +18,6 @@
 import os
 from typing import Optional
 
-import regex as re
 import torch
 from accelerate import init_empty_weights
 from huggingface_hub import snapshot_download
@@ -33,6 +32,7 @@
     DeepseekVLProcessor,
 )
 from transformers.image_utils import IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD
+from transformers.utils.safe import regex as re
 
 
 # fmt: off

src/transformers/models/deepseek_vl_hybrid/convert_deepseek_vl_hybrid_weights_to_hf.py

@@ -18,7 +18,6 @@
 import os
 from typing import Optional
 
-import regex as re
 import torch
 from accelerate import init_empty_weights
 from huggingface_hub import snapshot_download
@@ -39,6 +38,7 @@
     OPENAI_CLIP_STD,
     PILImageResampling,
 )
+from transformers.utils.safe import regex as re
 
 
 # fmt: off

src/transformers/models/deprecated/jukebox/tokenization_jukebox.py

@@ -22,12 +22,12 @@
 from typing import Any, Optional, Union
 
 import numpy as np
-import regex
 
 from ....tokenization_utils import AddedToken, PreTrainedTokenizer
 from ....tokenization_utils_base import BatchEncoding
 from ....utils import TensorType, is_torch_available, logging
 from ....utils.generic import _is_numpy
+from ...utils.safe import regex
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/deprecated/tapex/tokenization_tapex.py

@@ -20,12 +20,11 @@
 from functools import lru_cache
 from typing import Optional, Union
 
-import regex as re
-
 from ....file_utils import ExplicitEnum, PaddingStrategy, TensorType, add_end_docstrings, is_pandas_available
 from ....tokenization_utils import AddedToken, PreTrainedTokenizer
 from ....tokenization_utils_base import ENCODE_KWARGS_DOCSTRING, BatchEncoding, TextInput, TruncationStrategy
 from ....utils import logging
+from ...utils.safe import regex as re
 
 
 if is_pandas_available():

src/transformers/models/depth_pro/convert_depth_pro_weights_to_hf.py

@@ -17,7 +17,6 @@
 import os
 from typing import Optional
 
-import regex as re
 import torch
 from huggingface_hub import hf_hub_download
 
@@ -26,6 +25,7 @@
     DepthProForDepthEstimation,
     DepthProImageProcessorFast,
 )
+from transformers.utils.safe import regex as re
 
 
 # fmt: off

src/transformers/models/fastspeech2_conformer/tokenization_fastspeech2_conformer.py

@@ -18,10 +18,9 @@
 import os
 from typing import Optional
 
-import regex
-
 from ...tokenization_utils import PreTrainedTokenizer
 from ...utils import logging, requires_backends
+from ...utils.safe import regex
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/got_ocr2/convert_got_ocr2_weights_to_hf.py

@@ -18,7 +18,6 @@
 import os
 from typing import Optional
 
-import regex as re
 import torch
 from huggingface_hub import snapshot_download
 from safetensors import safe_open
@@ -33,6 +32,7 @@
 )
 from transformers.convert_slow_tokenizer import TikTokenConverter
 from transformers.tokenization_utils import AddedToken
+from transformers.utils.safe import regex as re
 
 
 if is_vision_available():

src/transformers/models/gpt2/tokenization_gpt2.py

@@ -19,10 +19,9 @@
 from functools import lru_cache
 from typing import Optional
 
-import regex as re
-
 from ...tokenization_utils import AddedToken, PreTrainedTokenizer
 from ...utils import logging
+from ...utils.safe import regex as re
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/gpt_oss/convert_gpt_oss_weights_to_hf.py

@@ -19,7 +19,6 @@
 from pathlib import Path
 from typing import Optional
 
-import regex as re
 import tiktoken
 import torch
 from safetensors.torch import load_file as safe_load
@@ -31,6 +30,7 @@
     PreTrainedTokenizerFast,
 )
 from transformers.convert_slow_tokenizer import TikTokenConverter
+from transformers.utils.safe import regex as re
 
 
 # fmt: off

src/transformers/models/layoutlmv3/tokenization_layoutlmv3.py

@@ -19,8 +19,6 @@
 from functools import lru_cache
 from typing import Optional, Union
 
-import regex as re
-
 from ...tokenization_utils import AddedToken, PreTrainedTokenizer
 from ...tokenization_utils_base import (
     BatchEncoding,
@@ -31,6 +29,7 @@
     TruncationStrategy,
 )
 from ...utils import PaddingStrategy, TensorType, add_end_docstrings, logging
+from ...utils.safe import regex as re
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/led/tokenization_led.py

@@ -19,11 +19,10 @@
 from functools import lru_cache
 from typing import Optional, Union
 
-import regex as re
-
 from ...tokenization_utils import AddedToken, PreTrainedTokenizer
 from ...tokenization_utils_base import BatchEncoding, EncodedInput
 from ...utils import PaddingStrategy, logging
+from ...utils.safe import regex as re
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/longformer/tokenization_longformer.py

@@ -18,10 +18,9 @@
 from functools import lru_cache
 from typing import Optional
 
-import regex as re
-
 from ...tokenization_utils import AddedToken, PreTrainedTokenizer
 from ...utils import logging
+from ...utils.safe import regex as re
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/luke/tokenization_luke.py

@@ -22,7 +22,6 @@
 from typing import Optional, Union
 
 import numpy as np
-import regex as re
 
 from ...tokenization_utils import PreTrainedTokenizer
 from ...tokenization_utils_base import (
@@ -38,6 +37,7 @@
     to_py_obj,
 )
 from ...utils import add_end_docstrings, is_torch_tensor, logging
+from ...utils.safe import regex as re
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/markuplm/tokenization_markuplm.py

@@ -19,8 +19,6 @@
 from functools import lru_cache
 from typing import Optional, Union
 
-import regex as re
-
 from ...file_utils import PaddingStrategy, TensorType, add_end_docstrings
 from ...tokenization_utils import AddedToken, PreTrainedTokenizer
 from ...tokenization_utils_base import (
@@ -33,6 +31,7 @@
     TruncationStrategy,
 )
 from ...utils import logging
+from ...utils.safe import regex as re
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/mllama/convert_mllama_weights_to_hf.py

@@ -19,7 +19,6 @@
 import os
 from typing import Optional
 
-import regex as re
 import torch
 import torch.nn.functional as F
 
@@ -33,6 +32,7 @@
 from transformers.convert_slow_tokenizer import TikTokenConverter
 from transformers.models.mllama.configuration_mllama import MllamaTextConfig, MllamaVisionConfig
 from transformers.models.mllama.image_processing_mllama import get_all_supported_aspect_ratios
+from transformers.utils.safe import regex as re
 
 
 # fmt: off

src/transformers/models/mvp/tokenization_mvp.py

@@ -18,10 +18,9 @@
 from functools import lru_cache
 from typing import Optional
 
-import regex as re
-
 from ...tokenization_utils import AddedToken, PreTrainedTokenizer
 from ...utils import logging
+from ...utils.safe import regex as re
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/pixtral/convert_pixtral_weights_to_hf.py

@@ -15,7 +15,6 @@
 import json
 import os
 
-import regex as re
 import torch
 from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
 from safetensors.torch import load_file as safe_load_file
@@ -28,6 +27,7 @@
     PixtralProcessor,
     PixtralVisionConfig,
 )
+from transformers.utils.safe import regex as re
 
 
 """

src/transformers/models/qwen2/tokenization_qwen2.py

@@ -20,10 +20,9 @@
 from functools import lru_cache
 from typing import Optional
 
-import regex as re
-
 from ...tokenization_utils import AddedToken, PreTrainedTokenizer
 from ...utils import logging
+from ...utils.safe import regex as re
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/recurrent_gemma/convert_recurrent_gemma_to_hf.py

@@ -19,6 +19,7 @@
 from accelerate import init_empty_weights
 
 from transformers import GemmaTokenizer, RecurrentGemmaConfig, RecurrentGemmaForCausalLM
+from transformers.utils.safe import regex as re
 
 
 try:
@@ -30,8 +31,6 @@
     )
     GemmaTokenizerFast = None
 
-import regex as re
-
 
 """
 Sample usage:
@@ -63,6 +62,7 @@
     num_hidden_layers=26,
 )
 
+
 gemma_7b_config = RecurrentGemmaConfig()
 
 CONFIG_MAPPING = {"2B": gemma_2b_config, "7B": gemma_7b_config}