feat(transformers): add SmolVLM (#1093)

* add SmolVLM

* resolve comments

* resolve comments

* resolve comments

* add copyright

Author: hadipash

mindone/transformers/__init__.py

@@ -54,6 +54,7 @@
     AutoImageProcessor,
     AutoModel,
     AutoModelForCausalLM,
+    AutoModelForImageTextToText,
     AutoModelForMaskedLM,
     AutoProcessor,
 )
@@ -436,6 +437,12 @@
     SiglipTextModel,
     SiglipVisionModel,
 )
+from .models.smolvlm import (
+    SmolVLMForConditionalGeneration,
+    SmolVLMModel,
+    SmolVLMPreTrainedModel,
+    SmolVLMVisionTransformer,
+)
 from .models.speecht5 import (
     SpeechT5ForSpeechToSpeech,
     SpeechT5ForSpeechToText,

mindone/transformers/models/__init__.py

@@ -74,6 +74,7 @@
     qwen2_vl,
     roberta,
     siglip,
+    smolvlm,
     speecht5,
     starcoder2,
     switch_transformers,

mindone/transformers/models/auto/__init__.py

@@ -17,5 +17,5 @@
 from .configuration_auto import AutoConfig
 from .feature_extraction_auto import AutoFeatureExtractor
 from .image_processing_auto import AutoImageProcessor
-from .modeling_auto import AutoModel, AutoModelForCausalLM, AutoModelForMaskedLM
+from .modeling_auto import AutoModel, AutoModelForCausalLM, AutoModelForImageTextToText, AutoModelForMaskedLM
 from .processing_auto import AutoProcessor

mindone/transformers/models/auto/configuration_auto.py

@@ -100,6 +100,8 @@
         ("rembert", "RemBertConfig"),
         ("siglip", "SiglipConfig"),
         ("siglip_vision_model", "SiglipVisionConfig"),
+        ("smolvlm", "SmolVLMConfig"),
+        ("smolvlm_vision", "SmolVLMVisionConfig"),
         ("speecht5", "SpeechT5Config"),
         ("t5", "T5Config"),
         ("umt5", "UMT5Config"),
@@ -189,6 +191,8 @@
         ("rembert", "RemBERT"),
         ("siglip", "SigLIP"),
         ("siglip_vision_model", "SiglipVisionModel"),
+        ("smolvlm", "SmolVLM"),
+        ("smolvlm_vision", "SmolVLMVisionTransformer"),
         ("speecht5", "SpeechT5"),
         ("t5", "T5"),
         ("t5v1.1", "T5v1.1"),
@@ -249,6 +253,7 @@
         ("clip_text_model", "clip"),
         ("aria_text", "aria"),
         ("siglip_vision_model", "siglip"),
+        ("smolvlm_vision", "smolvlm"),
         ("chinese_clip_vision_model", "chinese_clip"),
         ("rt_detr_resnet", "rt_detr"),
     ]

mindone/transformers/models/auto/modeling_auto.py

@@ -97,6 +97,8 @@
         ("rembert", "RemBertModel"),
         ("siglip", "SiglipModel"),
         ("siglip_vision_model", "SiglipVisionModel"),
+        ("smolvlm", "SmolVLMModel"),
+        ("smolvlm_vision", "SmolVLMVisionTransformer"),
         ("speecht5", "SpeechT5Model"),
         ("t5", "T5Model"),
         ("umt5", "UMT5Model"),
@@ -290,6 +292,7 @@
         ("paligemma", "PaliGemmaForConditionalGeneration"),
         ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"),
         ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
+        ("smolvlm", "SmolVLMForConditionalGeneration"),
     ]
 )
 

mindone/transformers/models/smolvlm/__init__.py

@@ -0,0 +1,20 @@
+# coding=utf-8
+# Copyright 2025 the HuggingFace Inc. team. All rights reserved.
+# Written by Orr Zohar
+#
+# This code is adapted from https://github.com/huggingface/transformers
+# with modifications to run transformers on mindspore.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .modeling_smolvlm import *