feature (transformer): Swin (#1173)

* init

* licenses

* update test script

* correct import

* correct import

* example

* updates

* add an example in docstring

* fxi ci error

Author: wtomin

mindone/transformers/__init__.py

@@ -49,14 +49,92 @@
     AriaTextPreTrainedModel,
 )
 from .models.auto import (
+    MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
+    MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING,
+    MODEL_FOR_AUDIO_XVECTOR_MAPPING,
+    MODEL_FOR_BACKBONE_MAPPING,
+    MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING,
+    MODEL_FOR_CAUSAL_LM_MAPPING,
+    MODEL_FOR_CTC_MAPPING,
+    MODEL_FOR_DEPTH_ESTIMATION_MAPPING,
+    MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING,
+    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
+    MODEL_FOR_IMAGE_MAPPING,
+    MODEL_FOR_IMAGE_SEGMENTATION_MAPPING,
+    MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING,
+    MODEL_FOR_IMAGE_TO_IMAGE_MAPPING,
+    MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING,
+    MODEL_FOR_KEYPOINT_DETECTION_MAPPING,
+    MODEL_FOR_MASK_GENERATION_MAPPING,
+    MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING,
+    MODEL_FOR_MASKED_LM_MAPPING,
+    MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
+    MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
+    MODEL_FOR_OBJECT_DETECTION_MAPPING,
+    MODEL_FOR_PRETRAINING_MAPPING,
+    MODEL_FOR_QUESTION_ANSWERING_MAPPING,
+    MODEL_FOR_RETRIEVAL_MAPPING,
+    MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING,
+    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
+    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
+    MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
+    MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
+    MODEL_FOR_TEXT_ENCODING_MAPPING,
+    MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING,
+    MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING,
+    MODEL_FOR_TIME_SERIES_CLASSIFICATION_MAPPING,
+    MODEL_FOR_TIME_SERIES_REGRESSION_MAPPING,
+    MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
+    MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING,
+    MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING,
+    MODEL_FOR_VISION_2_SEQ_MAPPING,
+    MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING,
+    MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING,
+    MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING,
+    MODEL_MAPPING,
+    MODEL_WITH_LM_HEAD_MAPPING,
+    AutoBackbone,
     AutoConfig,
     AutoFeatureExtractor,
     AutoImageProcessor,
     AutoModel,
+    AutoModelForAudioClassification,
+    AutoModelForAudioFrameClassification,
+    AutoModelForAudioXVector,
     AutoModelForCausalLM,
+    AutoModelForCTC,
+    AutoModelForDepthEstimation,
+    AutoModelForDocumentQuestionAnswering,
+    AutoModelForImageClassification,
+    AutoModelForImageSegmentation,
     AutoModelForImageTextToText,
+    AutoModelForImageToImage,
+    AutoModelForInstanceSegmentation,
+    AutoModelForKeypointDetection,
+    AutoModelForMaskedImageModeling,
     AutoModelForMaskedLM,
+    AutoModelForMaskGeneration,
+    AutoModelForMultipleChoice,
+    AutoModelForNextSentencePrediction,
+    AutoModelForObjectDetection,
+    AutoModelForPreTraining,
+    AutoModelForQuestionAnswering,
+    AutoModelForSemanticSegmentation,
+    AutoModelForSeq2SeqLM,
+    AutoModelForSequenceClassification,
+    AutoModelForSpeechSeq2Seq,
+    AutoModelForTableQuestionAnswering,
+    AutoModelForTextEncoding,
+    AutoModelForTextToSpectrogram,
+    AutoModelForTextToWaveform,
+    AutoModelForTokenClassification,
+    AutoModelForUniversalSegmentation,
+    AutoModelForVideoClassification,
     AutoModelForVision2Seq,
+    AutoModelForVisualQuestionAnswering,
+    AutoModelForZeroShotImageClassification,
+    AutoModelForZeroShotObjectDetection,
+    AutoModelWithLMHead,
     AutoProcessor,
 )
 from .models.bart import (
@@ -470,6 +548,13 @@
     Starcoder2Model,
     Starcoder2PreTrainedModel,
 )
+from .models.swin import (
+    SwinBackbone,
+    SwinForImageClassification,
+    SwinForMaskedImageModeling,
+    SwinModel,
+    SwinPreTrainedModel,
+)
 from .models.switch_transformers import (
     SwitchTransformersEncoderModel,
     SwitchTransformersForConditionalGeneration,

mindone/transformers/models/auto/__init__.py

@@ -18,10 +18,88 @@
 from .feature_extraction_auto import AutoFeatureExtractor
 from .image_processing_auto import AutoImageProcessor
 from .modeling_auto import (
+    MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
+    MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING,
+    MODEL_FOR_AUDIO_XVECTOR_MAPPING,
+    MODEL_FOR_BACKBONE_MAPPING,
+    MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING,
+    MODEL_FOR_CAUSAL_LM_MAPPING,
+    MODEL_FOR_CTC_MAPPING,
+    MODEL_FOR_DEPTH_ESTIMATION_MAPPING,
+    MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING,
+    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
+    MODEL_FOR_IMAGE_MAPPING,
+    MODEL_FOR_IMAGE_SEGMENTATION_MAPPING,
+    MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING,
+    MODEL_FOR_IMAGE_TO_IMAGE_MAPPING,
+    MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING,
+    MODEL_FOR_KEYPOINT_DETECTION_MAPPING,
+    MODEL_FOR_MASK_GENERATION_MAPPING,
+    MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING,
+    MODEL_FOR_MASKED_LM_MAPPING,
+    MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
+    MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
+    MODEL_FOR_OBJECT_DETECTION_MAPPING,
+    MODEL_FOR_PRETRAINING_MAPPING,
+    MODEL_FOR_QUESTION_ANSWERING_MAPPING,
+    MODEL_FOR_RETRIEVAL_MAPPING,
+    MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING,
+    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
+    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
+    MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
+    MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
+    MODEL_FOR_TEXT_ENCODING_MAPPING,
+    MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING,
+    MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING,
+    MODEL_FOR_TIME_SERIES_CLASSIFICATION_MAPPING,
+    MODEL_FOR_TIME_SERIES_REGRESSION_MAPPING,
+    MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
+    MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING,
+    MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING,
+    MODEL_FOR_VISION_2_SEQ_MAPPING,
+    MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING,
+    MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING,
+    MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING,
+    MODEL_MAPPING,
+    MODEL_WITH_LM_HEAD_MAPPING,
+    AutoBackbone,
     AutoModel,
+    AutoModelForAudioClassification,
+    AutoModelForAudioFrameClassification,
+    AutoModelForAudioXVector,
     AutoModelForCausalLM,
+    AutoModelForCTC,
+    AutoModelForDepthEstimation,
+    AutoModelForDocumentQuestionAnswering,
+    AutoModelForImageClassification,
+    AutoModelForImageSegmentation,
     AutoModelForImageTextToText,
+    AutoModelForImageToImage,
+    AutoModelForInstanceSegmentation,
+    AutoModelForKeypointDetection,
+    AutoModelForMaskedImageModeling,
     AutoModelForMaskedLM,
+    AutoModelForMaskGeneration,
+    AutoModelForMultipleChoice,
+    AutoModelForNextSentencePrediction,
+    AutoModelForObjectDetection,
+    AutoModelForPreTraining,
+    AutoModelForQuestionAnswering,
+    AutoModelForSemanticSegmentation,
+    AutoModelForSeq2SeqLM,
+    AutoModelForSequenceClassification,
+    AutoModelForSpeechSeq2Seq,
+    AutoModelForTableQuestionAnswering,
+    AutoModelForTextEncoding,
+    AutoModelForTextToSpectrogram,
+    AutoModelForTextToWaveform,
+    AutoModelForTokenClassification,
+    AutoModelForUniversalSegmentation,
+    AutoModelForVideoClassification,
     AutoModelForVision2Seq,
+    AutoModelForVisualQuestionAnswering,
+    AutoModelForZeroShotImageClassification,
+    AutoModelForZeroShotObjectDetection,
+    AutoModelWithLMHead,
 )
 from .processing_auto import AutoProcessor

mindone/transformers/models/auto/configuration_auto.py

@@ -100,6 +100,7 @@
         ("roberta", "RobertaConfig"),
         ("recurrent_gemma", "RecurrentGemmaConfig"),
         ("rembert", "RemBertConfig"),
+        ("swin", "SwinConfig"),
         ("siglip", "SiglipConfig"),
         ("siglip_vision_model", "SiglipVisionConfig"),
         ("smolvlm", "SmolVLMConfig"),
@@ -193,6 +194,7 @@
         ("qwen2_vl", "Qwen2VL"),
         ("recurrent_gemma", "RecurrentGemma"),
         ("rembert", "RemBERT"),
+        ("swin", "Swin Transformer"),
         ("siglip", "SigLIP"),
         ("siglip_vision_model", "SiglipVisionModel"),
         ("smolvlm", "SmolVLM"),

mindone/transformers/models/auto/modeling_auto.py

@@ -284,7 +284,11 @@
         ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
     ]
 )
-
+MODEL_FOR_RETRIEVAL_MAPPING_NAMES = OrderedDict(
+    [
+        # ("colpali", "ColPaliForRetrieval"),
+    ]
+)
 MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES = OrderedDict(
     [
         ("aria", "AriaForConditionalGeneration"),
@@ -563,6 +567,7 @@
 MODEL_FOR_BACKBONE_MAPPING_NAMES = OrderedDict(
     [
         ("hiera", "HieraBackbone"),
+        ("swin", "SwinBackbone"),
     ]
 )
 
@@ -649,6 +654,7 @@
 MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING = _LazyAutoMapping(
     CONFIG_MAPPING_NAMES, MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES
 )
+MODEL_FOR_RETRIEVAL_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_RETRIEVAL_MAPPING_NAMES)
 MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING = _LazyAutoMapping(
     CONFIG_MAPPING_NAMES, MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES
 )

mindone/transformers/models/swin/__init__.py

@@ -0,0 +1,7 @@
+from .modeling_swin import (
+    SwinBackbone,
+    SwinForImageClassification,
+    SwinForMaskedImageModeling,
+    SwinModel,
+    SwinPreTrainedModel,
+)