feat(transformers/pipelines): add object detection, zero shot classification, and zero shot object detection (#1193)

* feat(transformers/pipelines): add object detection, zero shot classification, and zero shot object detection

* fix bugs

* update to transformers v4.53

* add YOLOS and OWL-ViT models

* fixes

* fix iterator

Author: hadipash

mindone/transformers/__init__.py

@@ -476,6 +476,15 @@
     OPTModel,
     OPTPreTrainedModel,
 )
+from .models.owlvit import (
+    OwlViTForObjectDetection,
+    OwlViTImageProcessor,
+    OwlViTModel,
+    OwlViTPreTrainedModel,
+    OwlViTProcessor,
+    OwlViTTextModel,
+    OwlViTVisionModel,
+)
 from .models.paligemma import PaliGemmaForConditionalGeneration, PaliGemmaPreTrainedModel
 from .models.persimmon import (
     PersimmonForCausalLM,
@@ -638,6 +647,7 @@
     XLMRobertaXLModel,
     XLMRobertaXLPreTrainedModel,
 )
+from .models.yolos import YolosForObjectDetection, YolosImageProcessor, YolosModel, YolosPreTrainedModel
 from .pipelines import TextGenerationPipeline, pipeline
 from .processing_utils import ProcessorMixin
 

mindone/transformers/models/__init__.py

@@ -71,6 +71,7 @@
     mpt,
     mvp,
     opt,
+    owlvit,
     paligemma,
     persimmon,
     phi,
@@ -93,6 +94,7 @@
     vits,
     wav2vec2,
     xlm_roberta,
+    yolos,
 )
 
 if version.parse(transformers.__version__) >= version.parse("4.51.0"):

mindone/transformers/models/auto/configuration_auto.py

@@ -93,6 +93,7 @@
         ("mt5", "MT5Config"),
         ("megatron-bert", "MegatronBertConfig"),
         ("mixtral", "MixtralConfig"),
+        ("owlvit", "OwlViTConfig"),
         ("paligemma", "PaliGemmaConfig"),
         ("phi", "PhiConfig"),
         ("phi3", "Phi3Config"),
@@ -120,6 +121,7 @@
         ("whisper", "WhisperConfig"),
         ("xlm-roberta", "XLMRobertaConfig"),
         ("xlm-roberta-xl", "XLMRobertaXLConfig"),
+        ("yolos", "YolosConfig"),
         ("cohere2", "Cohere2Config"),
     ]
 )
@@ -195,6 +197,7 @@
         ("megatron-bert", "Megatron-BERT"),
         ("mistral", "Mistral"),
         ("mixtral", "Mixtral"),
+        ("owlvit", "OWL-ViT"),
         ("paligemma", "PaliGemma"),
         ("phi", "Phi"),
         ("phi3", "Phi3"),
@@ -222,6 +225,7 @@
         ("opt", "OPT"),
         ("xlm-roberta", "XLM-RoBERTa"),
         ("xlm-roberta-xl", "XLM-RoBERTa-XL"),
+        ("yolos", "YOLOS"),
         ("cohere2", "Cohere2"),
     ]
 )

mindone/transformers/models/auto/image_processing_auto.py

@@ -60,8 +60,10 @@
             ("llava_next", ("LlavaNextImageProcessor",)),
             ("llava_next_video", ("LlavaNextVideoImageProcessor",)),
             ("llava_onevision", ("LlavaOnevisionImageProcessor",)),
+            ("owlvit", ("OwlViTImageProcessor",)),
             ("segformer", ("SegformerImageProcessor",)),
             ("siglip", ("SiglipImageProcessor", "SiglipImageProcessorFast")),
+            ("yolos", ("YolosImageProcessor",)),
         ]
     )
 

mindone/transformers/models/auto/modeling_auto.py

@@ -92,6 +92,7 @@
         ("mt5", "MT5Model"),
         ("megatron-bert", "MegatronBertModel"),
         ("mixtral", "MixtralModel"),
+        ("owlvit", "OwlViTModel"),
         ("phi", "PhiModel"),
         ("phi3", "Phi3Model"),
         ("qwen2", "Qwen2Model"),
@@ -114,6 +115,7 @@
         ("whisper", "WhisperModel"),
         ("xlm-roberta", "XLMRobertaModel"),
         ("xlm-roberta-xl", "XLMRobertaXLModel"),
+        ("yolos", "YolosModel"),
         ("cohere2", "Cohere2Model"),
     ]
 )
@@ -235,6 +237,7 @@
         ("segformer", "SegformerModel"),
         ("siglip_vision_model", "SiglipVisionModel"),
         ("vit", "ViTModel"),
+        ("yolos", "YolosModel"),
     ]
 )
 

mindone/transformers/models/auto/processing_auto.py

@@ -55,6 +55,7 @@
         ("llava_next", "LlavaNextProcessor"),
         ("llava_next_video", "LlavaNextVideoProcessor"),
         ("llava_onevision", "LlavaOnevisionProcessor"),
+        ("owlvit", "OwlViTProcessor"),
         ("siglip", "SiglipProcessor"),
     ]
 )

mindone/transformers/models/owlvit/__init__.py

@@ -0,0 +1,17 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .image_processing_owlvit import *
+from .modeling_owlvit import *
+from .processing_owlvit import *