Add convbert model (#1036)

Author: liuchuting

mindone/transformers/__init__.py

@@ -155,6 +155,15 @@
     CLIPVisionModelWithProjection,
 )
 from .models.cohere2 import Cohere2ForCausalLM, Cohere2Model, Cohere2PreTrainedModel
+from .models.convbert import (
+    ConvBertForMaskedLM,
+    ConvBertForMultipleChoice,
+    ConvBertForQuestionAnswering,
+    ConvBertForSequenceClassification,
+    ConvBertForTokenClassification,
+    ConvBertLayer,
+    ConvBertModel,
+)
 from .models.deberta import (
     DebertaForMaskedLM,
     DebertaForQuestionAnswering,

mindone/transformers/generation/utils.py

@@ -1351,7 +1351,7 @@ def compute_transition_scores(
 
         ```python
         >>> from transformers import GPT2Tokenizer
-        >>> from mindway.transformers import AutoModelForCausalLM
+        >>> from mindone.transformers import AutoModelForCausalLM
         >>> import numpy as np
 
         >>> tokenizer = GPT2Tokenizer.from_pretrained("gpt2")

mindone/transformers/models/__init__.py

@@ -31,6 +31,7 @@
     camembert,
     clap,
     clip,
+    convbert,
     dpt,
     fuyu,
     gemma,

mindone/transformers/models/auto/configuration_auto.py

@@ -61,6 +61,7 @@
         ("helium", "HeliumConfig"),
         ("hiera", "HieraConfig"),
         ("camembert", "CamembertConfig"),
+        ("convbert", "ConvBertConfig"),
         ("idefics", "IdeficsConfig"),
         ("idefics2", "Idefics2Config"),
         ("idefics3", "Idefics3Config"),
@@ -184,6 +185,7 @@
         ("umt5", "UMT5"),
         ("wav2vec2", "Wav2Vec2"),
         ("whisper", "Whisper"),
+        ("convbert", "ConvBERT"),
         ("xlm-roberta", "XLM-RoBERTa"),
         ("xlm-roberta-xl", "XLM-RoBERTa-XL"),
         ("cohere2", "Cohere2"),

mindone/transformers/models/auto/modeling_auto.py

@@ -39,6 +39,7 @@
         ("bart", "BartModel"),
         ("camembert", "CamembertModel"),
         ("mvp", "MvpModel"),
+        ("convbert", "ConvBertModel"),
         ("bit", "BitModel"),
         ("blip", "BlipModel"),
         ("blip-2", "Blip2Model"),
@@ -143,6 +144,7 @@
         ("bert", "BertForMaskedLM"),
         ("deberta", "DebertaForMaskedLM"),
         ("deberta-v2", "DebertaV2ForMaskedLM"),
+        ("convbert", "ConvBertForMaskedLM"),
         ("gpt2", "GPT2LMHeadModel"),
         ("led", "LEDForConditionalGeneration"),
         ("camembert", "CamembertForMaskedLM"),
@@ -285,6 +287,7 @@
         ("mvp", "MvpForConditionalGeneration"),
         ("albert", "AlbertForMaskedLM"),
         ("bart", "BartForConditionalGeneration"),
+        ("convbert", "ConvBertForMaskedLM"),
         ("bert", "BertForMaskedLM"),
         ("roberta", "RobertaForMaskedLM"),
         ("camembert", "CamembertForMaskedLM"),
@@ -371,6 +374,7 @@
         ("llama", "LlamaForSequenceClassification"),
         ("persimmon", "PersimmonForSequenceClassification"),
         ("mobilebert", "MobileBertForSequenceClassification"),
+        ("convbert", "ConvBertForSequenceClassification"),
         ("mt5", "MT5ForSequenceClassification"),
         ("megatron-bert", "MegatronBertForSequenceClassification"),
         ("mistral", "MistralForSequenceClassification"),
@@ -398,6 +402,7 @@
         ("deberta", "DebertaForQuestionAnswering"),
         ("deberta-v2", "DebertaV2ForQuestionAnswering"),
         ("led", "LEDForQuestionAnswering"),
+        ("convbert", "ConvBertForQuestionAnswering"),
         ("llama", "LlamaForQuestionAnswering"),
         ("mobilebert", "MobileBertForQuestionAnswering"),
         ("megatron-bert", "MegatronBertForQuestionAnswering"),
@@ -446,6 +451,7 @@
         ("qwen2", "Qwen2ForTokenClassification"),
         ("roberta", "RobertaForTokenClassification"),
         ("rembert", "RemBertForTokenClassification"),
+        ("convbert", "ConvBertForTokenClassification"),
         ("t5", "T5ForTokenClassification"),
         ("umt5", "UMT5ForTokenClassification"),
         ("xlm-roberta", "XLMRobertaForTokenClassification"),
@@ -458,6 +464,7 @@
         # Model for Multiple Choice mapping
         ("camembert", "CamembertForMultipleChoice"),
         ("albert", "AlbertForMultipleChoice"),
+        ("convbert", "ConvBertForMultipleChoice"),
         ("bert", "BertForMultipleChoice"),
         ("roberta", "RobertaForMultipleChoice"),
         ("deberta-v2", "DebertaV2ForMultipleChoice"),

mindone/transformers/models/convbert/__init__.py

@@ -0,0 +1,25 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# This code is adapted from https://github.com/huggingface/transformers
+# with modifications to run transformers on mindspore.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from .modeling_convbert import (
+    ConvBertForMaskedLM,
+    ConvBertForMultipleChoice,
+    ConvBertForQuestionAnswering,
+    ConvBertForSequenceClassification,
+    ConvBertForTokenClassification,
+    ConvBertLayer,
+    ConvBertModel,
+)