git upgrade tokenizer

Author: DrownFish19

paddlenlp/transformers/__init__.py

@@ -53,12 +53,14 @@
 # isort: split
 from .bert.modeling import *
 from .bert.tokenizer import *
+from .bert.tokenizer_fast import *
 from .bert.configuration import *
 
 # isort: split
 from .gpt import *
 from .roberta.modeling import *
 from .roberta.tokenizer import *
+from .roberta.tokenizer_fast import *
 from .roberta.configuration import *
 from .electra.modeling import *
 from .electra.tokenizer import *

paddlenlp/transformers/auto/tokenizer.py

@@ -171,7 +171,7 @@ def get_configurations():
 
 
 def tokenizer_class_from_name(class_name: str):
-    if class_name == "PretrainedTokenizerFast":
+    if class_name in ["PretrainedTokenizerFast", "PreTrainedTokenizerFast"]:
         return PretrainedTokenizerFast
 
     for module_name, tokenizers in TOKENIZER_MAPPING_NAMES.items():
@@ -309,6 +309,8 @@ class AutoTokenizer:
     pretrained weights/vocabulary.
     AutoTokenizer is a generic tokenizer class that will be instantiated as one of the
     base tokenizer classes when created with the AutoTokenizer.from_pretrained() classmethod.
+
+    This class cannot be instantiated directly using `__init__()` (throws an error).
     """
 
     _tokenizer_mapping = get_configurations()
@@ -338,7 +340,7 @@ def _get_tokenizer_class_from_config(cls, pretrained_model_name_or_path, config_
                 try:
                     if tokenizer_class is None:
                         tokenizer_class = getattr(import_class, init_class)
-                except:
+                except AttributeError:
                     raise ValueError(f"Tokenizer class {init_class} is not currently imported.")
                 return tokenizer_class
             else:

paddlenlp/transformers/bert/__init__.py

@@ -11,3 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from .configuration import *
+from .modeling import *
+from .tokenizer import *
+from .tokenizer_fast import *