Fix #3768 (#3789)

* fix bug
* update

Author: JunnYu

paddlenlp/transformers/auto/modeling.py

@@ -89,6 +89,8 @@
     ("Bart", "bart"),
     ("GAUAlpha", "gau_alpha"),
     ("CodeGen", "codegen"),
+    ("CLIPVision", "clip"),
+    ("CLIPText", "clip"),
     ("CLIP", "clip"),
     ("Artist", "artist"),
     ("OPT", 'opt'),

paddlenlp/transformers/clip/modeling.py

@@ -533,7 +533,7 @@ def quick_gelu(x):
 
 F.quick_gelu = quick_gelu
 
-NEG_INF = -1e9  # float("-inf") -1e4 -1e9
+NEG_INF = -1e4  # float("-inf") -1e4 -1e9
 
 
 class VisionTransformer(nn.Layer):

ppdiffusers/examples/dreambooth/train_dreambooth.py

@@ -35,11 +35,11 @@
 from paddle.vision import transforms
 from paddle.optimizer import AdamW
 from tqdm.auto import tqdm
-from paddlenlp.transformers import AutoModel, AutoTokenizer
+from paddlenlp.transformers import BertModel, AutoTokenizer, CLIPTextModel
 from pathlib import Path
 
 
-def parse_args(input_args):
+def parse_args(input_args=None):
     parser = argparse.ArgumentParser(
         description="Simple example of a training dreambooth script.")
     parser.add_argument(
@@ -427,7 +427,11 @@ def main(args):
             os.path.join(args.pretrained_model_name_or_path, "tokenizer"))
 
     # Load models and create wrapper for stable diffusion
-    text_encoder = AutoModel.from_pretrained(
+    if "Taiyi-Stable-Diffusion-1B-Chinese-v0.1" in args.pretrained_model_name_or_path:
+        model_cls = BertModel
+    else:
+        model_cls = CLIPTextModel
+    text_encoder = model_cls.from_pretrained(
         os.path.join(args.pretrained_model_name_or_path, "text_encoder"))
     vae = AutoencoderKL.from_pretrained(args.pretrained_model_name_or_path,
                                         subfolder="vae")

ppdiffusers/examples/text_to_image/README.md

@@ -34,7 +34,6 @@ export dataset_name="lambdalabs/pokemon-blip-captions"
 python -u train_text_to_image.py \
   --pretrained_model_name_or_path=$MODEL_NAME \
   --dataset_name=$dataset_name \
-  --use_ema \
   --resolution=512 --center_crop --random_flip \
   --train_batch_size=1 \
   --gradient_accumulation_steps=4 \
@@ -108,7 +107,6 @@ export dataset_name="lambdalabs/pokemon-blip-captions"
 python -u -m paddle.distributed.launch --gpus "0,1,2,3" train_text_to_image.py \
   --pretrained_model_name_or_path=$MODEL_NAME \
   --dataset_name=$dataset_name \
-  --use_ema \
   --resolution=512 --center_crop --random_flip \
   --train_batch_size=1 \
   --gradient_accumulation_steps=4 \

ppdiffusers/examples/text_to_image/run_multi.sh

@@ -18,7 +18,6 @@ export dataset_name="lambdalabs/pokemon-blip-captions"
 python -u -m paddle.distributed.launch --gpus "0,1,2,3" train_text_to_image.py \
   --pretrained_model_name_or_path=$MODEL_NAME \
   --dataset_name=$dataset_name \
-  --use_ema \
   --resolution=512 --center_crop --random_flip \
   --train_batch_size=1 \
   --gradient_accumulation_steps=4 \

ppdiffusers/examples/text_to_image/run_single.sh

@@ -18,7 +18,6 @@ export dataset_name="lambdalabs/pokemon-blip-captions"
 python -u train_text_to_image.py \
   --pretrained_model_name_or_path=$MODEL_NAME \
   --dataset_name=$dataset_name \
-  --use_ema \
   --resolution=512 --center_crop --random_flip \
   --train_batch_size=1 \
   --gradient_accumulation_steps=4 \

ppdiffusers/examples/text_to_image/train_text_to_image.py

@@ -39,7 +39,7 @@
 from paddle.vision import transforms, BaseTransform
 from paddle.optimizer import AdamW
 from tqdm.auto import tqdm
-from paddlenlp.transformers import AutoModel, AutoTokenizer
+from paddlenlp.transformers import CLIPTextModel, AutoTokenizer, BertModel
 
 
 class Lambda(BaseTransform):
@@ -332,7 +332,12 @@ def main():
     # Load models and create wrapper for stable diffusion
     tokenizer = AutoTokenizer.from_pretrained(
         os.path.join(args.pretrained_model_name_or_path, "tokenizer"))
-    text_encoder = AutoModel.from_pretrained(
+
+    if "Taiyi-Stable-Diffusion-1B-Chinese-v0.1" in args.pretrained_model_name_or_path:
+        model_cls = BertModel
+    else:
+        model_cls = CLIPTextModel
+    text_encoder = model_cls.from_pretrained(
         os.path.join(args.pretrained_model_name_or_path, "text_encoder"))
     vae = AutoencoderKL.from_pretrained(args.pretrained_model_name_or_path,
                                         subfolder="vae")

ppdiffusers/examples/textual_inversion/train_textual_inversion.py

@@ -43,7 +43,7 @@
 from paddle.vision.transforms import RandomHorizontalFlip
 from paddle.optimizer import AdamW
 from tqdm.auto import tqdm
-from paddlenlp.transformers import AutoModel, AutoTokenizer, BertModel
+from paddlenlp.transformers import CLIPTextModel, AutoTokenizer, BertModel
 
 
 def get_writer(args):
@@ -468,7 +468,7 @@ def main():
     if args.tokenizer_name:
         tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name)
     elif args.pretrained_model_name_or_path:
-        tokenizer = AutoModel.from_pretrained(
+        tokenizer = AutoTokenizer.from_pretrained(
             os.path.join(args.pretrained_model_name_or_path, "tokenizer"))
 
     # Add the placeholder token in tokenizer
@@ -490,7 +490,11 @@ def main():
         args.placeholder_token)
 
     # Load models and create wrapper for stable diffusion
-    text_encoder = AutoModel.from_pretrained(
+    if "Taiyi-Stable-Diffusion-1B-Chinese-v0.1" in args.pretrained_model_name_or_path:
+        model_cls = BertModel
+    else:
+        model_cls = CLIPTextModel
+    text_encoder = model_cls.from_pretrained(
         os.path.join(args.pretrained_model_name_or_path, "text_encoder"))
     vae = AutoencoderKL.from_pretrained(args.pretrained_model_name_or_path,
                                         subfolder="vae")