[autonlp]add prompt candidates for text classification (#4867)

* add prompt candidates

* fix

Author: lugimzzz

paddlenlp/experimental/autonlp/text_classification.py

@@ -134,6 +134,7 @@ def _model_candidates(self) -> List[Dict[str, Any]]:
         chinese_models = hp.choice(
             "models",
             [
+                "ernie-1.0-large-zh-cw"  # 24-layer, 1024-hidden, 16-heads, 272M parameters.
                 "ernie-3.0-xbase-zh",  # 20-layer, 1024-hidden, 16-heads, 296M parameters.
                 "ernie-3.0-tiny-base-v2-zh",  # 12-layer, 768-hidden, 12-heads, 118M parameters.
                 "ernie-3.0-tiny-medium-v2-zh",  # 6-layer, 768-hidden, 12-heads, 75M parameters.
@@ -155,6 +156,21 @@ def _model_candidates(self) -> List[Dict[str, Any]]:
                 "ernie-2.0-large-en",  # 24-layer, 1024-hidden, 16-heads, 336M parameters. Trained on lower-cased English text.
             ],
         )
+        english_prompt_models = hp.choice(
+            "models",
+            [
+                # add deberta-v3 when we have it
+                "roberta-large",  # 24-layer, 1024-hidden, 16-heads, 334M parameters. Case-sensitive
+                "roberta-base",  # 12-layer, 768-hidden, 12-heads, 110M parameters. Case-sensitive
+            ],
+        )
+        chinese_prompt_models = hp.choice(
+            "models",
+            [
+                "ernie-1.0-large-zh-cw"  # 24-layer, 1024-hidden, 16-heads, 272M parameters.
+                "ernie-1.0-base-zh-cw"  # 12-layer, 768-hidden, 12-heads, 118M parameters.
+            ],
+        )
         return [
             # fast learning: high LR, small early stop patience
             {
@@ -202,7 +218,33 @@ def _model_candidates(self) -> List[Dict[str, Any]]:
                 "TrainingArguments.model_name_or_path": english_models,
                 "TrainingArguments.learning_rate": 5e-6,
             },
-            # Note: prompt tuning candidates not included for now due to lack of inference capability
+            # prompt tuning candidates
+            {
+                "preset": "prompt",
+                "language": "Chinese",
+                "trainer_type": "PromptTrainer",
+                "template.prompt": "{'mask'}{'soft'}“{'text': '" + self.text_column + "'}”",
+                "EarlyStoppingCallback.early_stopping_patience": 5,
+                "PromptTuningArguments.per_device_train_batch_size": train_batch_size,
+                "PromptTuningArguments.per_device_eval_batch_size": train_batch_size * 2,
+                "PromptTuningArguments.num_train_epochs": 100,
+                "PromptTuningArguments.model_name_or_path": chinese_prompt_models,
+                "PromptTuningArguments.learning_rate": 1e-5,
+                "PromptTuningArguments.ppt_learning_rate": 1e-4,
+            },
+            {
+                "preset": "prompt",
+                "language": "English",
+                "trainer_type": "PromptTrainer",
+                "template.prompt": "{'mask'}{'soft'}“{'text': '" + self.text_column + "'}”",
+                "EarlyStoppingCallback.early_stopping_patience": 5,
+                "PromptTuningArguments.per_device_train_batch_size": train_batch_size,
+                "PromptTuningArguments.per_device_eval_batch_size": train_batch_size * 2,
+                "PromptTuningArguments.num_train_epochs": 100,
+                "PromptTuningArguments.model_name_or_path": english_prompt_models,
+                "PromptTuningArguments.learning_rate": 1e-5,
+                "PromptTuningArguments.ppt_learning_rate": 1e-4,
+            },
         ]
 
     def _data_checks_and_inference(self):
@@ -247,6 +289,8 @@ def _data_checks_and_inference(self):
                                 raise ValueError(
                                     f"Label {label} is not found in the user-provided id2label argument: {self.id2label}"
                                 )
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir)
         id2label_path = os.path.join(self.output_dir, "id2label.json")
         with open(id2label_path, "w", encoding="utf-8") as f:
             json.dump(self.id2label, f, ensure_ascii=False)

paddlenlp/prompt/prompt_trainer.py

@@ -144,7 +144,7 @@ def _save(self, output_dir: Optional[str] = None, state_dict: Dict[str, Any] = N
     def load_state_dict_from_checkpoint(self, resume_from_checkpoint: os.PathLike = None):
         if resume_from_checkpoint is not None:
             self.template = AutoTemplate.load_from(
-                resume_from_checkpoint, self.tokenizer, self.args.max_seq_length, self._get_model()
+                resume_from_checkpoint, self.tokenizer, self.args.max_seq_length, self._get_model().plm
             )
         super(PromptTrainer, self).load_state_dict_from_checkpoint(resume_from_checkpoint)