fix chat_template bug. (#2552)

Author: Jonathans575

examples/config/qwen/lora_argument_qwen2_0p5b.json

@@ -1,7 +1,15 @@
 {
     "model_name_or_path": "PaddleNLP/Qwen2-0.5B-Instruct",
-    "dataset_name_or_path": "./data/sft",
+    "train_dataset_path": "./data/sft/train.json",
+    "train_dataset_prob": "1.0",
+    "train_dataset_type": "erniekit",
+    "eval_dataset_path": "./data/sft/dev.json",
+    "eval_dataset_prob": "1.0",
+    "eval_dataset_type": "erniekit",
+    "packing": true,
+    "mix_strategy": "random",
     "output_dir": "./checkpoints/qwen2_paddle_lora_ckpts",
+    "max_seq_len": 8192,
     "per_device_train_batch_size": 1,
     "gradient_accumulation_steps": 4,
     "per_device_eval_batch_size": 8,
@@ -32,5 +40,6 @@
     "unified_checkpoint": true,
     "use_flash_attention": false,
     "pissa": false,
-    "use_mora": false
+    "use_mora": false,
+    "encode_one_turn": true
   }

examples/config/qwen/sft_argument_qwen2_0p5b.json

@@ -1,5 +1,5 @@
 {
-    "model_name_or_path": "/root/.cache/aistudio/hub/models/PaddleNLP/Qwen2-0.5B-Instruct",
+    "model_name_or_path": "PaddleNLP/Qwen2-0.5B-Instruct",
     "train_dataset_path": "./data/sft/train.json",
     "train_dataset_prob": "1.0",
     "train_dataset_type": "erniekit",
@@ -39,5 +39,6 @@
     "zero_padding": true,
     "flash_mask": true,
     "unified_checkpoint": true,
-    "use_flash_attention": true
+    "use_flash_attention": true,
+    "encode_one_turn": true
   }

examples/run_finetune.py

@@ -222,6 +222,7 @@ def neft_post_hook(module, input, output):
         "greedy_intokens": data_args.greedy_intokens,
         "packing": data_args.packing,
         "mix_strategy": data_args.mix_strategy,
+        "encode_one_turn": data_args.encode_one_turn,
     }
 
     train_dataset = create_dataset_sft(

paddleformers/datasets/dpo.py

@@ -116,6 +116,7 @@ def create_dataset(**dataset_config):
         mask_out_eos_token=dataset_config["mask_out_eos_token"],
         packing=dataset_config["packing"],
         mix_strategy=dataset_config["mix_strategy"],
+        encode_one_turn=dataset_config["encode_one_turn"],
     )
     return sequence_dataset
 
@@ -389,6 +390,7 @@ def __init__(
         mask_out_eos_token: bool = True,
         packing: bool = False,
         mix_strategy: str = "random",
+        encode_one_turn: bool = True,
     ):
         self.example_dataset = dataset
         self.tokenizer = tokenizer
@@ -415,6 +417,7 @@ def __init__(
         self.mask_out_eos_token = mask_out_eos_token
         self.packing = packing
         self.mix_strategy = mix_strategy
+        self.encode_one_turn = encode_one_turn
         self.num_samples_each_epoch = num_samples_each_epoch
 
         # For new data concatenation mode
@@ -594,8 +597,12 @@ def __postprocess_before_concat(self, example):
         # encoded_messages: List[List[int]]
         if not self.tokenizer.chat_template:
             self.tokenizer.init_chat_template(NONE_CHAT_TEMPLATE)
-        chosen_encoded_messages = self.tokenizer.encode_chat_inputs(example.chosen)
-        rejected_encoded_messages = self.tokenizer.encode_chat_inputs(example.rejected)
+        chosen_encoded_messages = self.tokenizer.encode_chat_inputs(
+            example.chosen, encode_one_turn=self.encode_one_turn
+        )
+        rejected_encoded_messages = self.tokenizer.encode_chat_inputs(
+            example.rejected, encode_one_turn=self.encode_one_turn
+        )
 
         # chosen/rejected response
         response_token_ids_list = []

paddleformers/datasets/finetuning.py

@@ -82,6 +82,7 @@ def create_dataset(**dataset_config):
         greedy_intokens=dataset_config["greedy_intokens"],
         packing=dataset_config["packing"],
         mix_strategy=dataset_config["mix_strategy"],
+        encode_one_turn=dataset_config["encode_one_turn"],
     )
     return sequence_dataset
 
@@ -285,6 +286,7 @@ def __init__(
         greedy_intokens: bool = False,
         packing: bool = False,
         mix_strategy: str = "random",
+        encode_one_turn: bool = True,
     ):
         """Initialize SequenceDataset.
 
@@ -314,6 +316,7 @@ def __init__(
         self.greedy_intokens = greedy_intokens
         self.packing = packing
         self.mix_strategy = mix_strategy
+        self.encode_one_turn = encode_one_turn
         self.num_samples_each_epoch = num_samples_each_epoch
         self.reverse = True
 
@@ -536,7 +539,7 @@ def _postprocess_sequence(self, example, actual_example_num):
         if example.is_function_call:
             encoded_messages = self._postprocess_fc_sequence(example)
         else:
-            encoded_messages = self.tokenizer.encode_chat_inputs(example.request)
+            encoded_messages = self.tokenizer.encode_chat_inputs(example.request, encode_one_turn=self.encode_one_turn)
 
         num_reserved_tokens_for_each_dialog = 1  # only break_turn_token or end_token
         num_reserved_tokens_for_each_turn = 8

paddleformers/transformers/tokenizer_utils.py

@@ -377,6 +377,49 @@ def _encode_chat_inputs_openai_format(
 
         return conversation_ids
 
+    def _encode_chat_inputs_oneturn(
+        self,
+        conversations: Dict[str, Any],
+        add_generation_prompt=True,
+    ):
+        conversation_dict = {} if "tools" not in conversations else {"tools": conversations["tools"]}
+        conversation_dict["messages"] = (
+            [conversations["messages"][0]] if conversations["messages"][0]["role"] == "system" else []
+        )
+
+        if conversations["messages"][0]["role"] == "system":
+            conversations["messages"] = conversations["messages"][1:]
+
+        cur_str = ""
+        conversation_ids = []
+        for idx in range(0, len(conversations["messages"]), 2):
+            conversation_id = []
+            conversation_dict["messages"].append(conversations["messages"][idx])
+            round_str = self.apply_chat_template(
+                conversation_dict["messages"], add_generation_prompt=True, tokenize=False
+            )
+            # query: user prefix + user content + assist prefix
+            query = round_str[len(cur_str) :]
+            input_ids = self.convert_tokens_to_ids(self.tokenize(query))
+            conversation_id.append(input_ids)
+            cur_str = round_str
+
+            if idx + 1 < len(conversations["messages"]):
+                conversation_dict["messages"].append(conversations["messages"][idx + 1])
+                round_str = self.apply_chat_template(
+                    conversation_dict["messages"], add_generation_prompt=False, tokenize=False
+                )
+                # answer: assistant content
+                answer = round_str[len(cur_str) :]
+                output_ids = self.convert_tokens_to_ids(self.tokenize(answer))
+                conversation_id.append(output_ids)
+
+            conversation_ids.append(conversation_id)
+            conversation_dict["messages"] = []
+            cur_str = ""
+
+        return conversation_ids
+
     def _extract_non_learnable_parts(self, origin_msg: List[Dict[str, str]], split_s: List[str]):
         """Split the entire chat by specified words. Extract the non-learnable parts."""
         # TODO：We will upgrade this feature later
@@ -458,14 +501,18 @@ def encode_chat_inputs(
         if not self.chat_template:
             raise ValueError("chat_template is not set, please set chat_template first.")
         else:
+            encode_one_turn = kwargs.pop("encode_one_turn", True)
             add_generation_prompt = kwargs.pop("add_generation_prompt", True)
             if not isinstance(conversations, dict):
                 query = self._encode_chat_inputs(
                     conversations, context_data, add_generation_prompt=add_generation_prompt
                 )
             else:
                 conversations.update(add_generation_prompt=add_generation_prompt)
-                query = self._encode_chat_inputs_openai_format(conversations)
+                if encode_one_turn:
+                    query = self._encode_chat_inputs_oneturn(conversations)
+                else:
+                    query = self._encode_chat_inputs_openai_format(conversations)
         return query
 
     def decode_token(

paddleformers/trl/sftdata_config.py

@@ -57,6 +57,10 @@ class DataConfig:
             "help": "Strategy to use in dataset mixing (random/concat/interleave) (undersampling/oversampling)."
         },
     )
+    encode_one_turn: bool = field(
+        default=True,
+        metadata={"help": "Whether encode each round independently in a multi-round dialogue."},
+    )
     packing: bool = field(
         default=True,
         metadata={"help": "Enable sequences packing in training."},

tests/transformers/test_hf_tokenizer.py

@@ -128,21 +128,6 @@ def test_dict_apply_chat_template(self):
 
 
 class TestPaddleTokenizerMethod(unittest.TestCase):
-    def test_encode_chat_inputs(self):
-        tokenizer = AutoTokenizer.from_pretrained("PaddleNLP/Qwen2.5-7B", download_hub="aistudio")
-        query = [["你好", "您好，我是个人人工智能助手"], ["今天吃啥", "你可以选择不同的菜系"]]
-        encode_text = tokenizer.encode_chat_inputs(query)
-        dict_query = {
-            "messages": [
-                {"role": "user", "content": "你好"},
-                {"role": "assistant", "content": "您好，我是个人人工智能助手"},
-                {"role": "user", "content": "今天吃啥"},
-                {"role": "assistant", "content": "你可以选择不同的菜系"},
-            ]
-        }
-        encode_dict_text = tokenizer.encode_chat_inputs(dict_query)
-        self.assertListEqual(encode_text["conversations"], encode_dict_text)
-
     def test_tokenizer_decode_token(self) -> None:
         tokenizer = AutoTokenizer.from_pretrained("PaddleNLP/Qwen2.5-7B", download_hub="aistudio")
         test_cases = ["1. 百度 2. 腾讯", "hello world! I like eating banana", "🤓😖", "🤓😖testtest"]