[grpo] update apply template (#4833)

* Update bos_token bug

* Update grpo_trainer.py for special token encode error for deepseek model

* Update grpo_trainer.py for special token encode error for deepseek model

* Update grpo_trainer.py

Author: aacedar

swift/trainers/rlhf_trainer/grpo_trainer.py

@@ -1094,8 +1094,20 @@ def _apply_chat_template_to_messages_list(self, messages_list: InputsType):
             InferRequest.remove_response(messages)
             template_inputs, _ = StdTemplateInputs.from_dict({'messages': messages})
             res_context_list, _, _ = self.template._swift_encode(template_inputs)
-            prompts_text.append(''.join(elem for elem in res_context_list if isinstance(elem, str)))
 
+            # check the type and convert
+            processed_context = []
+            for context in res_context_list:
+                if isinstance(context, str):
+                    processed_context.append(context)
+                elif isinstance(context, list) and all(isinstance(x, int) for x in context):
+                    # decode the token ID to text
+                    decoded_text = self.template.tokenizer.decode(context)
+                    processed_context.append(decoded_text)
+                else:
+                    # other type value ,just add to process_context
+                    processed_context.append(str(context))
+            prompts_text.append(''.join(processed_context))
         return prompts_text
 
     @profiling_decorator