fix some bugs in dpo (#1565)

* update

* update

* update

* update doc

* update

Author: hjh0119

docs/source/LLM/自定义与拓展.md

@@ -123,27 +123,30 @@ system,instruction,input,output
 {"system": "123", "query": "AAAAA", "response": "BBBBB", "rejected_response": "CCCCC", "history": [["query1", "response1"], ["query2", "response2"]]}
 ```
 
-其中`system`和`history`为可选项
+- 其中`system`和`history`为可选项
 
 语言模型 (KTO)
 ```jsonl
 {"query": "11111", "response": "22222", "label": true}
 {"query": "aaaaa", "response": "bbbbb", "label": false}
 {"system": "123", "query": "AAAAA", "response": "BBBBB", "label": true, "history": [["query1", "response1"], ["query2", "response2"]]}
 ```
-注意`label`需要是bool类型, 不能是字符串
+- 注意`label`需要是bool类型, 不能是字符串
 
-其中`system`和`history`为可选项
+- 其中`system`和`history`为可选项
 
 
-视觉多模态大模型, 不同模型对图像数量的支持不同, 具体参考模型对应的最佳实践文档 （DPO/ORPO/SimPO/CPO）
+视觉多模态大模型（DPO/ORPO/SimPO/CPO）
+
 ```jsonl
 {"system": "123", "query": "11111", "response": "22222", "rejected_response": "33333", "images": ["image_path"], "history": [["query1", "response1"], ["query2", "response2"]]}
 {"system": "123", "query": "aaaaa", "response": "bbbbb", "rejected_response": "ccccc", "images": ["image_path"], "history": [["query1", "response1"], ["query2", "response2"]]}
 {"system": "123", "query": "AAAAA", "response": "BBBBB", "rejected_response": "CCCCC", "images": ["image_path"], "history": [["query1", "response1"], ["query2", "response2"]]}
 ```
 
-其中`system`和`history`为可选项
+- 不同模型对图像数量的支持不同, 具体参考模型对应的最佳实践文档
+
+- 其中`system`和`history`为可选项
 
 **Tool-Calling Agent**
 

docs/source_en/LLM/Customization.md

@@ -124,7 +124,7 @@ Language model (DPO/ORPO/SimPO/CPO)
 {"system": "123", "query": "aaaaa", "response": "bbbbb", "rejected_response": "ccccc", "history": [["query1", "response1"], ["query2", "response2"]]}
 {"system": "123", "query": "AAAAA", "response": "BBBBB", "rejected_response": "CCCCC", "history": [["query1", "response1"], ["query2", "response2"]]}
 ```
-(Where system and history are optional.)
+- system and history are optional.
 
 Language model (KTO)
 ```jsonl
@@ -134,19 +134,20 @@ Language model (KTO)
 ```
 Note:  `label` needs to be of type bool, not str.
 
-(Where system and history are optional.)
+- system and history are optional.
 
 
 Vision MLLM (DPO/ORPO/SimPO/CPO)
 
-Different models have varying support for the number of images. Please refer to the corresponding best practices document for each model.
 ```jsonl
 {"system": "123", "query": "11111", "response": "22222", "rejected_response": "33333", "images": ["image_path"], "history": [["query1", "response1"], ["query2", "response2"]]}
 {"system": "123", "query": "aaaaa", "response": "bbbbb", "rejected_response": "ccccc", "images": ["image_path"], "history": [["query1", "response1"], ["query2", "response2"]]}
 {"system": "123", "query": "AAAAA", "response": "BBBBB", "rejected_response": "CCCCC", "images": ["image_path"], "history": [["query1", "response1"], ["query2", "response2"]]}
 ```
 
-(Where system and history are optional.)
+- different models have varying support for the number of images. Please refer to the corresponding best practices document for each model.
+
+- system and history are optional.
 
 
 **Tool-Calling Agent**

swift/trainers/cpo_trainer.py

@@ -21,10 +21,17 @@ def __init__(self, *args, template: Template, test_oom_error=False, **kwargs):
         self.template = template
         kwargs.pop('gamma', None)
         is_vision = kwargs.pop('is_vision')
-
+        self.keys = []
         super().__init__(*args, **kwargs)
+        self.train_dataset = self.train_dataset.filter(lambda x: x['prompt_input_ids'] is not None)
+        if self.eval_dataset is not None:
+            self.eval_dataset = self.eval_dataset.filter(lambda x: x['prompt_input_ids'] is not None)
         train_ds_info = self.stat_dataset(self.train_dataset, self.is_encoder_decoder)
-        val_ds_info = self.stat_dataset(self.eval_dataset, self.is_encoder_decoder)
+        if self.eval_dataset is not None:
+            val_ds_info = self.stat_dataset(self.eval_dataset, self.is_encoder_decoder)
+            self.dataset_info = {'train_dataset': train_ds_info, 'val_dataset': val_ds_info}
+        else:
+            self.dataset_info = {'train_dataset': train_ds_info}
         self.dataset_info = {'train_dataset': train_ds_info, 'val_dataset': val_ds_info}
         if test_oom_error:
             self.train_dataset = sort_by_max_length(self.train_dataset, 20000)
@@ -53,6 +60,10 @@ def tokenize_row(self, feature, model: Union[PreTrainedModel, nn.Module] = None)
             prompt['response'] = None
             prompt_tokens = self.template.encode(prompt)[0]
 
+            # Skip examples that do not contain 'input_ids'
+            if 'input_ids' not in prompt_tokens:
+                return {k: None for k in self.keys}
+
             # resolve conflict in data_collator when labels are None, pop it afterwards
             prompt_tokens['labels'] = prompt_tokens['input_ids']
             # Batching image-related information for paired response using template
@@ -170,7 +181,8 @@ def tokenize_row(self, feature, model: Union[PreTrainedModel, nn.Module] = None)
                     labels=torch.tensor(batch['chosen_labels']))
 
             batch.update(prompt_tokens)
-
+        if not self.keys:
+            self.keys = (list(batch.keys()))
         return batch
 
     def concatenated_forward(
@@ -216,7 +228,7 @@ def concatenated_forward(
             model_kwargs['output_router_logits'] = True
 
         outputs = model(
-            concatenated_batch['concatenated_input_ids'],
+            input_ids=concatenated_batch['concatenated_input_ids'],
             attention_mask=concatenated_batch['concatenated_attention_mask'],
             use_cache=False,
             **model_kwargs,

swift/trainers/dpo_trainer.py

@@ -22,11 +22,18 @@ def __init__(self, *args, template: Template, sft_beta=0., test_oom_error=False,
         self.template = template
         self.sft_beta = sft_beta
         is_vision = kwargs.pop('is_vision')
-
+        self.keys = []
         super().__init__(*args, **kwargs)
+        self.train_dataset = self.train_dataset.filter(lambda x: x['prompt_input_ids'] is not None)
+        if self.eval_dataset is not None:
+            self.eval_dataset = self.eval_dataset.filter(lambda x: x['prompt_input_ids'] is not None)
         train_ds_info = self.stat_dataset(self.train_dataset, self.is_encoder_decoder)
-        val_ds_info = self.stat_dataset(self.eval_dataset, self.is_encoder_decoder)
-        self.dataset_info = {'train_dataset': train_ds_info, 'val_dataset': val_ds_info}
+
+        if self.eval_dataset is not None:
+            val_ds_info = self.stat_dataset(self.eval_dataset, self.is_encoder_decoder)
+            self.dataset_info = {'train_dataset': train_ds_info, 'val_dataset': val_ds_info}
+        else:
+            self.dataset_info = {'train_dataset': train_ds_info}
         if test_oom_error:
             self.train_dataset = sort_by_max_length(self.train_dataset, 20000)
         # performance
@@ -54,6 +61,10 @@ def tokenize_row(self, feature, model: Union[PreTrainedModel, nn.Module] = None)
             prompt['response'] = None
             prompt_tokens = self.template.encode(prompt)[0]
 
+            # Skip examples that do not contain 'input_ids'
+            if 'input_ids' not in prompt_tokens:
+                return {k: None for k in self.keys}
+
             # resolve conflict in data_collator when labels are None, pop it afterwards
             prompt_tokens['labels'] = prompt_tokens['input_ids']
             # Batching image-related information for paired response using template
@@ -171,7 +182,8 @@ def tokenize_row(self, feature, model: Union[PreTrainedModel, nn.Module] = None)
                     labels=torch.tensor(batch['chosen_labels']))
 
             batch.update(prompt_tokens)
-
+        if not self.keys:
+            self.keys = (list(batch.keys()))
         return batch
 
     def get_batch_loss_metrics(
@@ -289,7 +301,7 @@ def concatenated_forward(
             model_kwargs['output_router_logits'] = True
 
         outputs = model(
-            concatenated_batch['concatenated_input_ids'],
+            input_ids=concatenated_batch['concatenated_input_ids'],
             attention_mask=concatenated_batch['concatenated_attention_mask'],
             use_cache=False,
             **model_kwargs,

swift/trainers/kto_trainer.py

@@ -86,7 +86,11 @@ def __init__(self, *args, template: Template, test_oom_error=False, **kwargs):
         is_vision = kwargs.pop('is_vision')
         super().__init__(*args, **kwargs)
         train_ds_info = self.stat_dataset(self.train_dataset)
-        val_ds_info = self.stat_dataset(self.eval_dataset)
+        if self.eval_dataset is not None:
+            val_ds_info = self.stat_dataset(self.eval_dataset, self.is_encoder_decoder)
+            self.dataset_info = {'train_dataset': train_ds_info, 'val_dataset': val_ds_info}
+        else:
+            self.dataset_info = {'train_dataset': train_ds_info}
         self.dataset_info = {'train_dataset': train_ds_info, 'val_dataset': val_ds_info}
         if test_oom_error:
             self.train_dataset = sort_by_max_length(self.train_dataset, 20000)

swift/trainers/orpo_trainer.py

@@ -20,9 +20,17 @@ class ORPOTrainer(PushToMsHubMixin, SwiftMixin, HFORPOTrainer):
     def __init__(self, *args, template: Template, test_oom_error=False, **kwargs):
         self.template = template
         is_vision = kwargs.pop('is_vision')
+        self.keys = []
         super().__init__(*args, **kwargs)
+        self.train_dataset = self.train_dataset.filter(lambda x: x['prompt_input_ids'] is not None)
+        if self.eval_dataset is not None:
+            self.eval_dataset = self.eval_dataset.filter(lambda x: x['prompt_input_ids'] is not None)
         train_ds_info = self.stat_dataset(self.train_dataset, self.is_encoder_decoder)
-        val_ds_info = self.stat_dataset(self.eval_dataset, self.is_encoder_decoder)
+        if self.eval_dataset is not None:
+            val_ds_info = self.stat_dataset(self.eval_dataset, self.is_encoder_decoder)
+            self.dataset_info = {'train_dataset': train_ds_info, 'val_dataset': val_ds_info}
+        else:
+            self.dataset_info = {'train_dataset': train_ds_info}
         self.dataset_info = {'train_dataset': train_ds_info, 'val_dataset': val_ds_info}
         if test_oom_error:
             self.train_dataset = sort_by_max_length(self.train_dataset, 20000)
@@ -51,6 +59,10 @@ def tokenize_row(self, feature, model: Union[PreTrainedModel, nn.Module] = None)
             prompt['response'] = None
             prompt_tokens = self.template.encode(prompt)[0]
 
+            # Skip examples that do not contain 'input_ids'
+            if 'input_ids' not in prompt_tokens:
+                return {k: None for k in self.keys}
+
             # resolve conflict in data_collator when labels are None, pop it afterwards
             prompt_tokens['labels'] = prompt_tokens['input_ids']
             # Batching image-related information for paired response using template
@@ -168,7 +180,8 @@ def tokenize_row(self, feature, model: Union[PreTrainedModel, nn.Module] = None)
                     labels=torch.tensor(batch['chosen_labels']))
 
             batch.update(prompt_tokens)
-
+        if not self.keys:
+            self.keys = (list(batch.keys()))
         return batch
 
     def concatenated_forward(
@@ -214,7 +227,7 @@ def concatenated_forward(
             model_kwargs['output_router_logits'] = True
 
         outputs = model(
-            concatenated_batch['concatenated_input_ids'],
+            input_ids=concatenated_batch['concatenated_input_ids'],
             attention_mask=concatenated_batch['concatenated_attention_mask'],
             use_cache=False,
             **model_kwargs,