Fix dataflow for PaddleOCR-VL

ernie/dataset/vl_sft_reader/vl_sft_dataset_json.py

@@ -751,7 +751,7 @@ def gen_sample_list(self):
         for i, _ in enumerate(self.task_group):
             sample_size = int(self.weight_list[i] * self.length)
             print(
-                f"Take {sample_size} samples from {self.task_group[i]._file_name} (total length: {len(self.task_group[i].exs)}) to construct current sample list"
+                f"Take {sample_size} samples from {self.task_group[i]._file_name} to construct current sample list"
             )
             indices.extend([i] * sample_size)
         return indices

erniekit/train/ocr_vl_sft/pretraining_trainer.py

@@ -1655,7 +1655,11 @@ def lr_ratio_fn(param):
 
         return self.optimizer
 
-    def save_model(self, output_dir=None):
+    def save_model(
+        self,
+        output_dir=None,
+        merge_tensor_parallel=False,
+    ):
         """
         Saves the model and associated configuration files to the specified directory.
 
@@ -1669,7 +1673,7 @@ def save_model(self, output_dir=None):
             None
 
         """
-        super().save_model(output_dir)
+        super().save_model(output_dir, merge_tensor_parallel)
         if self.args.should_save:
             with open(
                 os.path.join(output_dir, "static_name_to_dyg_name.json"), "w"

erniekit/train/ocr_vl_sft/workflow.py

@@ -116,6 +116,7 @@ def run_ocr_vl_sft(
     preprocess_args.batch_size = finetuning_args.batch_size
     finetuning_args.max_seq_len = data_args.max_seq_len
     finetuning_args.max_seq_length = data_args.max_seq_len
+    finetuning_args.packing = data_args.packing
 
     # create output dir
     os.makedirs(finetuning_args.output_dir, exist_ok=True)