[grpo] check eval_dataset length (#4781)

* check evalds length

* check valds in trainargs

* set default split_dataset_ratio 0 for grpo

* fix generation_batch_size check

Author: hjh0119

swift/llm/argument/train_args.py

@@ -169,6 +169,8 @@ def __post_init__(self) -> None:
 
         if getattr(self, 'accelerator_config', None) is None:
             self.accelerator_config = {'dispatch_batches': False}
+        if self.split_dataset_ratio == 0 and not self.val_dataset:
+            self.eval_strategy = 'no'
         self.training_args = TrainerFactory.get_training_args(self)
         self.training_args.remove_unused_columns = False
         self._add_version()

swift/trainers/arguments.py

@@ -225,6 +225,7 @@ class GRPOArgumentsMixin:
 
     # dataset
     dataset_shuffle: Optional[bool] = True
+    split_dataset_ratio: float = 0.0
 
 
 @dataclass

swift/trainers/rlhf_arguments.py

@@ -80,7 +80,7 @@ def check_num_generations(self):
         # check num_generations for trl < 0.18
         num_processes = self.world_size
 
-        if self.generation_batch_size % self.per_device_train_batch_size * num_processes != 0:
+        if self.generation_batch_size % (self.per_device_train_batch_size * num_processes) != 0:
             raise ValueError(
                 f'generation_batch_size ({self.generation_batch_size}) must be divisible by the global batch size '
                 f'({self.per_device_train_batch_size * num_processes}).')

swift/trainers/rlhf_trainer/grpo_trainer.py

@@ -188,6 +188,13 @@ def __init__(self,
         vllm_client = kwargs.pop('vllm_client')  # for external vllm
 
         super().__init__(model, ref_model, *_args, **kwargs)
+        if self.args.eval_strategy != 'no':
+            total_eval_batch_size = self.args.per_device_eval_batch_size * \
+                self.accelerator.num_processes // self.args.num_generations
+            assert len(self.eval_dataset) >= total_eval_batch_size, (
+                f'eval_dataset size {len(self.eval_dataset)} is smaller than '
+                f'total_eval_batch_size {total_eval_batch_size}. '
+                f'Please increase the size of eval_dataset or set a larger value for split_dataset_ratio.')
         # Multi-step
         self.num_iterations = args.num_iterations  # = 𝜇 in the GRPO paper