[megatron] fix eval_iters -1 (#4847)

Author: Jintao-Huang

swift/megatron/train/trainers/trainer.py

@@ -46,8 +46,10 @@ def initialize_megatron(*_args, **kwargs):
                 else:
                     raise ValueError(
                         'You are using a streaming training dataset. Please explicitly specify `--train_iters`.')
-            if val_dataset is not None and args.eval_iters < 0:
-                if hasattr(val_dataset, '__len__'):
+            if args.eval_iters < 0:
+                if val_dataset is None:
+                    args.eval_iters = 0
+                elif hasattr(val_dataset, '__len__'):
                     dataset_sample = len(val_dataset) // step_batch_size * step_batch_size
                     args.eval_iters = max(dataset_sample // args.global_batch_size, 1)
                 else:

swift/megatron/train/utils.py

@@ -15,8 +15,7 @@ def swift_datasets_provider(train_val_test_num_samples):
         nonlocal val_dataset
         args = get_args()
         data_parallel_size = mpu.get_data_parallel_world_size()
-        step_batch_size = \
-            args.micro_batch_size * data_parallel_size
+        step_batch_size = args.micro_batch_size * data_parallel_size
         # To avoid errors caused by the validation set being insufficient to complete a single step.
         if val_dataset is not None and len(val_dataset) < step_batch_size:
             val_dataset = None

tests/megatron/test_train.py

@@ -17,7 +17,6 @@ def test_sft():
             train_iters=100,
             model_author='swift',
             model_name='swift-robot',
-            eval_iters=5,
             sequence_parallel=True,
             finetune=True))