fix gpu calculating for train mode

Author: pan-x-c

examples/grpo_gsm8k/gsm8k.yaml

@@ -59,6 +59,7 @@ buffer:
       name: gsm8k_buffer
       storage_type: queue
       path: 'sqlite:///gsm8k.db'
+    # sft_warmup_steps: 0
     # sft_warmup_dataset: # Uncomment these to enable sft warmup
     #   name: warmup_data
     #   storage_type: file

trinity/common/config.py

@@ -266,7 +266,7 @@ class TrainerConfig:
     # trainer configs
     actor_use_kl_loss: bool = False
     actor_kl_loss_coef: float = 0.001
-    actor_entropy_coef: float = 0.001
+    actor_entropy_coeff: float = 0.001
     actor_grad_clip: float = 1.0
     actor_clip_ratio: float = 0.2
     # TODO: extract more train-related params from underlying trainer engine

trinity/common/verl_config.py

@@ -270,16 +270,19 @@ class veRLConfig:
 
     def synchronize_config(self, config: Config) -> None:
         """Synchronize config."""
-        rollout_gpu_num = (
-            config.explorer.rollout_model.tensor_parallel_size
-            * config.explorer.rollout_model.engine_num
-            + sum(
-                [
-                    model.tensor_parallel_size * model.engine_num
-                    for model in config.explorer.auxiliary_models
-                ]
+        if config.mode != "train":
+            rollout_gpu_num = (
+                config.explorer.rollout_model.tensor_parallel_size
+                * config.explorer.rollout_model.engine_num
+                + sum(
+                    [
+                        model.tensor_parallel_size * model.engine_num
+                        for model in config.explorer.auxiliary_models
+                    ]
+                )
             )
-        )
+        else:
+            rollout_gpu_num = 0
         rollout_node_num = rollout_gpu_num // config.cluster.gpu_per_node
         self.trainer.nnodes = config.cluster.node_num - rollout_node_num
         self.actor_rollout_ref.model.path = config.model.model_path