merge verl 0.4.0

pyproject.toml

@@ -21,7 +21,7 @@ classifiers = [
 ]
 requires-python = ">=3.10"
 dependencies = [
-    "verl==0.3.0.post1",
+    "verl==0.4.0",
     "ray[default]>=2.45.0",
     "vllm==0.8.5.post1",
     "tensordict==0.6.2",

trinity/common/verl_config.py

@@ -82,6 +82,7 @@ class Actor:
     tau: float = 0.001  # strength of regularization w.r.t. old / ref policy
     opmd_baseline: str = "mean"  # mean / logavgexp, applicable to opmd
     use_uid: bool = False  # True / False, applicable to pairwise_opmd
+    loss_agg_mode: str = "token-mean"  # do not set
 
 
 @dataclass
@@ -99,12 +100,20 @@ class _ValKwargs:
     do_sample: bool = False
 
 
+@dataclass
+class _MultiTurn:
+    enable: bool = False
+
+
 @dataclass
 class Rollout:
     # do not set
     val_kwargs: _ValKwargs = field(default_factory=_ValKwargs)
+    multi_turn: _MultiTurn = field(default_factory=_MultiTurn)
     temperature: float = 1.0
     n: int = 1  # > 1 for grpo
+    log_prob_micro_batch_size: Optional[int] = None
+    log_prob_micro_batch_size_per_gpu: int = 1
 
 
 @dataclass
@@ -148,6 +157,7 @@ class Critic:
     cliprange_value: float = 0.0
     checkpoint: Checkpoint = field(default_factory=Checkpoint)
     rollout_n: int = 1
+    loss_agg_mode: str = "token-mean"
 
 
 @dataclass