开启offload试试#

Author: root

agentlightning/verl/daemon.py

@@ -459,22 +459,22 @@ def get_train_data_batch(self, max_prompt_length, max_response_length, device):
         n_transition = len(input_ids_list)
         print("***************************************",n_transition)
 
-        # # 直接扔掉多余的 transitions，限制最大数量
-        MAX_TRANSITIONS = 96 
-        if n_transition > MAX_TRANSITIONS:
-            # 确保所有列表长度一致
-            input_ids_list = input_ids_list[:MAX_TRANSITIONS]
-            input_attention_mask_list = input_attention_mask_list[:MAX_TRANSITIONS]
-            response_ids_list = response_ids_list[:MAX_TRANSITIONS]
-            response_attention_mask_list = response_attention_mask_list[:MAX_TRANSITIONS]
-            reward_list = reward_list[:MAX_TRANSITIONS]
-            data_id_list = data_id_list[:MAX_TRANSITIONS]
-            rollout_id_list = rollout_id_list[:MAX_TRANSITIONS]
-            turn_index_list = turn_index_list[:MAX_TRANSITIONS]
-            is_drop_list = is_drop_list[:MAX_TRANSITIONS]
+        # # 直接扔掉多余的 transitions，限制最大数量(会报错)
+        # MAX_TRANSITIONS = 96 
+        # if n_transition > MAX_TRANSITIONS:
+        #     # 确保所有列表长度一致
+        #     input_ids_list = input_ids_list[:MAX_TRANSITIONS]
+        #     input_attention_mask_list = input_attention_mask_list[:MAX_TRANSITIONS]
+        #     response_ids_list = response_ids_list[:MAX_TRANSITIONS]
+        #     response_attention_mask_list = response_attention_mask_list[:MAX_TRANSITIONS]
+        #     reward_list = reward_list[:MAX_TRANSITIONS]
+        #     data_id_list = data_id_list[:MAX_TRANSITIONS]
+        #     rollout_id_list = rollout_id_list[:MAX_TRANSITIONS]
+        #     turn_index_list = turn_index_list[:MAX_TRANSITIONS]
+        #     is_drop_list = is_drop_list[:MAX_TRANSITIONS]
 
-            n_transition = MAX_TRANSITIONS
-
+        #     n_transition = MAX_TRANSITIONS
+        #     print("********************MAX_TRANSITIONS*******************",n_transition)
         batch_input_ids = torch.LongTensor(input_ids_list).to(device)
         input_attention_mask = torch.LongTensor(input_attention_mask_list).to(device)
         batch_response_ids = torch.LongTensor(response_ids_list).to(device)

examples/werewolf/train.sh

@@ -24,7 +24,7 @@ python -m agentlightning.verl \
     actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \
     actor_rollout_ref.rollout.multi_turn.format=hermes \
     actor_rollout_ref.model.path=${BASE_MODEL} \
-    data.max_prompt_length=11264 \
+    data.max_prompt_length=12288 \
     data.max_response_length=1024 \
     data.truncation='error' \
     trainer.val_before_train=True \
@@ -36,12 +36,12 @@ python -m agentlightning.verl \
     actor_rollout_ref.actor.clip_ratio_low=0.2 \
     actor_rollout_ref.actor.clip_ratio_high=0.3 \
     actor_rollout_ref.model.enable_gradient_checkpointing=True \
-    actor_rollout_ref.actor.fsdp_config.param_offload=False \
-    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \
+    actor_rollout_ref.actor.fsdp_config.param_offload=True \
+    actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \
     actor_rollout_ref.rollout.name=vllm \
     actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \
     actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \
-    actor_rollout_ref.ref.fsdp_config.param_offload=False \
+    actor_rollout_ref.ref.fsdp_config.param_offload=True \
     algorithm.use_kl_in_reward=False \
     trainer.default_local_dir='/root/dataDisk/checkpoints' \
     trainer.rollout_data_dir='/root/dataDisk/rollout' \