Claude/fix raft function registration 01 maq5c t7dvc g wg h3e n shz tu

.claude/settings.local.json

@@ -0,0 +1,9 @@
+{
+  "permissions": {
+    "allow": [
+      "WebFetch(domain:arxiv.org)"
+    ],
+    "deny": [],
+    "ask": []
+  }
+}

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
     rev: 23.7.0
     hooks:
     - id: black
-      language_version: python3.10
+#      language_version: python3.10
       args: [--line-length=100]
 
   - repo: https://github.com/pycqa/isort

benchmark/config/gsm8k-template.yaml

@@ -60,8 +60,8 @@ explorer:
     engine_num: 2
     tensor_parallel_size: 1
     enforce_eager: false
-    enable_prefix_caching: false
-    enable_chunked_prefill: false
+    enable_prefix_caching: true
+    enable_chunked_prefill: true
     gpu_memory_utilization: 0.9
     dtype: bfloat16
     seed: 42

examples/R3L/alfworld/RAFT_1.5B.yaml

@@ -0,0 +1,72 @@
+project: "ALFWORLD"
+name: "ALFWORLD_RFT_Qwen_1.5B_RAFT_Baseline"
+checkpoint_root_dir: ${oc.env:TRINITY_CHECKPOINT_ROOT_DIR,./checkpoints}
+algorithm:
+  algorithm_type: raft
+  repeat_times: 1
+  optimizer:
+    lr: 1e-6
+model:
+  model_path: ${oc.env:TRINITY_MODEL_PATH,Qwen/Qwen2.5-1.5B-Instruct}
+  max_response_tokens: 512
+  max_model_len: 20480
+cluster:
+  node_num: 1
+  gpu_per_node: 4
+buffer:
+  total_epochs: 20
+  batch_size: 96
+  explorer_input:
+    taskset:
+      name: alfworld
+      storage_type: file
+      path: 'examples/R3L/alfworld/alfworld_data'
+      split: 'train'
+      format:
+        prompt_key: 'task_id'
+      rollout_args:
+        temperature: 1.0
+        logprobs: 0
+    eval_tasksets:
+      - name: alfworld-eval
+        storage_type: file
+        path: 'examples/R3L/alfworld/alfworld_data'
+        split: 'test'
+        format:
+          prompt_key: 'task_id'
+        rollout_args:
+          temperature: 0.4
+    default_workflow_type: 'RAFT_baseline_alfworld_workflow'
+  trainer_input:
+    experience_buffer:
+      name: alfworld_raft_baseline_1.5B_buffer
+      storage_type: queue
+      replay_buffer:
+        enable: true
+        priority_fn: decay_limit_randomization
+      path: 'sqlite:///alfworld_raft_baseline_1.5B.db'
+explorer:
+  runner_per_model: 32
+  eval_interval: 20
+  rollout_model:
+    engine_num: 2
+    tensor_parallel_size: 1
+    enable_prefix_caching: false
+    enforce_eager: false
+    dtype: bfloat16
+    seed: 0
+    gpu_memory_utilization: 0.7
+    enable_chunked_prefill: true
+synchronizer:
+  sync_style: dynamic_by_explorer
+  sync_method: 'nccl'
+  sync_interval: 1
+  sync_timeout: 12000
+trainer:
+  save_interval: 20
+  grad_clip: 1.0
+  use_dynamic_bsz: true
+  max_token_len_per_gpu: 10240
+  ulysses_sequence_parallel_size: 1
+monitor:
+  monitor_type: wandb

examples/R3L/alfworld/RAFT_7B.yaml

@@ -0,0 +1,72 @@
+project: "ALFWORLD"
+name: "ALFWORLD_RFT_Qwen_7B_RAFT_Baseline"
+checkpoint_root_dir: ${oc.env:TRINITY_CHECKPOINT_ROOT_DIR,./checkpoints}
+algorithm:
+  algorithm_type: raft
+  repeat_times: 8
+  optimizer:
+    lr: 1e-6
+model:
+  model_path: ${oc.env:TRINITY_MODEL_PATH,Qwen/Qwen2.5-7B-Instruct}
+  max_response_tokens: 512
+  max_model_len: 20480
+cluster:
+  node_num: 1
+  gpu_per_node: 4
+buffer:
+  total_epochs: 20
+  batch_size: 1
+  explorer_input:
+    taskset:
+      name: alfworld
+      storage_type: file
+      path: 'examples/R3L/alfworld/alfworld_data'
+      split: 'train'
+      format:
+        prompt_key: 'task_id'
+      rollout_args:
+        temperature: 1.0
+        logprobs: 0
+    eval_tasksets:
+      - name: alfworld-eval
+        storage_type: file
+        path: 'examples/R3L/alfworld/alfworld_data'
+        split: 'test'
+        format:
+          prompt_key: 'task_id'
+        rollout_args:
+          temperature: 0.4
+    default_workflow_type: 'RAFT_baseline_alfworld_workflow'
+  trainer_input:
+    experience_buffer:
+      name: alfworld_raft_baseline_7B_buffer
+      storage_type: queue
+      replay_buffer:
+        enable: true
+        priority_fn: decay_limit_randomization
+      path: 'sqlite:///alfworld_raft_baseline_7B.db'
+explorer:
+  runner_per_model: 32
+  eval_interval: 20
+  rollout_model:
+    engine_num: 2
+    tensor_parallel_size: 1
+    enable_prefix_caching: false
+    enforce_eager: false
+    dtype: bfloat16
+    seed: 0
+    gpu_memory_utilization: 0.7
+    enable_chunked_prefill: true
+synchronizer:
+  sync_style: dynamic_by_explorer
+  sync_method: 'nccl'
+  sync_interval: 1
+  sync_timeout: 12000
+trainer:
+  save_interval: 20
+  grad_clip: 1.0
+  use_dynamic_bsz: true
+  max_token_len_per_gpu: 10240
+  ulysses_sequence_parallel_size: 1
+monitor:
+  monitor_type: wandb

examples/R3L/alfworld/dapo_1.5B.yaml

@@ -0,0 +1,88 @@
+project: "ALFWORLD"
+name: "ALFWORLD_RFT_Qwen_1.5B_DAPO"
+checkpoint_root_dir: ${oc.env:TRINITY_CHECKPOINT_ROOT_DIR,./checkpoints}
+algorithm:
+  algorithm_type: grpo
+  kl_loss_fn: k3
+  kl_loss_fn_args:
+    kl_coef: 0.01
+  repeat_times: 8
+  optimizer:
+    lr: 1e-6
+    lr_warmup_steps: 20
+  policy_loss_fn_args:
+    clip_range_low: 0.2
+    clip_range_high: 0.28
+model:
+  model_path: ${oc.env:TRINITY_MODEL_PATH,Qwen/Qwen2.5-1.5B-Instruct}
+  max_response_tokens: 512
+  max_model_len: 20480
+cluster:
+  node_num: 1
+  gpu_per_node: 4
+buffer:
+  total_epochs: 20
+  batch_size: 48 # 如果是96会OOM
+  explorer_input:
+    taskset:
+      name: alfworld
+      storage_type: file
+      path: 'examples/R3L/alfworld/alfworld_data'
+      split: 'train'
+      format:
+        prompt_key: 'task_id'
+      rollout_args:
+        temperature: 1.0
+        logprobs: 0
+      workflow_args:
+        enable_overlong_penalty: true
+        penalty_factor: 1.0
+        max_response_length: 512
+        cache_length: 400
+    eval_tasksets:
+      - name: alfworld-eval
+        storage_type: file
+        path: 'examples/R3L/alfworld/alfworld_data'
+        split: 'test'
+        format:
+          prompt_key: 'task_id'
+        rollout_args:
+          temperature: 0.4
+    default_workflow_type: 'dapo_alfworld_workflow'
+  trainer_input:
+    experience_buffer:
+      name: alfworld_dapo_1.5B_buffer
+      storage_type: queue
+      replay_buffer:
+        enable: true
+        priority_fn: decay_limit_randomization
+      path: 'sqlite:///alfworld_dapo_1.5B.db'
+explorer:
+  runner_per_model: 32
+  eval_interval: 20
+  rollout_model:
+    engine_num: 2
+    tensor_parallel_size: 1
+    enable_prefix_caching: false
+    enforce_eager: false
+    dtype: bfloat16
+    seed: 42
+    gpu_memory_utilization: 0.7
+    enable_chunked_prefill: true
+data_processor:
+  experience_pipeline:
+    operators:
+      - name: "OPMD_filter"
+synchronizer:
+  sync_style: dynamic_by_explorer
+  sync_method: 'nccl'
+  sync_interval: 1
+  sync_timeout: 12000
+trainer:
+  save_interval: 20
+  grad_clip: 1.0
+  use_dynamic_bsz: true
+  max_token_len_per_gpu: 10240
+  ulysses_sequence_parallel_size: 1
+monitor:
+  monitor_type: wandb

examples/R3L/alfworld/dapo_7B.yaml

@@ -0,0 +1,88 @@
+project: "ALFWORLD"
+name: "ALFWORLD_RFT_Qwen_7B_DAPO"
+checkpoint_root_dir: ${oc.env:TRINITY_CHECKPOINT_ROOT_DIR,./checkpoints}
+algorithm:
+  algorithm_type: grpo
+  kl_loss_fn: k3
+  kl_loss_fn_args:
+    kl_coef: 0.01
+  repeat_times: 8
+  optimizer:
+    lr: 1e-6
+    lr_warmup_steps: 20
+  policy_loss_fn_args:
+    clip_range_low: 0.2
+    clip_range_high: 0.28
+model:
+  model_path: ${oc.env:TRINITY_MODEL_PATH,Qwen/Qwen2.5-7B-Instruct}
+  max_response_tokens: 512
+  max_model_len: 20480
+cluster:
+  node_num: 1
+  gpu_per_node: 4
+buffer:
+  total_epochs: 20
+  batch_size: 96
+  explorer_input:
+    taskset:
+      name: alfworld
+      storage_type: file
+      path: 'examples/R3L/alfworld/alfworld_data'
+      split: 'train'
+      format:
+        prompt_key: 'task_id'
+      rollout_args:
+        temperature: 1.0
+        logprobs: 0
+      workflow_args:
+        enable_overlong_penalty: true
+        penalty_factor: 1.0
+        max_response_length: 512
+        cache_length: 400
+    eval_tasksets:
+      - name: alfworld-eval
+        storage_type: file
+        path: 'examples/R3L/alfworld/alfworld_data'
+        split: 'test'
+        format:
+          prompt_key: 'task_id'
+        rollout_args:
+          temperature: 0.4
+    default_workflow_type: 'dapo_alfworld_workflow'
+  trainer_input:
+    experience_buffer:
+      name: alfworld_dapo_7B_buffer
+      storage_type: queue
+      replay_buffer:
+        enable: true
+        priority_fn: decay_limit_randomization
+      path: 'sqlite:///alfworld_dapo_7B.db'
+explorer:
+  runner_per_model: 32
+  eval_interval: 20
+  rollout_model:
+    engine_num: 2
+    tensor_parallel_size: 1
+    enable_prefix_caching: false
+    enforce_eager: false
+    dtype: bfloat16
+    seed: 42
+    gpu_memory_utilization: 0.7
+    enable_chunked_prefill: true
+data_processor:
+  experience_pipeline:
+    operators:
+      - name: "OPMD_filter"
+synchronizer:
+  sync_style: dynamic_by_explorer
+  sync_method: 'nccl'
+  sync_interval: 1
+  sync_timeout: 12000
+trainer:
+  save_interval: 20
+  grad_clip: 1.0
+  use_dynamic_bsz: true
+  max_token_len_per_gpu: 10240
+  ulysses_sequence_parallel_size: 1
+monitor:
+  monitor_type: wandb