Trinity-RFT/examples/R3L/alfworld/dapo_1.5B.yaml at f9ab6de6dfc4dd0fc4d710969cff3479a820d2c2 · shiweijiezero/Trinity-RFT · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
project: "ALFWORLD"
name: "ALFWORLD_RFT_Qwen_1.5B_DAPO"
checkpoint_root_dir: ${oc.env:TRINITY_CHECKPOINT_ROOT_DIR,./checkpoints}
algorithm:
  algorithm_type: grpo
  kl_loss_fn: k3
  kl_loss_fn_args:
    kl_coef: 0.01
  repeat_times: 8
  optimizer:
    lr: 1e-6
    lr_warmup_steps: 20
  policy_loss_fn_args:
    clip_range_low: 0.2
    clip_range_high: 0.28
model:
  model_path: ${oc.env:TRINITY_MODEL_PATH,Qwen/Qwen2.5-1.5B-Instruct}
  max_response_tokens: 512
  max_model_len: 20480
cluster:
  node_num: 1
  gpu_per_node: 4
buffer:
  total_epochs: 20
  batch_size: 48 # 如果是96会OOM
  explorer_input:
    taskset:
      name: alfworld
      storage_type: file
      path: 'examples/R3L/alfworld/alfworld_data'
      split: 'train'
      format:
        prompt_key: 'task_id'
      rollout_args:
        temperature: 1.0
        logprobs: 0
      workflow_args:
        enable_overlong_penalty: true
        penalty_factor: 1.0
        max_response_length: 512
        cache_length: 400
    eval_tasksets:
      - name: alfworld-eval
        storage_type: file
        path: 'examples/R3L/alfworld/alfworld_data'
        split: 'test'
        format:
          prompt_key: 'task_id'
        rollout_args:
          temperature: 0.4
    default_workflow_type: 'dapo_alfworld_workflow'
  trainer_input:
    experience_buffer:
      name: alfworld_dapo_1.5B_buffer
      storage_type: queue
      replay_buffer:
        enable: true
        priority_fn: decay_limit_randomization
      path: 'sqlite:///alfworld_dapo_1.5B.db'
explorer:
  runner_per_model: 32
  eval_interval: 20
  rollout_model:
    engine_num: 2
    tensor_parallel_size: 1
    enable_prefix_caching: false
    enforce_eager: false
    dtype: bfloat16
    seed: 42
    gpu_memory_utilization: 0.7
    enable_chunked_prefill: true
data_processor:
  experience_pipeline:
    operators:
      - name: "OPMD_filter"
synchronizer:
  sync_style: dynamic_by_explorer
  sync_method: 'nccl'
  sync_interval: 1
  sync_timeout: 12000
trainer:
  save_interval: 20
  grad_clip: 1.0
  use_dynamic_bsz: true
  max_token_len_per_gpu: 10240
  ulysses_sequence_parallel_size: 1
monitor:
  monitor_type: wandb