fix: fix pi grpo yaml inconsistency (RLinf#450)

Signed-off-by: chenkang <455130517@qq.com>

Author: chenkang455

examples/embodiment/config/libero_10_grpo_openpi.yaml

@@ -38,9 +38,6 @@ runner:
   resume_dir: null
 
 algorithm:
-  auto_reset: False
-  ignore_terminations: False
-  use_fixed_reset_state_ids: True
   normalize_advantages: True
   kl_penalty: kl  # how to estimate kl divergence: kl or kl_penalty
   group_size: 8
@@ -52,7 +49,7 @@ algorithm:
   logprob_type: chunk_level
   entropy_type: token_level
 
-  update_epoch: 4
+  update_epoch: 2
   adv_type: grpo
   loss_type: actor
   loss_agg_func: "token-mean" 
@@ -67,7 +64,7 @@ algorithm:
   gamma: 0.99
   gae_lambda: 0.95
 
-  filter_rewards: False
+  filter_rewards: True
   rewards_lower_bound: 0.1
   rewards_upper_bound: 0.9
   # params for generation
@@ -97,18 +94,17 @@ env:
 
   train:
     total_num_envs: 64
-    max_episode_steps: 480 # max episode steps for truncation
-    max_steps_per_rollout_epoch: 480
     reward_coef: ${algorithm.reward_coef}
     group_size: ${algorithm.group_size}
+    max_episode_steps: 480 # max episode steps for truncation
+    max_steps_per_rollout_epoch: 480
   eval:
     total_num_envs: 500
     auto_reset: True
     ignore_terminations: True
     max_episode_steps: 480
     max_steps_per_rollout_epoch: 480
     group_size: 1
-    use_fixed_reset_state_ids: True
     is_eval: True
     video_cfg:
       save_video: True

examples/embodiment/config/libero_10_grpo_openpi_pi05.yaml

@@ -63,7 +63,7 @@ algorithm:
   gamma: 0.99
   gae_lambda: 0.95
 
-  filter_rewards: False
+  filter_rewards: True
   rewards_lower_bound: 0.1
   rewards_upper_bound: 0.9
   # params for generation
@@ -142,6 +142,7 @@ actor:
   # Override the default values in model/pi0_5
   model:
     model_path: "/path/to/model/RLinf-Pi05-SFT"
+    num_steps: 4
 
   optim:
     lr: 5.0e-6

examples/embodiment/config/libero_goal_grpo_openpi.yaml

@@ -49,7 +49,7 @@ algorithm:
   logprob_type: chunk_level
   entropy_type: token_level
 
-  update_epoch: 4
+  update_epoch: 2
   adv_type: grpo
   loss_type: actor
   loss_agg_func: "token-mean" 
@@ -64,7 +64,7 @@ algorithm:
   gamma: 0.99
   gae_lambda: 0.95
 
-  filter_rewards: False
+  filter_rewards: True
   rewards_lower_bound: 0.1
   rewards_upper_bound: 0.9
   # params for generation
@@ -105,7 +105,6 @@ env:
     max_episode_steps: 320
     max_steps_per_rollout_epoch: 320
     group_size: 1
-    use_fixed_reset_state_ids: True
     is_eval: True
     video_cfg:
       save_video: True
@@ -126,7 +125,7 @@ rollout:
   model:
     model_path: "/path/to/model/RLinf-Pi0-SFT-Spatial-Object-Goal"
     precision: ${actor.model.precision}
-    
+
 actor:
   group_name: "ActorGroup"
   channel:

examples/embodiment/config/libero_goal_grpo_openpi_pi05.yaml

@@ -48,7 +48,7 @@ algorithm:
   reward_type: chunk_level
   logprob_type: chunk_level
   entropy_type: token_level
-  update_epoch: 4
+  update_epoch: 1
 
   adv_type: grpo
   loss_type: actor
@@ -64,7 +64,7 @@ algorithm:
   gamma: 0.99
   gae_lambda: 0.95
 
-  filter_rewards: False
+  filter_rewards: True
   rewards_lower_bound: 0.1
   rewards_upper_bound: 0.9
   # params for generation
@@ -142,8 +142,11 @@ actor:
   # Override the default values in model/pi0_5
   model:
     model_path: "/path/to/model/RLinf-Pi05-SFT"
-    model_type: "openpi"
     num_action_chunks: 5
+    model_type: "openpi"
+    # openpi specific parameters
+    openpi:
+      noise_level: 0.3
 
   optim:
     lr: 5.0e-6

examples/embodiment/config/libero_object_grpo_openpi.yaml

@@ -49,7 +49,7 @@ algorithm:
   logprob_type: chunk_level
   entropy_type: token_level
 
-  update_epoch: 4
+  update_epoch: 2
   adv_type: grpo
   loss_type: actor
   loss_agg_func: "token-mean" 
@@ -64,7 +64,7 @@ algorithm:
   gamma: 0.99
   gae_lambda: 0.95
 
-  filter_rewards: False
+  filter_rewards: True
   rewards_lower_bound: 0.1
   rewards_upper_bound: 0.9
   # params for generation
@@ -98,16 +98,13 @@ env:
     group_size: ${algorithm.group_size}
     max_episode_steps: 240
     max_steps_per_rollout_epoch: 240
-    use_fixed_reset_state_ids: True
-    use_ordered_reset_state_ids: False
   eval:
     total_num_envs: 500
     auto_reset: True
     ignore_terminations: True
     max_episode_steps: 240
     max_steps_per_rollout_epoch: 240
     group_size: 1
-    use_fixed_reset_state_ids: True
     is_eval: True
     video_cfg:
       save_video: True

examples/embodiment/config/libero_object_grpo_openpi_pi05.yaml

@@ -64,7 +64,7 @@ algorithm:
   gamma: 0.99
   gae_lambda: 0.95
 
-  filter_rewards: False
+  filter_rewards: True
   rewards_lower_bound: 0.1
   rewards_upper_bound: 0.9
   # params for generation
@@ -98,8 +98,6 @@ env:
     group_size: ${algorithm.group_size}
     max_episode_steps: 240
     max_steps_per_rollout_epoch: 240
-    use_fixed_reset_state_ids: True
-    use_ordered_reset_state_ids: False
   eval:
     total_num_envs: 500
     auto_reset: True
@@ -147,7 +145,8 @@ actor:
     model_path: "/path/to/model/RLinf-Pi05-SFT"
     model_type: "openpi"
     num_action_chunks: 5 # interface for the env
-    num_steps: 3
+    openpi:
+      noise_level: 0.3
 
   optim:
     lr: 5.0e-6

examples/embodiment/config/libero_spatial_grpo_openpi.yaml

@@ -50,7 +50,7 @@ algorithm:
   logprob_type: chunk_level
   entropy_type: token_level
 
-  update_epoch: 4
+  update_epoch: 2
   adv_type: grpo
   loss_type: actor
   loss_agg_func: "token-mean" 
@@ -65,7 +65,7 @@ algorithm:
   gamma: 0.99
   gae_lambda: 0.95
 
-  filter_rewards: False
+  filter_rewards: True
   rewards_lower_bound: 0.1
   rewards_upper_bound: 0.9
   # params for generation

examples/embodiment/config/libero_spatial_grpo_openpi_pi05.yaml

@@ -64,7 +64,7 @@ algorithm:
   gamma: 0.99
   gae_lambda: 0.95
 
-  filter_rewards: False
+  filter_rewards: True
   rewards_lower_bound: 0.1
   rewards_upper_bound: 0.9
   # params for generation