Skip to content

Commit c368211

Browse files
DelinQuDelinQu
committed
openvla intergration
1 parent bcf6296 commit c368211

12 files changed

+1029
-2
lines changed

‎README.md‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,20 @@
1-
# SimplerEnv: Simulated Manipulation Policy Evaluation Environments for Real Robot Setups
1+
# SimplerEnv: Simulated Manipulation Policy Evaluation Environments for Real Robot Setups + OpenVLA policy
2+
3+
> [!IMPORTANT]
4+
> This a fork repo from [SimplerEnv](https://github.com/DelinQu/SimplerEnv) with [openvla](https://github.com/openvla/openvla) policy intergration. The evaluation results are not very satisfactory, possibly due to my code implementation and the OpenVLA model itself. I need about one day to complete all the experiments and organize the data before report the results. If you have any questions, please feel free to suggest code modifications and raise issues.
5+
> * the evaluation scripts keep consistent with original repo under ./scripts/
6+
7+
```bash
8+
openvla_bridge.sh
9+
openvla_drawer_variant_agg.sh
10+
openvla_drawer_visual_matching.sh
11+
openvla_move_near_variant_agg.sh
12+
openvla_move_near_visual_matching.sh
13+
openvla_pick_coke_can_variant_agg.sh
14+
openvla_pick_coke_can_visual_matching.sh
15+
openvla_put_in_drawer_variant_agg.sh
16+
openvla_put_in_drawer_visual_matching.sh
17+
```
218

319
[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/simpler-env/SimplerEnv/blob/main/example.ipynb)
420

‎scripts/openvla_bridge.sh‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
gpu_id=0
2+
policy_model=openvla
3+
ckpt_path="openvla/openvla-7b"
4+
5+
scene_name=bridge_table_1_v1
6+
robot=widowx
7+
rgb_overlay_path=ManiSkill2_real2sim/data/real_inpainting/bridge_real_eval_1.png
8+
robot_init_x=0.147
9+
robot_init_y=0.028
10+
export DISPLAY=:1.0
11+
# VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/nvidia_icd.json
12+
# python simpler_env/main_inference.py --policy-model ${policy_model} --ckpt-path ${ckpt_path} \
13+
# --robot ${robot} --policy-setup widowx_bridge \
14+
# --control-freq 5 --sim-freq 500 --max-episode-steps 60 \
15+
# --env-name PutCarrotOnPlateInScene-v0 --scene-name ${scene_name} \
16+
# --rgb-overlay-path ${rgb_overlay_path} \
17+
# --robot-init-x ${robot_init_x} ${robot_init_x} 1 --robot-init-y ${robot_init_y} ${robot_init_y} 1 --obj-variation-mode episode --obj-episode-range 0 24 \
18+
# --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1;
19+
#
20+
# python simpler_env/main_inference.py --policy-model ${policy_model} --ckpt-path ${ckpt_path} \
21+
# --robot ${robot} --policy-setup widowx_bridge \
22+
# --control-freq 5 --sim-freq 500 --max-episode-steps 60 \
23+
# --env-name StackGreenCubeOnYellowCubeBakedTexInScene-v0 --scene-name ${scene_name} \
24+
# --rgb-overlay-path ${rgb_overlay_path} \
25+
# --robot-init-x ${robot_init_x} ${robot_init_x} 1 --robot-init-y ${robot_init_y} ${robot_init_y} 1 --obj-variation-mode episode --obj-episode-range 0 24 \
26+
# --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1;
27+
#
28+
# python simpler_env/main_inference.py --policy-model ${policy_model} --ckpt-path ${ckpt_path} \
29+
# --robot ${robot} --policy-setup widowx_bridge \
30+
# --control-freq 5 --sim-freq 500 --max-episode-steps 60 \
31+
# --env-name PutSpoonOnTableClothInScene-v0 --scene-name ${scene_name} \
32+
# --rgb-overlay-path ${rgb_overlay_path} \
33+
# --robot-init-x ${robot_init_x} ${robot_init_x} 1 --robot-init-y ${robot_init_y} ${robot_init_y} 1 --obj-variation-mode episode --obj-episode-range 0 24 \
34+
# --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1;
35+
36+
37+
scene_name=bridge_table_1_v2
38+
robot=widowx_sink_camera_setup
39+
rgb_overlay_path=ManiSkill2_real2sim/data/real_inpainting/bridge_sink.png
40+
robot_init_x=0.127
41+
robot_init_y=0.06
42+
43+
python simpler_env/main_inference.py --policy-model ${policy_model} --ckpt-path ${ckpt_path} \
44+
--robot ${robot} --policy-setup widowx_bridge \
45+
--control-freq 5 --sim-freq 500 --max-episode-steps 120 \
46+
--env-name PutEggplantInBasketScene-v0 --scene-name ${scene_name} \
47+
--rgb-overlay-path ${rgb_overlay_path} \
48+
--robot-init-x ${robot_init_x} ${robot_init_x} 1 --robot-init-y ${robot_init_y} ${robot_init_y} 1 --obj-variation-mode episode --obj-episode-range 0 24 \
49+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1;
50+

‎scripts/openvla_drawer_variant_agg.sh‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# shader_dir=rt means that we turn on ray-tracing rendering; this is quite crucial for the open / close drawer task as policies often rely on shadows to infer depth
2+
declare -a ckpt_paths=(
3+
"openvla/openvla-7b"
4+
)
5+
6+
declare -a env_names=(
7+
OpenTopDrawerCustomInScene-v0
8+
OpenMiddleDrawerCustomInScene-v0
9+
OpenBottomDrawerCustomInScene-v0
10+
CloseTopDrawerCustomInScene-v0
11+
CloseMiddleDrawerCustomInScene-v0
12+
CloseBottomDrawerCustomInScene-v0
13+
)
14+
15+
EXTRA_ARGS="--enable-raytracing"
16+
17+
18+
# base setup
19+
scene_name=frl_apartment_stage_simple
20+
21+
EvalSim() {
22+
echo ${ckpt_path} ${env_name}
23+
24+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
25+
--robot google_robot_static \
26+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
27+
--env-name ${env_name} --scene-name ${scene_name} \
28+
--robot-init-x 0.65 0.85 3 --robot-init-y -0.2 0.2 3 \
29+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0.0 0.0 1 \
30+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
31+
${EXTRA_ARGS}
32+
}
33+
34+
35+
for ckpt_path in "${ckpt_paths[@]}"; do
36+
for env_name in "${env_names[@]}"; do
37+
EvalSim
38+
done
39+
done
40+
41+
42+
# backgrounds
43+
44+
declare -a scene_names=(
45+
"modern_bedroom_no_roof"
46+
"modern_office_no_roof"
47+
)
48+
49+
for scene_name in "${scene_names[@]}"; do
50+
for ckpt_path in "${ckpt_paths[@]}"; do
51+
for env_name in "${env_names[@]}"; do
52+
EXTRA_ARGS="--additional-env-build-kwargs shader_dir=rt"
53+
EvalSim
54+
done
55+
done
56+
done
57+
58+
59+
# lightings
60+
scene_name=frl_apartment_stage_simple
61+
62+
for ckpt_path in "${ckpt_paths[@]}"; do
63+
for env_name in "${env_names[@]}"; do
64+
EXTRA_ARGS="--additional-env-build-kwargs shader_dir=rt light_mode=brighter"
65+
EvalSim
66+
EXTRA_ARGS="--additional-env-build-kwargs shader_dir=rt light_mode=darker"
67+
EvalSim
68+
done
69+
done
70+
71+
72+
# new cabinets
73+
scene_name=frl_apartment_stage_simple
74+
75+
for ckpt_path in "${ckpt_paths[@]}"; do
76+
for env_name in "${env_names[@]}"; do
77+
EXTRA_ARGS="--additional-env-build-kwargs shader_dir=rt station_name=mk_station2"
78+
EvalSim
79+
EXTRA_ARGS="--additional-env-build-kwargs shader_dir=rt station_name=mk_station3"
80+
EvalSim
81+
done
82+
done

‎scripts/openvla_drawer_visual_matching.sh‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
# shader_dir=rt means that we turn on ray-tracing rendering; this is quite crucial for the open / close drawer task as policies often rely on shadows to infer depth
2+
declare -a ckpt_paths=(
3+
"openvla/openvla-7b"
4+
)
5+
6+
declare -a env_names=(
7+
OpenTopDrawerCustomInScene-v0
8+
OpenMiddleDrawerCustomInScene-v0
9+
OpenBottomDrawerCustomInScene-v0
10+
CloseTopDrawerCustomInScene-v0
11+
CloseMiddleDrawerCustomInScene-v0
12+
CloseBottomDrawerCustomInScene-v0
13+
)
14+
15+
# URDF variations
16+
declare -a urdf_version_arr=("recolor_cabinet_visual_matching_1" "recolor_tabletop_visual_matching_1" "recolor_tabletop_visual_matching_2" None)
17+
18+
for urdf_version in "${urdf_version_arr[@]}"; do
19+
20+
EXTRA_ARGS="--enable-raytracing --additional-env-build-kwargs station_name=mk_station_recolor light_mode=simple disable_bad_material=True urdf_version=${urdf_version}"
21+
22+
EvalOverlay() {
23+
# A0
24+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
25+
--robot google_robot_static \
26+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
27+
--env-name ${env_name} --scene-name dummy_drawer \
28+
--robot-init-x 0.644 0.644 1 --robot-init-y -0.179 -0.179 1 \
29+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 -0.03 -0.03 1 \
30+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
31+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_a0.png \
32+
${EXTRA_ARGS}
33+
34+
# A1
35+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
36+
--robot google_robot_static \
37+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
38+
--env-name ${env_name} --scene-name dummy_drawer \
39+
--robot-init-x 0.765 0.765 1 --robot-init-y -0.182 -0.182 1 \
40+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 -0.02 -0.02 1 \
41+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
42+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_a1.png \
43+
${EXTRA_ARGS}
44+
45+
# A2
46+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
47+
--robot google_robot_static \
48+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
49+
--env-name ${env_name} --scene-name dummy_drawer \
50+
--robot-init-x 0.889 0.889 1 --robot-init-y -0.203 -0.203 1 \
51+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 -0.06 -0.06 1 \
52+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
53+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_a2.png \
54+
${EXTRA_ARGS}
55+
56+
# B0
57+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
58+
--robot google_robot_static \
59+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
60+
--env-name ${env_name} --scene-name dummy_drawer \
61+
--robot-init-x 0.652 0.652 1 --robot-init-y 0.009 0.009 1 \
62+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1 \
63+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
64+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_b0.png \
65+
${EXTRA_ARGS}
66+
67+
# B1
68+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
69+
--robot google_robot_static \
70+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
71+
--env-name ${env_name} --scene-name dummy_drawer \
72+
--robot-init-x 0.752 0.752 1 --robot-init-y 0.009 0.009 1 \
73+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1 \
74+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
75+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_b1.png \
76+
${EXTRA_ARGS}
77+
78+
# B2
79+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
80+
--robot google_robot_static \
81+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
82+
--env-name ${env_name} --scene-name dummy_drawer \
83+
--robot-init-x 0.851 0.851 1 --robot-init-y 0.035 0.035 1 \
84+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1 \
85+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
86+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_b2.png \
87+
${EXTRA_ARGS}
88+
89+
# C0
90+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
91+
--robot google_robot_static \
92+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
93+
--env-name ${env_name} --scene-name dummy_drawer \
94+
--robot-init-x 0.665 0.665 1 --robot-init-y 0.224 0.224 1 \
95+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1 \
96+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
97+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_c0.png \
98+
${EXTRA_ARGS}
99+
100+
# C1
101+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
102+
--robot google_robot_static \
103+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
104+
--env-name ${env_name} --scene-name dummy_drawer \
105+
--robot-init-x 0.765 0.765 1 --robot-init-y 0.222 0.222 1 \
106+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 -0.025 -0.025 1 \
107+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
108+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_c1.png \
109+
${EXTRA_ARGS}
110+
111+
# C2
112+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
113+
--robot google_robot_static \
114+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
115+
--env-name ${env_name} --scene-name dummy_drawer \
116+
--robot-init-x 0.865 0.865 1 --robot-init-y 0.222 0.222 1 \
117+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 -0.025 -0.025 1 \
118+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
119+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_c2.png \
120+
${EXTRA_ARGS}
121+
}
122+
123+
124+
for ckpt_path in "${ckpt_paths[@]}"; do
125+
for env_name in "${env_names[@]}"; do
126+
EvalOverlay
127+
done
128+
done
129+
130+
131+
132+
done

0 commit comments

Comments
 (0)