Skip to content

Commit 0a44c35

Browse files
DelinQuDelinQu
committed
openvla intergration
1 parent bcf6296 commit 0a44c35

11 files changed

+1012
-1
lines changed

‎scripts/openvla_bridge.sh‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
gpu_id=0
2+
policy_model=openvla
3+
ckpt_path="openvla/openvla-7b"
4+
5+
scene_name=bridge_table_1_v1
6+
robot=widowx
7+
rgb_overlay_path=ManiSkill2_real2sim/data/real_inpainting/bridge_real_eval_1.png
8+
robot_init_x=0.147
9+
robot_init_y=0.028
10+
export DISPLAY=:1.0
11+
# VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/nvidia_icd.json
12+
# python simpler_env/main_inference.py --policy-model ${policy_model} --ckpt-path ${ckpt_path} \
13+
# --robot ${robot} --policy-setup widowx_bridge \
14+
# --control-freq 5 --sim-freq 500 --max-episode-steps 60 \
15+
# --env-name PutCarrotOnPlateInScene-v0 --scene-name ${scene_name} \
16+
# --rgb-overlay-path ${rgb_overlay_path} \
17+
# --robot-init-x ${robot_init_x} ${robot_init_x} 1 --robot-init-y ${robot_init_y} ${robot_init_y} 1 --obj-variation-mode episode --obj-episode-range 0 24 \
18+
# --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1;
19+
#
20+
# python simpler_env/main_inference.py --policy-model ${policy_model} --ckpt-path ${ckpt_path} \
21+
# --robot ${robot} --policy-setup widowx_bridge \
22+
# --control-freq 5 --sim-freq 500 --max-episode-steps 60 \
23+
# --env-name StackGreenCubeOnYellowCubeBakedTexInScene-v0 --scene-name ${scene_name} \
24+
# --rgb-overlay-path ${rgb_overlay_path} \
25+
# --robot-init-x ${robot_init_x} ${robot_init_x} 1 --robot-init-y ${robot_init_y} ${robot_init_y} 1 --obj-variation-mode episode --obj-episode-range 0 24 \
26+
# --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1;
27+
#
28+
# python simpler_env/main_inference.py --policy-model ${policy_model} --ckpt-path ${ckpt_path} \
29+
# --robot ${robot} --policy-setup widowx_bridge \
30+
# --control-freq 5 --sim-freq 500 --max-episode-steps 60 \
31+
# --env-name PutSpoonOnTableClothInScene-v0 --scene-name ${scene_name} \
32+
# --rgb-overlay-path ${rgb_overlay_path} \
33+
# --robot-init-x ${robot_init_x} ${robot_init_x} 1 --robot-init-y ${robot_init_y} ${robot_init_y} 1 --obj-variation-mode episode --obj-episode-range 0 24 \
34+
# --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1;
35+
36+
37+
scene_name=bridge_table_1_v2
38+
robot=widowx_sink_camera_setup
39+
rgb_overlay_path=ManiSkill2_real2sim/data/real_inpainting/bridge_sink.png
40+
robot_init_x=0.127
41+
robot_init_y=0.06
42+
43+
python simpler_env/main_inference.py --policy-model ${policy_model} --ckpt-path ${ckpt_path} \
44+
--robot ${robot} --policy-setup widowx_bridge \
45+
--control-freq 5 --sim-freq 500 --max-episode-steps 120 \
46+
--env-name PutEggplantInBasketScene-v0 --scene-name ${scene_name} \
47+
--rgb-overlay-path ${rgb_overlay_path} \
48+
--robot-init-x ${robot_init_x} ${robot_init_x} 1 --robot-init-y ${robot_init_y} ${robot_init_y} 1 --obj-variation-mode episode --obj-episode-range 0 24 \
49+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1;
50+

‎scripts/openvla_drawer_variant_agg.sh‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# shader_dir=rt means that we turn on ray-tracing rendering; this is quite crucial for the open / close drawer task as policies often rely on shadows to infer depth
2+
declare -a ckpt_paths=(
3+
"openvla/openvla-7b"
4+
)
5+
6+
declare -a env_names=(
7+
OpenTopDrawerCustomInScene-v0
8+
OpenMiddleDrawerCustomInScene-v0
9+
OpenBottomDrawerCustomInScene-v0
10+
CloseTopDrawerCustomInScene-v0
11+
CloseMiddleDrawerCustomInScene-v0
12+
CloseBottomDrawerCustomInScene-v0
13+
)
14+
15+
EXTRA_ARGS="--enable-raytracing"
16+
17+
18+
# base setup
19+
scene_name=frl_apartment_stage_simple
20+
21+
EvalSim() {
22+
echo ${ckpt_path} ${env_name}
23+
24+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
25+
--robot google_robot_static \
26+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
27+
--env-name ${env_name} --scene-name ${scene_name} \
28+
--robot-init-x 0.65 0.85 3 --robot-init-y -0.2 0.2 3 \
29+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0.0 0.0 1 \
30+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
31+
${EXTRA_ARGS}
32+
}
33+
34+
35+
for ckpt_path in "${ckpt_paths[@]}"; do
36+
for env_name in "${env_names[@]}"; do
37+
EvalSim
38+
done
39+
done
40+
41+
42+
# backgrounds
43+
44+
declare -a scene_names=(
45+
"modern_bedroom_no_roof"
46+
"modern_office_no_roof"
47+
)
48+
49+
for scene_name in "${scene_names[@]}"; do
50+
for ckpt_path in "${ckpt_paths[@]}"; do
51+
for env_name in "${env_names[@]}"; do
52+
EXTRA_ARGS="--additional-env-build-kwargs shader_dir=rt"
53+
EvalSim
54+
done
55+
done
56+
done
57+
58+
59+
# lightings
60+
scene_name=frl_apartment_stage_simple
61+
62+
for ckpt_path in "${ckpt_paths[@]}"; do
63+
for env_name in "${env_names[@]}"; do
64+
EXTRA_ARGS="--additional-env-build-kwargs shader_dir=rt light_mode=brighter"
65+
EvalSim
66+
EXTRA_ARGS="--additional-env-build-kwargs shader_dir=rt light_mode=darker"
67+
EvalSim
68+
done
69+
done
70+
71+
72+
# new cabinets
73+
scene_name=frl_apartment_stage_simple
74+
75+
for ckpt_path in "${ckpt_paths[@]}"; do
76+
for env_name in "${env_names[@]}"; do
77+
EXTRA_ARGS="--additional-env-build-kwargs shader_dir=rt station_name=mk_station2"
78+
EvalSim
79+
EXTRA_ARGS="--additional-env-build-kwargs shader_dir=rt station_name=mk_station3"
80+
EvalSim
81+
done
82+
done

‎scripts/openvla_drawer_visual_matching.sh‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
# shader_dir=rt means that we turn on ray-tracing rendering; this is quite crucial for the open / close drawer task as policies often rely on shadows to infer depth
2+
declare -a ckpt_paths=(
3+
"openvla/openvla-7b"
4+
)
5+
6+
declare -a env_names=(
7+
OpenTopDrawerCustomInScene-v0
8+
OpenMiddleDrawerCustomInScene-v0
9+
OpenBottomDrawerCustomInScene-v0
10+
CloseTopDrawerCustomInScene-v0
11+
CloseMiddleDrawerCustomInScene-v0
12+
CloseBottomDrawerCustomInScene-v0
13+
)
14+
15+
# URDF variations
16+
declare -a urdf_version_arr=("recolor_cabinet_visual_matching_1" "recolor_tabletop_visual_matching_1" "recolor_tabletop_visual_matching_2" None)
17+
18+
for urdf_version in "${urdf_version_arr[@]}"; do
19+
20+
EXTRA_ARGS="--enable-raytracing --additional-env-build-kwargs station_name=mk_station_recolor light_mode=simple disable_bad_material=True urdf_version=${urdf_version}"
21+
22+
EvalOverlay() {
23+
# A0
24+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
25+
--robot google_robot_static \
26+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
27+
--env-name ${env_name} --scene-name dummy_drawer \
28+
--robot-init-x 0.644 0.644 1 --robot-init-y -0.179 -0.179 1 \
29+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 -0.03 -0.03 1 \
30+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
31+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_a0.png \
32+
${EXTRA_ARGS}
33+
34+
# A1
35+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
36+
--robot google_robot_static \
37+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
38+
--env-name ${env_name} --scene-name dummy_drawer \
39+
--robot-init-x 0.765 0.765 1 --robot-init-y -0.182 -0.182 1 \
40+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 -0.02 -0.02 1 \
41+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
42+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_a1.png \
43+
${EXTRA_ARGS}
44+
45+
# A2
46+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
47+
--robot google_robot_static \
48+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
49+
--env-name ${env_name} --scene-name dummy_drawer \
50+
--robot-init-x 0.889 0.889 1 --robot-init-y -0.203 -0.203 1 \
51+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 -0.06 -0.06 1 \
52+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
53+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_a2.png \
54+
${EXTRA_ARGS}
55+
56+
# B0
57+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
58+
--robot google_robot_static \
59+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
60+
--env-name ${env_name} --scene-name dummy_drawer \
61+
--robot-init-x 0.652 0.652 1 --robot-init-y 0.009 0.009 1 \
62+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1 \
63+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
64+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_b0.png \
65+
${EXTRA_ARGS}
66+
67+
# B1
68+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
69+
--robot google_robot_static \
70+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
71+
--env-name ${env_name} --scene-name dummy_drawer \
72+
--robot-init-x 0.752 0.752 1 --robot-init-y 0.009 0.009 1 \
73+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1 \
74+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
75+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_b1.png \
76+
${EXTRA_ARGS}
77+
78+
# B2
79+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
80+
--robot google_robot_static \
81+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
82+
--env-name ${env_name} --scene-name dummy_drawer \
83+
--robot-init-x 0.851 0.851 1 --robot-init-y 0.035 0.035 1 \
84+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1 \
85+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
86+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_b2.png \
87+
${EXTRA_ARGS}
88+
89+
# C0
90+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
91+
--robot google_robot_static \
92+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
93+
--env-name ${env_name} --scene-name dummy_drawer \
94+
--robot-init-x 0.665 0.665 1 --robot-init-y 0.224 0.224 1 \
95+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1 \
96+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
97+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_c0.png \
98+
${EXTRA_ARGS}
99+
100+
# C1
101+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
102+
--robot google_robot_static \
103+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
104+
--env-name ${env_name} --scene-name dummy_drawer \
105+
--robot-init-x 0.765 0.765 1 --robot-init-y 0.222 0.222 1 \
106+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 -0.025 -0.025 1 \
107+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
108+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_c1.png \
109+
${EXTRA_ARGS}
110+
111+
# C2
112+
python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
113+
--robot google_robot_static \
114+
--control-freq 3 --sim-freq 513 --max-episode-steps 113 \
115+
--env-name ${env_name} --scene-name dummy_drawer \
116+
--robot-init-x 0.865 0.865 1 --robot-init-y 0.222 0.222 1 \
117+
--robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 -0.025 -0.025 1 \
118+
--obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
119+
--rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_c2.png \
120+
${EXTRA_ARGS}
121+
}
122+
123+
124+
for ckpt_path in "${ckpt_paths[@]}"; do
125+
for env_name in "${env_names[@]}"; do
126+
EvalOverlay
127+
done
128+
done
129+
130+
131+
132+
done

0 commit comments

Comments
 (0)