Merge pull request #24 from facebookresearch:chaoyi/sam3

jc-bao · web-flow · commit 473037314894 · 2026-01-26T12:14:21.000-05:00
Chaoyi/sam3
diff --git a/docs/guide/quick-start.md b/docs/guide/quick-start.md
@@ -115,11 +115,19 @@ uv run examples/run_mjwp.py \
 Set environment variables:
 
 ```bash
+# humanoid only
 export TASK=dance
 export HAND_TYPE=humanoid
 export DATA_ID=0
 export ROBOT_TYPE=unitree_g1
 export DATASET_NAME=lafan
+
+# humanoid + object
+export TASK=move_largebox
+export HAND_TYPE=humanoid_object
+export DATA_ID=0
+export ROBOT_TYPE=unitree_g1
+export DATASET_NAME=omomo
 ```
 
 ### Run IK
@@ -141,18 +149,32 @@ uv run spider/process_datasets/locomujoco.py \
   --data-id=${DATA_ID} \
   --robot-type=${ROBOT_TYPE} \
   --embodiment-type=${HAND_TYPE}
+
+# for humanoid + object, we don't have a pipeline yet
+# you can directly download data from omniretarget
+# https://huggingface.co/datasets/omniretarget/OmniRetarget_Dataset
 ```
 
 ### Run Physics-Based Retargeting
 
 ```bash
+# for humanoid only
 uv run examples/run_mjwp.py \
   +override=humanoid \
   dataset_name=${DATASET_NAME} \
   task=${TASK} \
   data_id=${DATA_ID} \
   robot_type=${ROBOT_TYPE} \
   embodiment_type=${HAND_TYPE}
+
+# for humanoid + object, we don't have a pipeline yet
+uv run examples/run_mjwp.py \
+  +override=humanoid_object \
+  dataset_name=${DATASET_NAME} \
+  task=${TASK} \
+  data_id=${DATA_ID} \
+  robot_type=${ROBOT_TYPE} \
+  embodiment_type=${HAND_TYPE}
 ```
 
 ## Understanding the Output
diff --git a/docs/usage/viewers.md b/docs/usage/viewers.md
@@ -10,8 +10,42 @@ SPIDER supports the following viewer configurations:
 |--------|-------------|----------|
 | `mujoco` | Native MuJoCo viewer | Fast local visualization |
 | `rerun` | Rerun.io viewer | Remote development, rich logging |
+| `viser` | Viser web viewer | Lightweight remote visualization |
 
-To enable both, try `mujoco-rerun` viewer.
+To enable both, try `mujoco-rerun` or `mujoco-viser` viewer.
+
+## Viser Viewer
+
+[Viser](https://github.com/nerfstudio-project/viser) provides a lightweight web-based viewer for 3D scenes.
+
+### Local Usage
+
+```bash
+uv run examples/run_mjwp.py viewer=viser
+```
+
+This will:
+1. Start a Viser server
+2. Log simulation data in real-time
+3. Print a URL to open in your browser
+
+::: warning
+If you run headless (no X display), you may need to set a display before launching:
+
+```bash
+export DISPLAY=:1
+# or
+export MUJOCO_GL=egl
+```
+:::
+
+::: info
+If you see an import error, install the dependency:
+
+```bash
+pip install viser
+```
+:::
 
 ## Rerun Viewer
 
diff --git a/examples/config/default.yaml b/examples/config/default.yaml
@@ -43,6 +43,7 @@ num_samples: 1024
 temperature: 0.1
 max_num_iterations: 32
 improvement_threshold: 0.01
+improvement_check_steps: 1
 # Noise scheduling
 first_ctrl_noise_scale: 0.5
 last_ctrl_noise_scale: 1.0
@@ -68,7 +69,7 @@ object_rot_threshold: 0.3 # radians
 
 # === VISUALIZATION CONFIGURATION ===
 show_viewer: true
-viewer: mujoco-rerun # "mujoco" | "rerun" | "isaac"
+viewer: mujoco-rerun # "mujoco" | "rerun" | "viser" | "isaac"
 rerun_spawn: true
 save_video: true
 save_rerun: false
diff --git a/examples/config/override/gigahand.yaml b/examples/config/override/gigahand.yaml
@@ -0,0 +1,10 @@
+# @package _global_
+# Override configuration for gigahand dataset
+dataset_name: gigahand
+robot_type: xhand
+embodiment_type: bimanual
+task: p36-tea
+data_id: 10
+horizon: 2.0
+temperature: 1.0
+pos_noise_scale: 0.05
diff --git a/examples/config/override/oakink.yaml b/examples/config/override/oakink.yaml
@@ -0,0 +1,11 @@
+# @package _global_
+task: pick_spoon_bowl
+robot_type: xhand
+embodiment_type: bimanual
+data_id: 0
+temperature: 1.0
+horizon: 2.0
+base_pos_rew_scale: 0.1
+base_rot_rew_scale: 0.03
+rot_rew_scale: 0.1
+joint_noise_scale: 0.03
diff --git a/examples/run_hdmi.py b/examples/run_hdmi.py
@@ -45,12 +45,12 @@
     sync_env,
 )
 from spider.viewers import (
+    log_frame,
     render_image,
     setup_renderer,
     setup_viewer,
     update_viewer,
 )
-from spider.viewers.rerun_viewer import log_frame
 
 
 def main(config: Config):
@@ -221,7 +221,7 @@ def main(config: Config):
                         config, renderer, mj_model, mj_data, mj_data_ref
                     )
                     images.append(image)
-                if "rerun" in config.viewer:
+                if "rerun" in config.viewer or "viser" in config.viewer:
                     # manually log the state
                     log_frame(
                         mj_data,
diff --git a/examples/run_mjwp.py b/examples/run_mjwp.py
@@ -49,8 +49,13 @@
     step_env,
     sync_env,
 )
-from spider.viewers import render_image, setup_renderer, setup_viewer, update_viewer
-from spider.viewers.rerun_viewer import log_frame
+from spider.viewers import (
+    log_frame,
+    render_image,
+    setup_renderer,
+    setup_viewer,
+    update_viewer,
+)
 
 
 def main(config: Config):
@@ -174,7 +179,7 @@ def main(config: Config):
                             config, renderer, mj_model, mj_data, mj_data_ref
                         )
                         images.append(image)
-                if "rerun" in config.viewer:
+                if "rerun" in config.viewer or "viser" in config.viewer:
                     # manually log the state
                     log_frame(
                         mj_data,
diff --git a/pyproject.toml b/pyproject.toml
@@ -21,6 +21,7 @@ dependencies = [
   "opencv-python",
   "newton[examples] @ git+https://github.com/newton-physics/newton.git",
   "rerun-sdk==0.26.2",
+  "viser",
   "ipdb",
   "pymeshlab",
   "open3d",
diff --git a/spider/assets/robots/xhand/left.xml b/spider/assets/robots/xhand/left.xml
@@ -108,15 +108,15 @@
                       <geom name="left_thumb_rota2_visual" type="mesh" contype="0" conaffinity="0"
                         group="1" density="0" rgba="0.384314 0.384314 0.384314 1"
                         mesh="left_hand_thumb_rota_link2" />
-                      <geom name="collision_hand_left_thumb_0" type="capsule" size="0.008"
-                        fromto="0 0 0 0.045 0 0"
+                      <geom name="collision_hand_left_thumb_0" type="capsule" size="0.015"
+                        fromto="0 0 0 0.04 0 0"
                         rgba="0 1 0 1"
                         group="3" />
                       <geom size="0.001" pos="0.0504017 0 0" contype="0" conaffinity="0" group="1"
                         density="0" rgba="0.698039 0.698039 0.698039 1" />
-                      <site name="left_thumb_tip" pos="0.0504017 0 0" />
-                      <site name="track_hand_left_thumb_tip" pos="0.0504017 0 0" />
-                      <site name="trace_hand_left_thumb_tip" pos="0.0504017 0 0" />
+                      <site name="left_thumb_tip" pos="0.045 0 -0.015" />
+                      <site name="track_hand_left_thumb_tip" pos="0.045 0 -0.015" />
+                      <site name="trace_hand_left_thumb_tip" pos="0.045 0 -0.015" />
                     </body>
                   </body>
                 </body>
@@ -153,9 +153,9 @@
                         fromto="0 0 0 0 0 -0.03" rgba="0 1 0 1" group="3" />
                       <geom size="0.001" pos="0 0 -0.0425" contype="0" conaffinity="0" group="1"
                         density="0" rgba="0.698039 0.698039 0.698039 1" />
-                      <site name="left_index_tip" pos="0 0 -0.0425" />
-                      <site name="track_hand_left_index_tip" pos="0 0 -0.0425" />
-                      <site name="trace_hand_left_index_tip" pos="0 0 -0.0425" />
+                      <site name="left_index_tip" pos="0 -0.005 -0.03" />
+                      <site name="track_hand_left_index_tip" pos="0 -0.005 -0.03" />
+                      <site name="trace_hand_left_index_tip" pos="0 -0.005 -0.03" />
                     </body>
                   </body>
                 </body>
@@ -182,9 +182,9 @@
                       fromto="0 0 0 0 0 -0.03" rgba="0 1 0 1" group="3" />
                     <geom size="0.001" pos="0 0 -0.0425" contype="0" conaffinity="0" group="1"
                       density="0" rgba="0.698039 0.698039 0.698039 1" />
-                    <site name="left_middle_tip" pos="0 0 -0.0425" />
-                    <site name="track_hand_left_middle_tip" pos="0 0 -0.0425" />
-                    <site name="trace_hand_left_middle_tip" pos="0 0 -0.0425" />
+                    <site name="left_middle_tip" pos="0 -0.005 -0.03" />
+                    <site name="track_hand_left_middle_tip" pos="0 -0.005 -0.03" />
+                    <site name="trace_hand_left_middle_tip" pos="0 -0.005 -0.03" />
                   </body>
                 </body>
                 <body name="left_hand_ring_link1" pos="-0.016 -0.0065 -0.1052">
@@ -210,9 +210,9 @@
                       fromto="0 0 0 0 0 -0.03" rgba="0 1 0 1" group="3" />
                     <geom size="0.001" pos="0 0 -0.0425" contype="0" conaffinity="0" group="1"
                       density="0" rgba="0.698039 0.698039 0.698039 1" />
-                    <site name="left_ring_tip" pos="0 0 -0.0425" />
-                    <site name="track_hand_left_ring_tip" pos="0 0 -0.0425" />
-                    <site name="trace_hand_left_ring_tip" pos="0 0 -0.0425" />
+                    <site name="left_ring_tip" pos="0 -0.005 -0.03" />
+                    <site name="track_hand_left_ring_tip" pos="0 -0.005 -0.03" />
+                    <site name="trace_hand_left_ring_tip" pos="0 -0.005 -0.03" />
                   </body>
                 </body>
                 <body name="left_hand_pinky_link1" pos="-0.036 -0.0065 -0.1022">
@@ -238,9 +238,9 @@
                       fromto="0 0 0 0 0 -0.03" rgba="0 1 0 1" group="3" />
                     <geom size="0.001" pos="0 0 -0.0425" contype="0" conaffinity="0" group="1"
                       density="0" rgba="0.698039 0.698039 0.698039 1" />
-                    <site name="left_pinky_tip" pos="0 0 -0.0425" />
-                    <site name="track_hand_left_pinky_tip" pos="0 0 -0.0425" />
-                    <site name="trace_hand_left_pinky_tip" pos="0 0 -0.0425" />
+                    <site name="left_pinky_tip" pos="0 -0.005 -0.03" />
+                    <site name="track_hand_left_pinky_tip" pos="0 -0.005 -0.03" />
+                    <site name="trace_hand_left_pinky_tip" pos="0 -0.005 -0.03" />
                   </body>
                 </body>
               </body>
@@ -282,4 +282,4 @@
     <position name="left_pinky_joint1_position" joint="left_hand_pinky_joint1" />
     <position name="left_pinky_joint2_position" joint="left_hand_pinky_joint2" />
   </actuator>
-</mujoco>
+</mujoco>
diff --git a/spider/assets/robots/xhand/right.xml b/spider/assets/robots/xhand/right.xml
@@ -101,15 +101,16 @@
                       <geom name="right_thumb_rota2_visual" type="mesh" contype="0" conaffinity="0"
                         group="1" density="0" rgba="0.792157 0.819608 0.933333 1"
                         mesh="right_hand_thumb_rota_link2" />
-                      <geom name="collision_hand_right_thumb_0" type="capsule" size="0.008"
-                        fromto="0 0 0 0.045 0 0"
+                      <geom name="collision_hand_right_thumb_0" type="capsule" size="0.015"
+                        fromto="0 0 0 0.04 0 0"
                         rgba="0 1 0 1"
                         group="3" />
+
                       <geom size="0.001" pos="0.0504025 0 0" contype="0" conaffinity="0" group="1"
                         density="0" rgba="0.698039 0.698039 0.698039 1" />
-                      <site name="right_thumb_tip" pos="0.0504025 0 0" />
-                      <site name="track_hand_right_thumb_tip" pos="0.0504025 0 0" />
-                      <site name="trace_hand_right_thumb_tip" pos="0.0504025 0 0" />
+                      <site name="right_thumb_tip" pos="0.045 0 -0.015" />
+                      <site name="track_hand_right_thumb_tip" pos="0.045 0 -0.015" />
+                      <site name="trace_hand_right_thumb_tip" pos="0.045 0 -0.015" />
                     </body>
                   </body>
                 </body>
@@ -145,9 +146,9 @@
                         fromto="0 0 0 0 0 -0.03" rgba="0 1 0 1" group="3" />
                       <geom size="0.001" pos="0 0 -0.0425" contype="0" conaffinity="0" group="1"
                         density="0" rgba="0.698039 0.698039 0.698039 1" />
-                      <site name="right_index_tip" pos="0 0 -0.0425" />
-                      <site name="track_hand_right_index_tip" pos="0 0 -0.0425" />
-                      <site name="trace_hand_right_index_tip" pos="0 0 -0.0425" />
+                      <site name="right_index_tip" pos="0 0.005 -0.03" />
+                      <site name="track_hand_right_index_tip" pos="0 0.005 -0.03" />
+                      <site name="trace_hand_right_index_tip" pos="0 0.005 -0.03" />
                     </body>
                   </body>
                 </body>
@@ -174,9 +175,9 @@
                       fromto="0 0 0 0 0 -0.03" rgba="0 1 0 1" group="3" />
                     <geom size="0.001" pos="0 0 -0.042107" contype="0" conaffinity="0" group="1"
                       density="0" rgba="0.698039 0.698039 0.698039 1" />
-                    <site name="right_middle_tip" pos="0 0 -0.042107" />
-                    <site name="track_hand_right_middle_tip" pos="0 0 -0.042107" />
-                    <site name="trace_hand_right_middle_tip" pos="0 0 -0.042107" />
+                    <site name="right_middle_tip" pos="0 0.005 -0.03" />
+                    <site name="track_hand_right_middle_tip" pos="0 0.005 -0.03" />
+                    <site name="trace_hand_right_middle_tip" pos="0 0.005 -0.03" />
                   </body>
                 </body>
                 <body name="right_hand_ring_link1" pos="-0.016 0.0065 -0.1052">
@@ -202,9 +203,9 @@
                       fromto="0 0 0 0 0 -0.03" rgba="0 1 0 1" group="3" />
                     <geom size="0.001" pos="0 0 -0.0425" contype="0" conaffinity="0" group="1"
                       density="0" rgba="0.698039 0.698039 0.698039 1" />
-                    <site name="right_ring_tip" pos="0 0 -0.0425" />
-                    <site name="track_hand_right_ring_tip" pos="0 0 -0.0425" />
-                    <site name="trace_hand_right_ring_tip" pos="0 0 -0.0425" />
+                    <site name="right_ring_tip" pos="0 0.005 -0.03" />
+                    <site name="track_hand_right_ring_tip" pos="0 0.005 -0.03" />
+                    <site name="trace_hand_right_ring_tip" pos="0 0.005 -0.03" />
                   </body>
                 </body>
                 <body name="right_hand_pinky_link1" pos="-0.036 0.0065 -0.1022">
@@ -230,9 +231,9 @@
                       fromto="0 0 0 0 0 -0.03" rgba="0 1 0 1" group="3" />
                     <geom size="0.001" pos="0 0 -0.0425" contype="0" conaffinity="0" group="1"
                       density="0" rgba="0.698039 0.698039 0.698039 1" />
-                    <site name="right_pinky_tip" pos="0 0 -0.0425" />
-                    <site name="track_hand_right_pinky_tip" pos="0 0 -0.0425" />
-                    <site name="trace_hand_right_pinky_tip" pos="0 0 -0.0425" />
+                    <site name="right_pinky_tip" pos="0 0.005 -0.03" />
+                    <site name="track_hand_right_pinky_tip" pos="0 0.005 -0.03" />
+                    <site name="trace_hand_right_pinky_tip" pos="0 0.005 -0.03" />
                   </body>
                 </body>
               </body>
diff --git a/spider/config.py b/spider/config.py
@@ -101,7 +101,7 @@ class Config:
 
     # === VISUALIZATION CONFIGURATION ===
     show_viewer: bool = True
-    viewer: str = "mujoco"  # "mujoco" | "rerun" | "isaac"
+    viewer: str = "mujoco"  # "mujoco" | "rerun" | "viser" | "isaac"
     rerun_spawn: bool = False
     save_video: bool = True
     save_info: bool = True
diff --git a/spider/preprocess/ik.py b/spider/preprocess/ik.py
@@ -312,7 +312,7 @@ def main(
     # special case: allegro hand
     if robot_type in ["allegro", "metahand"]:
         sites_for_mimic.remove("right_pinky_tip")
-        sites_for_mimic.remove("left_pinky_tip")
+        sites_for_mimic.remove("left_pninky_tip")
 
     if embodiment_type == "right":
         sites_for_mimic = [s for s in sites_for_mimic if "right" in s]
diff --git a/spider/viewers/__init__.py b/spider/viewers/__init__.py
diff --git a/spider/viewers/viser_viewer.py b/spider/viewers/viser_viewer.py
diff --git a/uv.lock b/uv.lock
