Skip to content

Commit d915a5a

Browse files
nvddrnvddr
committed
Multi-channel recording support for hand and controller trackers
Introduce per-channel serialization for trackers that produce multiple independent data streams (e.g. left/right hand, left/right controller). - Add ITracker::get_record_channels() and channel_index param to ITrackerImpl::serialize() for multi-channel trackers - Update McapRecorder to create separate MCAP channels per sub-channel - HandTracker: serialize each hand individually (delete hands.fbs) - ControllerTracker: split API to get_left_controller()/get_right_controller(), store snapshots directly as structs, update ControllerData schema to hold a single ControllerSnapshot - Update all downstream: plugins, examples, source nodes, tests, bindings
1 parent e15384f commit d915a5a

39 files changed

+559
-385
lines changed

‎examples/oxr/python/test_controller_tracker.py‎

Lines changed: 29 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
Test script for ControllerTracker with simplified API.
66
77
Demonstrates:
8-
- Getting complete controller data for both left and right controllers
8+
- Getting left and right controller data via get_left_controller() and get_right_controller()
99
"""
1010

1111
import sys
@@ -57,22 +57,21 @@
5757

5858
# Test 5: Check initial controller state
5959
print("[Test 5] Checking controller state...")
60-
controller_data = controller_tracker.get_controller_data(session)
61-
left_snap = controller_data.left_controller
62-
right_snap = controller_data.right_controller
60+
left_snapshot = controller_tracker.get_left_controller(session)
61+
right_snapshot = controller_tracker.get_right_controller(session)
6362
print(
64-
f" Left controller: {'ACTIVE' if left_snap and left_snap.is_active else 'INACTIVE'}"
63+
f" Left controller: {'ACTIVE' if left_snapshot.is_active else 'INACTIVE'}"
6564
)
6665
print(
67-
f" Right controller: {'ACTIVE' if right_snap and right_snap.is_active else 'INACTIVE'}"
66+
f" Right controller: {'ACTIVE' if right_snapshot.is_active else 'INACTIVE'}"
6867
)
6968

70-
if left_snap and left_snap.is_active and left_snap.grip_pose.is_valid:
71-
pos = left_snap.grip_pose.pose.position
69+
if left_snapshot.is_active and left_snapshot.grip_pose.is_valid:
70+
pos = left_snapshot.grip_pose.pose.position
7271
print(f" Left grip position: [{pos.x:.3f}, {pos.y:.3f}, {pos.z:.3f}]")
7372

74-
if right_snap and right_snap.is_active and right_snap.grip_pose.is_valid:
75-
pos = right_snap.grip_pose.pose.position
73+
if right_snapshot.is_active and right_snapshot.grip_pose.is_valid:
74+
pos = right_snapshot.grip_pose.pose.position
7675
print(f" Right grip position: [{pos.x:.3f}, {pos.y:.3f}, {pos.z:.3f}]")
7776
print()
7877

@@ -101,40 +100,23 @@
101100
current_time = time.time()
102101
if current_time - last_status_print >= 0.5: # Print every 0.5 seconds
103102
elapsed = current_time - start_time
104-
controller_data = controller_tracker.get_controller_data(session)
105-
left_snap = controller_data.left_controller
106-
right_snap = controller_data.right_controller
103+
left_snapshot = controller_tracker.get_left_controller(session)
104+
right_snapshot = controller_tracker.get_right_controller(session)
107105

108106
# Show current state
109-
left_trigger = left_snap.inputs.trigger_value if left_snap else 0.0
110-
left_squeeze = left_snap.inputs.squeeze_value if left_snap else 0.0
111-
left_stick_x = left_snap.inputs.thumbstick_x if left_snap else 0.0
112-
left_stick_y = left_snap.inputs.thumbstick_y if left_snap else 0.0
113-
left_primary = (
114-
left_snap.inputs.primary_click if left_snap else False
115-
)
116-
left_secondary = (
117-
left_snap.inputs.secondary_click if left_snap else False
118-
)
119-
120-
right_trigger = (
121-
right_snap.inputs.trigger_value if right_snap else 0.0
122-
)
123-
right_squeeze = (
124-
right_snap.inputs.squeeze_value if right_snap else 0.0
125-
)
126-
right_stick_x = (
127-
right_snap.inputs.thumbstick_x if right_snap else 0.0
128-
)
129-
right_stick_y = (
130-
right_snap.inputs.thumbstick_y if right_snap else 0.0
131-
)
132-
right_primary = (
133-
right_snap.inputs.primary_click if right_snap else False
134-
)
135-
right_secondary = (
136-
right_snap.inputs.secondary_click if right_snap else False
137-
)
107+
left_trigger = left_snapshot.inputs.trigger_value
108+
left_squeeze = left_snapshot.inputs.squeeze_value
109+
left_stick_x = left_snapshot.inputs.thumbstick_x
110+
left_stick_y = left_snapshot.inputs.thumbstick_y
111+
left_primary = left_snapshot.inputs.primary_click
112+
left_secondary = left_snapshot.inputs.secondary_click
113+
114+
right_trigger = right_snapshot.inputs.trigger_value
115+
right_squeeze = right_snapshot.inputs.squeeze_value
116+
right_stick_x = right_snapshot.inputs.thumbstick_x
117+
right_stick_y = right_snapshot.inputs.thumbstick_y
118+
right_primary = right_snapshot.inputs.primary_click
119+
right_secondary = right_snapshot.inputs.secondary_click
138120

139121
# Build status strings
140122
left_status = f"Trig={left_trigger:.2f} Sq={left_squeeze:.2f}"
@@ -177,7 +159,7 @@
177159

178160
def print_controller_summary(hand_name, snapshot):
179161
print(f" {hand_name} Controller:")
180-
if snapshot and snapshot.is_active:
162+
if snapshot.is_active:
181163
print(" Status: ACTIVE")
182164

183165
# Poses from snapshot
@@ -212,10 +194,11 @@ def print_controller_summary(hand_name, snapshot):
212194
else:
213195
print(" Status: INACTIVE")
214196

215-
controller_data = controller_tracker.get_controller_data(session)
216-
print_controller_summary("Left", controller_data.left_controller)
197+
left_snapshot = controller_tracker.get_left_controller(session)
198+
right_snapshot = controller_tracker.get_right_controller(session)
199+
print_controller_summary("Left", left_snapshot)
217200
print()
218-
print_controller_summary("Right", controller_data.right_controller)
201+
print_controller_summary("Right", right_snapshot)
219202
print()
220203

221204
# Cleanup

‎examples/retargeting/python/dual_source_teleop_example.py‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@
2121
its OWN tracker -- the second source's tracker was never given to DeviceIO
2222
2323
Pipeline structure:
24-
ControllersSource("ctrl_left") ──> Se3AbsRetargeter ──> "left_ee_pose"
25-
ControllersSource("ctrl_right") ──> Se3AbsRetargeter ──> "right_ee_pose"
24+
ControllersSource("controller_left") ──> Se3AbsRetargeter ──> "left_ee_pose"
25+
ControllersSource("controller_right") ──> Se3AbsRetargeter ──> "right_ee_pose"
2626
└──> OutputCombiner
2727
"""
2828

@@ -60,16 +60,16 @@ def main() -> int:
6060
# source's tracker is orphaned.
6161

6262
print("[Step 1] Creating two ControllersSource nodes...")
63-
ctrl_left_source = ControllersSource(name="ctrl_left")
64-
ctrl_right_source = ControllersSource(name="ctrl_right")
63+
controller_left_source = ControllersSource(name="controller_left")
64+
controller_right_source = ControllersSource(name="controller_right")
6565
print(
66-
f" ✓ ControllersSource('ctrl_left') - tracker id: {id(ctrl_left_source.get_tracker())}"
66+
f" ✓ ControllersSource('controller_left') - tracker id: {id(controller_left_source.get_tracker())}"
6767
)
6868
print(
69-
f" ✓ ControllersSource('ctrl_right') - tracker id: {id(ctrl_right_source.get_tracker())}"
69+
f" ✓ ControllersSource('controller_right') - tracker id: {id(controller_right_source.get_tracker())}"
7070
)
7171
print(
72-
f" ⚠ Both trackers are type: {type(ctrl_left_source.get_tracker()).__name__}"
72+
f" ⚠ Both trackers are type: {type(controller_left_source.get_tracker()).__name__}"
7373
)
7474
print(" Only one will survive deduplication in TeleopSession.__enter__()")
7575

@@ -91,10 +91,12 @@ def main() -> int:
9191
left_se3 = Se3AbsRetargeter(left_se3_config, name="left_se3")
9292
connected_left = left_se3.connect(
9393
{
94-
ControllersSource.LEFT: ctrl_left_source.output(ControllersSource.LEFT),
94+
ControllersSource.LEFT: controller_left_source.output(
95+
ControllersSource.LEFT
96+
),
9597
}
9698
)
97-
print(" ✓ Left SE3: ctrl_left_source.controller_left -> left_ee_pose")
99+
print(" ✓ Left SE3: controller_left_source.controller_left -> left_ee_pose")
98100

99101
# Right arm SE3 retargeter (using right controller from second source)
100102
right_se3_config = Se3RetargeterConfig(
@@ -106,10 +108,12 @@ def main() -> int:
106108
right_se3 = Se3AbsRetargeter(right_se3_config, name="right_se3")
107109
connected_right = right_se3.connect(
108110
{
109-
ControllersSource.RIGHT: ctrl_right_source.output(ControllersSource.RIGHT),
111+
ControllersSource.RIGHT: controller_right_source.output(
112+
ControllersSource.RIGHT
113+
),
110114
}
111115
)
112-
print(" ✓ Right SE3: ctrl_right_source.controller_right -> right_ee_pose")
116+
print(" ✓ Right SE3: controller_right_source.controller_right -> right_ee_pose")
113117

114118
# ==================================================================
115119
# Step 3: Combine outputs

‎examples/retargeting/python/sources_example.py‎

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,10 @@ def main():
132132
hand_left_raw = hand_tracker.get_left_hand(session)
133133
hand_right_raw = hand_tracker.get_right_hand(session)
134134
head_raw = head_tracker.get_head(session)
135-
controller_data_raw = controller_tracker.get_controller_data(
135+
left_controller_raw = controller_tracker.get_left_controller(
136+
session
137+
)
138+
right_controller_raw = controller_tracker.get_right_controller(
136139
session
137140
)
138141

@@ -163,9 +166,9 @@ def main():
163166
for input_name, group_type in controllers_input_spec.items():
164167
tg = TensorGroup(group_type)
165168
if "left" in input_name.lower():
166-
tg[0] = controller_data_raw.left_controller
169+
tg[0] = left_controller_raw
167170
elif "right" in input_name.lower():
168-
tg[0] = controller_data_raw.right_controller
171+
tg[0] = right_controller_raw
169172
controllers_inputs[input_name] = tg
170173

171174
# ====================================================
@@ -218,21 +221,25 @@ def main():
218221
)
219222

220223
# Extract controller data
221-
left_ctrl = all_data[ControllersSource.LEFT]
222-
left_ctrl_active = left_ctrl[11] # is_active boolean
223-
left_trigger = left_ctrl[10] # trigger_value float
224+
left_controller = all_data[ControllersSource.LEFT]
225+
left_controller_active = left_controller[11] # is_active boolean
226+
left_trigger = left_controller[10] # trigger_value float
224227

225-
right_ctrl = all_data[ControllersSource.RIGHT]
226-
right_ctrl_active = right_ctrl[11] # is_active boolean
227-
right_trigger = right_ctrl[10] # trigger_value float
228+
right_controller = all_data[ControllersSource.RIGHT]
229+
right_controller_active = right_controller[11] # is_active boolean
230+
right_trigger = right_controller[10] # trigger_value float
228231

229232
print(" Controllers:")
230-
print(f" Left: {'ACTIVE' if left_ctrl_active else 'INACTIVE'}")
231-
if left_ctrl_active:
233+
print(
234+
f" Left: {'ACTIVE' if left_controller_active else 'INACTIVE'}"
235+
)
236+
if left_controller_active:
232237
print(f" Trigger: {left_trigger:4.2f}")
233238

234-
print(f" Right: {'ACTIVE' if right_ctrl_active else 'INACTIVE'}")
235-
if right_ctrl_active:
239+
print(
240+
f" Right: {'ACTIVE' if right_controller_active else 'INACTIVE'}"
241+
)
242+
if right_controller_active:
236243
print(f" Trigger: {right_trigger:4.2f}")
237244

238245
print()

‎examples/teleop_session_manager/python/external_inputs_example.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -236,13 +236,13 @@ def main() -> int:
236236
# specs = session.get_external_input_specs()
237237
# # specs == {"delta_manual": {"controller": ..., "ee_state": ...}}
238238
#
239-
# ctrl_tg = TensorGroup(ControllerInput())
240-
# ctrl_tg[0] = my_controller_data
239+
# controller_tg = TensorGroup(ControllerInput())
240+
# controller_tg[0] = my_controller_data
241241
# ee_tg = TensorGroup(RobotEndEffectorState())
242242
# ee_tg[0] = my_ee_position
243243
#
244244
# result = session.step(external_inputs={
245-
# "delta_manual": {"controller": ctrl_tg, "ee_state": ee_tg}
245+
# "delta_manual": {"controller": controller_tg, "ee_state": ee_tg}
246246
# })
247247
#
248248
# This works with any retargeter -- ValueInput is just a convenience

‎examples/teleop_session_manager/python/teleop_session_example.py‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@ def __init__(self, name: str):
5151
# Store previous positions for velocity computation
5252
self._prev_hand_left: Optional[np.ndarray] = None
5353
self._prev_hand_right: Optional[np.ndarray] = None
54-
self._prev_ctrl_left: Optional[np.ndarray] = None
55-
self._prev_ctrl_right: Optional[np.ndarray] = None
54+
self._prev_controller_left: Optional[np.ndarray] = None
55+
self._prev_controller_right: Optional[np.ndarray] = None
5656
self._prev_time: Optional[float] = None
5757

5858
def input_spec(self):
@@ -102,11 +102,15 @@ def compute(
102102
)
103103

104104
# Get controller positions (first tensor in ControllerInput is position array)
105-
ctrl_left_pos = inputs[ControllersSource.LEFT][0] # (3,) array
106-
ctrl_right_pos = inputs[ControllersSource.RIGHT][0]
105+
controller_left_pos = inputs[ControllersSource.LEFT][0] # (3,) array
106+
controller_right_pos = inputs[ControllersSource.RIGHT][0]
107107

108-
ctrl_left = np.array([ctrl_left_pos[0], ctrl_left_pos[1], ctrl_left_pos[2]])
109-
ctrl_right = np.array([ctrl_right_pos[0], ctrl_right_pos[1], ctrl_right_pos[2]])
108+
controller_left = np.array(
109+
[controller_left_pos[0], controller_left_pos[1], controller_left_pos[2]]
110+
)
111+
controller_right = np.array(
112+
[controller_right_pos[0], controller_right_pos[1], controller_right_pos[2]]
113+
)
110114

111115
current_time = time.time()
112116

@@ -120,17 +124,17 @@ def compute(
120124
hand_right_vel = float(
121125
np.linalg.norm(hand_right_wrist - self._prev_hand_right) / dt
122126
)
123-
ctrl_left_vel = float(
124-
np.linalg.norm(ctrl_left - self._prev_ctrl_left) / dt
127+
controller_left_vel = float(
128+
np.linalg.norm(controller_left - self._prev_controller_left) / dt
125129
)
126-
ctrl_right_vel = float(
127-
np.linalg.norm(ctrl_right - self._prev_ctrl_right) / dt
130+
controller_right_vel = float(
131+
np.linalg.norm(controller_right - self._prev_controller_right) / dt
128132
)
129133

130134
outputs["hand_velocity_left"][0] = hand_left_vel
131135
outputs["hand_velocity_right"][0] = hand_right_vel
132-
outputs["controller_velocity_left"][0] = ctrl_left_vel
133-
outputs["controller_velocity_right"][0] = ctrl_right_vel
136+
outputs["controller_velocity_left"][0] = controller_left_vel
137+
outputs["controller_velocity_right"][0] = controller_right_vel
134138
else:
135139
outputs["hand_velocity_left"][0] = 0.0
136140
outputs["hand_velocity_right"][0] = 0.0
@@ -146,8 +150,8 @@ def compute(
146150
# Store current positions for next frame
147151
self._prev_hand_left = hand_left_wrist
148152
self._prev_hand_right = hand_right_wrist
149-
self._prev_ctrl_left = ctrl_left
150-
self._prev_ctrl_right = ctrl_right
153+
self._prev_controller_left = controller_left
154+
self._prev_controller_right = controller_right
151155
self._prev_time = current_time
152156

153157

@@ -196,14 +200,14 @@ def main():
196200

197201
hand_left_vel = result["hand_velocity_left"][0]
198202
hand_right_vel = result["hand_velocity_right"][0]
199-
ctrl_left_vel = result["controller_velocity_left"][0]
200-
ctrl_right_vel = result["controller_velocity_right"][0]
203+
controller_left_vel = result["controller_velocity_left"][0]
204+
controller_right_vel = result["controller_velocity_right"][0]
201205

202206
if session.frame_count % 30 == 0:
203207
elapsed = session.get_elapsed_time()
204208
print(
205209
f"[{elapsed:5.1f}s] Hand L/R: {hand_left_vel:.3f}/{hand_right_vel:.3f} m/s "
206-
f"Ctrl L/R: {ctrl_left_vel:.3f}/{ctrl_right_vel:.3f} m/s"
210+
f"Controller L/R: {controller_left_vel:.3f}/{controller_right_vel:.3f} m/s"
207211
)
208212

209213
time.sleep(0.016) # ~60 FPS

0 commit comments

Comments
 (0)