diff --git a/build.sh b/build.sh index 6df7acee..09bc4429 100755 --- a/build.sh +++ b/build.sh @@ -25,5 +25,10 @@ if [ $EULA_STATUS -ne 0 ]; then exit 1 fi +# Provision libsurvive into _build/target-deps on Linux (no-op if already installed) +if [[ "$(uname -s)" == "Linux" ]]; then + bash "${SCRIPT_DIR}/source/scripts/build_libsurvive.sh" "${SCRIPT_DIR}" +fi + set -e source "$SCRIPT_DIR/repo.sh" build $@ || exit $? diff --git a/source/apps/isaacsim.exp.base.kit b/source/apps/isaacsim.exp.base.kit index 0cf7cba9..4fc7382c 100644 --- a/source/apps/isaacsim.exp.base.kit +++ b/source/apps/isaacsim.exp.base.kit @@ -44,6 +44,7 @@ keywords = ["experience", "app", "usd"] # That makes it browsable in UI with "ex "isaacsim.simulation_app" = {} "isaacsim.storage.native" = {} "isaacsim.util.debug_draw" = {} +"isaacsim.xr.input_devices" = {} "omni.isaac.core_archive" = {} "omni.isaac.ml_archive" = {} "omni.kit.loop-isaac" = {} diff --git a/source/apps/isaacsim.exp.extscache.kit b/source/apps/isaacsim.exp.extscache.kit index b1eaccf5..a988e7bc 100644 --- a/source/apps/isaacsim.exp.extscache.kit +++ b/source/apps/isaacsim.exp.extscache.kit @@ -101,6 +101,7 @@ folders = [ "isaacsim.util.debug_draw" = {} "isaacsim.util.merge_mesh" = {} "isaacsim.util.physics" = {} +"isaacsim.xr.input_devices" = {} "isaacsim.xr.openxr" = {} "omni.exporter.urdf" = {} "omni.isaac.app.selector" = {} diff --git a/source/extensions/isaacsim.core.includes/include/isaacsim/core/includes/core/Platform.h b/source/extensions/isaacsim.core.includes/include/isaacsim/core/includes/core/Platform.h index cd449a7d..805b91ab 100644 --- a/source/extensions/isaacsim.core.includes/include/isaacsim/core/includes/core/Platform.h +++ b/source/extensions/isaacsim.core.includes/include/isaacsim/core/includes/core/Platform.h @@ -15,6 +15,7 @@ #pragma once +#include #include #include diff --git a/source/extensions/isaacsim.xr.input_devices/bindings/isaacsim.xr.input_devices/IsaacsimXrInputDevicesBindings.cpp b/source/extensions/isaacsim.xr.input_devices/bindings/isaacsim.xr.input_devices/IsaacsimXrInputDevicesBindings.cpp new file mode 100644 index 00000000..caa9b0b3 --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/bindings/isaacsim.xr.input_devices/IsaacsimXrInputDevicesBindings.cpp @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +CARB_BINDINGS("isaacsim.xr.input_devices.python") + +namespace +{ +PYBIND11_MODULE(_isaac_xr_input_devices, m) +{ + using namespace carb; + using namespace isaacsim::xr::input_devices; + + auto carbModule = py::module::import("carb"); + + py::class_(m, "IsaacSimManusTracker") + .def(py::init<>()) + .def("initialize", &IsaacSimManusTracker::initialize, R"( + Initialize Manus SDK (adapted from existing implementation). + + Returns: + bool: True if initialization was successful, False otherwise + )") + .def("get_glove_data", &IsaacSimManusTracker::get_glove_data, R"( + Get glove data in IsaacSim format. + + Returns: + Dict[str, List[float]]: Dictionary mapping glove data keys to values + )") + .def("cleanup", &IsaacSimManusTracker::cleanup, R"( + Cleanup SDK resources. + )"); +} +} diff --git a/source/extensions/isaacsim.xr.input_devices/config/extension.toml b/source/extensions/isaacsim.xr.input_devices/config/extension.toml new file mode 100644 index 00000000..d9800eb4 --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/config/extension.toml @@ -0,0 +1,31 @@ +[package] +version = "1.0.0" +category = "XR" +title = "Isaac Sim XR Input Devices" +description = "Provides XR input device support for Manus gloves and Vive trackers" +authors = ["yuanchenl@nvidia.com"] +repository = "https://gitlab-master.nvidia.com/omniverse/isaac/omni_isaac_sim/-/tree/develop/source/extensions/isaacsim.xr.input_devices" +keywords = ["isaac", "xr", "manus", "vive", "trackers", "gloves", "input", "devices"] +changelog = "docs/CHANGELOG.md" +readme = "docs/README.md" +writeTarget.kit = true + +[dependencies] +"omni.kit.xr.core" = {} +"isaacsim.core.api" = {} +"isaacsim.core.deprecation_manager" = {} +"isaacsim.core.utils" = {} + +[[python.module]] +name = "isaacsim.xr.input_devices" + +[[test]] +dependencies = [ + "isaacsim.core.api", + "isaacsim.core.utils", +] +args = [ + "--/app/asyncRendering=0", + "--/app/fastShutdown=1", + "--vulkan", +] diff --git a/source/extensions/isaacsim.xr.input_devices/data/icon.png b/source/extensions/isaacsim.xr.input_devices/data/icon.png new file mode 100644 index 00000000..56a758d9 --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/data/icon.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30231e6fe2c3747b0456cb696d99fb7ebce25af1c3ac08c08e44736f20916a5e +size 2698 diff --git a/source/extensions/isaacsim.xr.input_devices/data/preview.png b/source/extensions/isaacsim.xr.input_devices/data/preview.png new file mode 100644 index 00000000..02350eb3 --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/data/preview.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf6ccd9d5e1e2fbce73e99ee009a67536adac455157bb68a1e80a187b4ba313a +size 19265 diff --git a/source/extensions/isaacsim.xr.input_devices/docs/CHANGELOG.md b/source/extensions/isaacsim.xr.input_devices/docs/CHANGELOG.md new file mode 100644 index 00000000..5fa488fc --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/docs/CHANGELOG.md @@ -0,0 +1,35 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [1.0.0] - 2025-08-08 + +### Added +- Initial release of `isaacsim.xr.input_devices` extension +- Manus gloves integration via C++ tracker with Python bindings +- Vive tracker integration via `pysurvive` (libsurvive) with mock fallback +- Unified Python API: `get_xr_device_integration().get_all_device_data()` +- Left/right wrist mapping resolution between Vive trackers and OpenXR wrists +- Static scene-to-lighthouse transform estimation with clustering and averaging +- Per-device connection status and last-data timestamps +- Sample visualization script for Manus and Vive devices +- Documentation and simple CLI-style tests + +### Details +- Update cadence is rate-limited to 100 Hz +- Wrist mapping considers both pairings (WM0→L/WM1→R vs WM1→L/WM0→R), + accumulates per-frame translation/rotation errors, and chooses the better pairing +- Chosen pairing candidates are clustered (position/orientation margins) and averaged + to produce a stable `scene_T_lighthouse_static` transform +- Vive tracker poses are transformed into scene coordinates using the static transform +- Manus joint poses (relative to the wrist) and transformed into scene coordinates + using the resolved wrist mapping from vive tracker poses + +### Dependencies +- Isaac Sim Core APIs +- Omniverse Kit runtime (for logging and extension lifecycle) +- Manus SDK (optional; otherwise uses mock data) +- libsurvive / pysurvive (optional; otherwise uses mock data) diff --git a/source/extensions/isaacsim.xr.input_devices/docs/README.md b/source/extensions/isaacsim.xr.input_devices/docs/README.md new file mode 100644 index 00000000..0457b347 --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/docs/README.md @@ -0,0 +1,163 @@ +# IsaacSim XR Input Devices Extension + +This extension provides XR input device support for Manus gloves and Vive trackers, with a unified Python API and automatic wrist mapping between trackers and OpenXR wrists. + +## Features + +- **Manus gloves**: Real-time hand/finger joint poses via C++ tracker with Python bindings +- **Vive trackers**: 6DOF poses via libsurvive (`pysurvive`) with mock fallback +- **Left/Right mapping**: Resolves which Vive tracker corresponds to left/right wrist +- **Scene alignment**: Estimates a stable scene↔lighthouse transform from early samples +- **Unified API**: Single call to fetch all device data and device status +- **Visualization sample**: Renders gloves/trackers as cubes in Isaac Sim + +## Prerequisites + +### Hardware +- **Manus Gloves**: Manus Prime/MetaGloves with license dongle +- **Vive Trackers**: Lighthouse tracking (SteamVR base stations + trackers) + +### Software +- **Manus SDK**: Bundled in Isaac Sim target-deps (mock used if missing) +- **libsurvive / pysurvive**: Required for real Vive tracking; otherwise mock data is used + +Install libsurvive (either system-wide or in your home directory): + +System-wide: +```bash +sudo apt update +sudo apt install -y build-essential zlib1g-dev libx11-dev libusb-1.0-0-dev \ + freeglut3-dev liblapacke-dev libopenblas-dev libatlas-base-dev cmake + +git clone https://github.com/cntools/libsurvive.git +cd libsurvive +sudo cp ./useful_files/81-vive.rules /etc/udev/rules.d/ +sudo udevadm control --reload-rules && sudo udevadm trigger +make && cmake . && make -j"$(nproc)" +sudo make install && sudo ldconfig +``` + +User (home) install (recommended during development): +```bash +git clone https://github.com/cntools/libsurvive.git ~/libsurvive +cd ~/libsurvive +sudo cp ./useful_files/81-vive.rules /etc/udev/rules.d/ +sudo udevadm control --reload-rules && sudo udevadm trigger +make && cmake . && make -j"$(nproc)" + +# Ensure Python can find pysurvive bindings +export PYTHONPATH="$HOME/libsurvive/bindings/python:$PYTHONPATH" +``` + +Note: The Vive tracker wrapper currently prepends a libsurvive Python path. Adjust it as needed for your environment or set `PYTHONPATH` as shown above. + +## Building + +```bash +# Build Isaac Sim (includes this extension) +./build.sh +``` + +## Usage + +### Sample +Run the visualization sample that renders Manus joints (red cubes) and Vive trackers (blue cubes): + +```bash +cd /path/to/IsaacSim +./_build/linux-x86_64/release/python.sh \ + source/standalone_examples/api/isaacsim.xr.input_devices/manus_vive_tracking_sample.py +``` + +### Python API + +```python +from isaacsim.xr.input_devices.impl.xr_device_integration import get_xr_device_integration + +# Obtain the shared integration instance from the extension +integration = get_xr_device_integration() + +# Fetch all device data +all_data = integration.get_all_device_data() + +manus_data = all_data.get('manus_gloves', {}) +vive_data = all_data.get('vive_trackers', {}) +status = all_data.get('device_status', {}) + +print(f"Manus connected: {status.get('manus_gloves', {}).get('connected', False)}") +print(f"Vive connected: {status.get('vive_trackers', {}).get('connected', False)}") +``` + +### Data Format + +```python +{ + 'manus_gloves': { + 'left_0': { + 'position': [x, y, z], + 'orientation': [w, x, y, z] + }, + 'left_1': { ... }, + 'right_0': { ... }, + # ... per-joint entries + }, + 'vive_trackers': { + '': { + 'position': [x, y, z], + 'orientation': [w, x, y, z] + }, + # e.g., 'WM0', 'WM1', or device names from libsurvive + }, + 'device_status': { + 'manus_gloves': {'connected': bool, 'last_data_time': float}, + 'vive_trackers': {'connected': bool, 'last_data_time': float}, + 'left_hand_connected': bool, + 'right_hand_connected': bool + } +} +``` + +## How left/right mapping is determined + +- Detect connected OpenXR wrists (left/right) and available Vive wrist markers (e.g., WM0/WM1) +- For each frame, compute candidate transforms for both pairings: + - Pair A: WM0→Left, WM1→Right + - Pair B: WM1→Left, WM0→Right +- Accumulate translation/rotation deltas per pairing when both wrists and trackers are present +- Choose the pairing: + - Prefer the pairing with more samples initially + - Once there are enough paired frames, choose the one with lower accumulated error +- Cluster the chosen pairing’s transforms and average them to estimate a stable + scene↔lighthouse transform (`scene_T_lighthouse_static`) +- Use the resolved mapping and transform to place all Vive and Manus data in scene coordinates + +## Troubleshooting + +- Manus license issues: replug license dongle; ensure SDK libraries are discoverable +- libsurvive conflicts: ensure SteamVR is NOT running concurrently +- No Vive devices: verify udev rules and USB connections (solid tracker LED) +- Python import for `pysurvive`: set `PYTHONPATH` to libsurvive bindings path + +## Extension Layout + +``` +isaacsim.xr.input_devices/ +├── bindings/ # Pybind11 bindings +├── include/ # C++ headers +├── plugins/ # C++ implementations +├── python/ +│ └── impl/ +│ ├── extension.py # Extension lifecycle +│ ├── xr_device_integration.py # Orchestrates devices & transforms +│ ├── manus_tracker.py # Manus wrapper +│ └── vive_tracker.py # Vive wrapper +└── docs/ + ├── README.md + └── CHANGELOG.md +``` + +## License + +Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. + +SPDX-License-Identifier: Apache-2.0 diff --git a/source/extensions/isaacsim.xr.input_devices/include/isaacsim/xr/input_devices/ManusTracker.h b/source/extensions/isaacsim.xr.input_devices/include/isaacsim/xr/input_devices/ManusTracker.h new file mode 100644 index 00000000..bba10dff --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/include/isaacsim/xr/input_devices/ManusTracker.h @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "ManusSDK.h" + +#ifdef _MSC_VER +# if ISAACSIM_XR_INPUT_DEVICES_EXPORT +# define ISAACSIM_XR_INPUT_DEVICES_DLL_EXPORT __declspec(dllexport) +# else +# define ISAACSIM_XR_INPUT_DEVICES_DLL_EXPORT __declspec(dllimport) +# endif +#else +# define ISAACSIM_XR_INPUT_DEVICES_DLL_EXPORT __attribute__((visibility("default"))) +#endif + +namespace isaacsim +{ +namespace xr +{ +namespace input_devices +{ + +class ISAACSIM_XR_INPUT_DEVICES_DLL_EXPORT IsaacSimManusTracker +{ +public: + IsaacSimManusTracker(); + ~IsaacSimManusTracker(); + + bool initialize(); + std::unordered_map> get_glove_data(); + void cleanup(); + +private: + static IsaacSimManusTracker* s_instance; + static std::mutex s_instance_mutex; + + // ManusSDK specific members + void RegisterCallbacks(); + void ConnectToGloves(); + void DisconnectFromGloves(); + + // Callback functions + static void OnSkeletonStream(const SkeletonStreamInfo* skeleton_stream_info); + static void OnLandscapeStream(const Landscape* landscape); + static void OnErgonomicsStream(const ErgonomicsStream* ergonomics_stream); + + // Data storage (following isaac-deploy pattern) + std::mutex output_map_mutex; + std::mutex landscape_mutex; + std::unordered_map> output_map; + std::optional left_glove_id; + std::optional right_glove_id; + bool is_connected = false; + + // Legacy member for compatibility + std::unordered_map> m_glove_data; + std::mutex m_data_mutex; +}; + +} // namespace input_devices +} // namespace xr +} // namespace isaacsim diff --git a/source/extensions/isaacsim.xr.input_devices/plugins/isaacsim.xr.input_devices/ManusTracker.cpp b/source/extensions/isaacsim.xr.input_devices/plugins/isaacsim.xr.input_devices/ManusTracker.cpp new file mode 100644 index 00000000..02a137e8 --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/plugins/isaacsim.xr.input_devices/ManusTracker.cpp @@ -0,0 +1,301 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include +#include +#include +#include +#include +#include + +#include "ManusSDK.h" +#include "ManusSDKTypeInitializers.h" + +#include + +namespace isaacsim +{ +namespace xr +{ +namespace input_devices +{ + +IsaacSimManusTracker* IsaacSimManusTracker::s_instance = nullptr; +std::mutex IsaacSimManusTracker::s_instance_mutex; + +IsaacSimManusTracker::IsaacSimManusTracker() = default; + +IsaacSimManusTracker::~IsaacSimManusTracker() { + cleanup(); +} + +bool IsaacSimManusTracker::initialize() { + { + std::lock_guard lock(s_instance_mutex); + if (s_instance != nullptr) { + CARB_LOG_ERROR("ManusTracker instance already exists - only one instance allowed"); + return false; + } + s_instance = this; + } + + CARB_LOG_INFO("Initializing Manus SDK..."); + const SDKReturnCode t_InitializeResult = CoreSdk_InitializeIntegrated(); + if (t_InitializeResult != SDKReturnCode::SDKReturnCode_Success) { + CARB_LOG_ERROR("Failed to initialize Manus SDK, error code: %d", static_cast(t_InitializeResult)); + std::lock_guard lock(s_instance_mutex); + s_instance = nullptr; + return false; + } + CARB_LOG_INFO("Manus SDK initialized successfully"); + + RegisterCallbacks(); + + CoordinateSystemVUH t_VUH; + CoordinateSystemVUH_Init(&t_VUH); + t_VUH.handedness = Side::Side_Right; + t_VUH.up = AxisPolarity::AxisPolarity_PositiveZ; + t_VUH.view = AxisView::AxisView_XFromViewer; + t_VUH.unitScale = 1.0f; + + CARB_LOG_INFO("Setting up coordinate system (Z-up, right-handed, meters)..."); + const SDKReturnCode t_CoordinateResult = CoreSdk_InitializeCoordinateSystemWithVUH(t_VUH, true); + + if (t_CoordinateResult != SDKReturnCode::SDKReturnCode_Success) { + CARB_LOG_ERROR("Failed to initialize Manus SDK coordinate system, error code: %d", static_cast(t_CoordinateResult)); + std::lock_guard lock(s_instance_mutex); + s_instance = nullptr; + return false; + } + CARB_LOG_INFO("Coordinate system initialized successfully"); + + ConnectToGloves(); + return true; +} + +std::unordered_map> IsaacSimManusTracker::get_glove_data() { + std::lock_guard lock(output_map_mutex); + return output_map; +} + +void IsaacSimManusTracker::cleanup() { + std::lock_guard lock(s_instance_mutex); + if (s_instance == this) { + CoreSdk_RegisterCallbackForRawSkeletonStream(nullptr); + CoreSdk_RegisterCallbackForLandscapeStream(nullptr); + CoreSdk_RegisterCallbackForErgonomicsStream(nullptr); + DisconnectFromGloves(); + CoreSdk_ShutDown(); + s_instance = nullptr; + } +} + +void IsaacSimManusTracker::RegisterCallbacks() { + CoreSdk_RegisterCallbackForRawSkeletonStream(OnSkeletonStream); + CoreSdk_RegisterCallbackForLandscapeStream(OnLandscapeStream); + CoreSdk_RegisterCallbackForErgonomicsStream(OnErgonomicsStream); +} + +void IsaacSimManusTracker::ConnectToGloves() { + bool connected = false; + const int max_attempts = 30; // Maximum connection attempts + const auto retry_delay = std::chrono::milliseconds(1000); // 1 second delay between attempts + int attempts = 0; + + CARB_LOG_INFO("Looking for Manus gloves..."); + + while (!connected && attempts < max_attempts) { + attempts++; + + if (const auto start_result = CoreSdk_LookForHosts(1, false); + start_result != SDKReturnCode::SDKReturnCode_Success) { + CARB_LOG_ERROR("Failed to look for hosts (attempt %d/%d)", attempts, max_attempts); + std::this_thread::sleep_for(retry_delay); + continue; + } + + uint32_t number_of_hosts_found{}; + if (const auto number_result = CoreSdk_GetNumberOfAvailableHostsFound(&number_of_hosts_found); + number_result != SDKReturnCode::SDKReturnCode_Success) { + CARB_LOG_ERROR("Failed to get number of available hosts (attempt %d/%d)", attempts, max_attempts); + std::this_thread::sleep_for(retry_delay); + continue; + } + + if (number_of_hosts_found == 0) { + CARB_LOG_ERROR("Failed to find hosts (attempt %d/%d)", attempts, max_attempts); + std::this_thread::sleep_for(retry_delay); + continue; + } + + std::vector available_hosts(number_of_hosts_found); + + if (const auto hosts_result = + CoreSdk_GetAvailableHostsFound(available_hosts.data(), number_of_hosts_found); + hosts_result != SDKReturnCode::SDKReturnCode_Success) { + CARB_LOG_ERROR("Failed to get available hosts (attempt %d/%d)", attempts, max_attempts); + std::this_thread::sleep_for(retry_delay); + continue; + } + + if (const auto connect_result = CoreSdk_ConnectToHost(available_hosts[0]); + connect_result == SDKReturnCode::SDKReturnCode_NotConnected) { + CARB_LOG_ERROR("Failed to connect to host (attempt %d/%d)", attempts, max_attempts); + std::this_thread::sleep_for(retry_delay); + continue; + } + + connected = true; + is_connected = true; + CARB_LOG_INFO("Successfully connected to Manus host after %d attempts", attempts); + } + + if (!connected) { + CARB_LOG_ERROR("Failed to connect to Manus gloves after %d attempts", max_attempts); + throw std::runtime_error("Failed to connect to Manus gloves"); + } +} + +void IsaacSimManusTracker::DisconnectFromGloves() { + if (is_connected) { + CoreSdk_Disconnect(); + is_connected = false; + CARB_LOG_INFO("Disconnected from Manus gloves"); + } +} + +void IsaacSimManusTracker::OnSkeletonStream(const SkeletonStreamInfo* skeleton_stream_info) { + CARB_LOG_INFO("OnSkeletonStream callback triggered with %u skeletons", skeleton_stream_info->skeletonsCount); + std::lock_guard instance_lock(s_instance_mutex); + if (!s_instance) { + return; + } + + std::lock_guard output_lock(s_instance->output_map_mutex); + + for (uint32_t i = 0; i < skeleton_stream_info->skeletonsCount; i++) { + RawSkeletonInfo skeleton_info; + CoreSdk_GetRawSkeletonInfo(i, &skeleton_info); + + std::vector nodes(skeleton_info.nodesCount); + skeleton_info.publishTime = skeleton_stream_info->publishTime; + CoreSdk_GetRawSkeletonData(i, nodes.data(), skeleton_info.nodesCount); + + uint32_t glove_id = skeleton_info.gloveId; + + // Check if glove ID matches any known glove + bool is_left_glove, is_right_glove; + { + std::lock_guard landscape_lock(s_instance->landscape_mutex); + is_left_glove = s_instance->left_glove_id && glove_id == *s_instance->left_glove_id; + is_right_glove = s_instance->right_glove_id && glove_id == *s_instance->right_glove_id; + } + + if (!is_left_glove && !is_right_glove) { + CARB_LOG_WARN("Skipping data from unknown glove ID: %u", glove_id); + continue; + } + + std::string prefix = is_left_glove ? "left" : "right"; + + // Store position data (3 floats per node: x, y, z) + std::string pos_key = prefix + "_position"; + s_instance->output_map[pos_key].resize(skeleton_info.nodesCount * 3); + + // Store orientation data (4 floats per node: w, x, y, z) + std::string orient_key = prefix + "_orientation"; + s_instance->output_map[orient_key].resize(skeleton_info.nodesCount * 4); + + for (uint32_t j = 0; j < skeleton_info.nodesCount; j++) { + const auto& position = nodes[j].transform.position; + s_instance->output_map[pos_key][j * 3 + 0] = position.x; + s_instance->output_map[pos_key][j * 3 + 1] = position.y; + s_instance->output_map[pos_key][j * 3 + 2] = position.z; + + const auto& orientation = nodes[j].transform.rotation; + s_instance->output_map[orient_key][j * 4 + 0] = orientation.w; + s_instance->output_map[orient_key][j * 4 + 1] = orientation.x; + s_instance->output_map[orient_key][j * 4 + 2] = orientation.y; + s_instance->output_map[orient_key][j * 4 + 3] = orientation.z; + } + + CARB_LOG_INFO("Updated %s glove data with %u nodes", prefix.c_str(), skeleton_info.nodesCount); + } +} + +void IsaacSimManusTracker::OnLandscapeStream(const Landscape* landscape) { + CARB_LOG_INFO("OnLandscapeStream callback triggered"); + std::lock_guard instance_lock(s_instance_mutex); + if (!s_instance) { + return; + } + + const auto& gloves = landscape->gloveDevices; + CARB_LOG_INFO("Processing %u gloves in landscape", gloves.gloveCount); + + std::lock_guard landscape_lock(s_instance->landscape_mutex); + + // We only support one left and one right glove + if (gloves.gloveCount > 2) { + CARB_LOG_ERROR("Invalid number of gloves detected: %u", gloves.gloveCount); + return; + } + + // Extract glove IDs from landscape data + for (uint32_t i = 0; i < gloves.gloveCount; i++) { + const GloveLandscapeData& glove = gloves.gloves[i]; + if (glove.side == Side::Side_Left) { + s_instance->left_glove_id = glove.id; + CARB_LOG_INFO("Left glove detected with ID: %u", glove.id); + } else if (glove.side == Side::Side_Right) { + s_instance->right_glove_id = glove.id; + CARB_LOG_INFO("Right glove detected with ID: %u", glove.id); + } + } +} + +void IsaacSimManusTracker::OnErgonomicsStream(const ErgonomicsStream* ergonomics_stream) { + std::lock_guard instance_lock(s_instance_mutex); + if (!s_instance) { + return; + } + + std::lock_guard output_lock(s_instance->output_map_mutex); + + for (uint32_t i = 0; i < ergonomics_stream->dataCount; i++) { + if (ergonomics_stream->data[i].isUserID) continue; + + uint32_t glove_id = ergonomics_stream->data[i].id; + + // Check if glove ID matches any known glove + bool is_left_glove, is_right_glove; + { + std::lock_guard landscape_lock(s_instance->landscape_mutex); + is_left_glove = s_instance->left_glove_id && glove_id == *s_instance->left_glove_id; + is_right_glove = s_instance->right_glove_id && glove_id == *s_instance->right_glove_id; + } + + if (!is_left_glove && !is_right_glove) { + CARB_LOG_WARN("Skipping ergonomics data from unknown glove ID: %u", glove_id); + continue; + } + + std::string prefix = is_left_glove ? "left" : "right"; + std::string angle_key = prefix + "_angle"; + s_instance->output_map[angle_key].clear(); + s_instance->output_map[angle_key].reserve(ErgonomicsDataType_MAX_SIZE); + + for (int j = 0; j < ErgonomicsDataType_MAX_SIZE; j++) { + float value = ergonomics_stream->data[i].data[j]; + s_instance->output_map[angle_key].push_back(value); + } + + CARB_LOG_INFO("Updated %s glove ergonomics data", prefix.c_str()); + } +} + +} // namespace input_devices +} // namespace xr +} // namespace isaacsim diff --git a/source/extensions/isaacsim.xr.input_devices/premake5.lua b/source/extensions/isaacsim.xr.input_devices/premake5.lua new file mode 100644 index 00000000..560df14c --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/premake5.lua @@ -0,0 +1,55 @@ +-- SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +local ext = get_current_extension_info() +project_ext(ext) + +-- Python Bindings for Carbonite Plugin +project_ext_bindings { + ext = ext, + project_name = "isaacsim.xr.input_devices.python", + module = "_isaac_xr_input_devices", + src = "bindings", + target_subdir = "isaacsim/xr/input_devices", +} +staticruntime("Off") +add_files("impl", "plugins") +add_files("iface", "include") +defines { "ISAACSIM_XR_INPUT_DEVICES_EXPORT" } + +includedirs { + "%{root}/source/extensions/isaacsim.core.includes/include", + "%{root}/source/extensions/isaacsim.xr.input_devices/include", + "%{root}/source/extensions/isaacsim.xr.input_devices/plugins", + "%{root}/_build/target-deps/manus_sdk/include", + "%{root}/_build/target-deps/libsurvive/include", +} +libdirs { + "%{root}/_build/target-deps/manus_sdk/lib", + "%{root}/_build/target-deps/libsurvive/lib", +} +links {"ManusSDK_Integrated", "survive"} + +repo_build.prebuild_link { + { "python/impl", ext.target_dir .. "/isaacsim/xr/input_devices/impl" }, + { "python/tests", ext.target_dir .. "/isaacsim/xr/input_devices/tests" }, + { "docs", ext.target_dir .. "/docs" }, + { "data", ext.target_dir .. "/data" }, +} + +repo_build.prebuild_copy { + { "python/*.py", ext.target_dir .. "/isaacsim/xr/input_devices" }, + { "%{root}/_build/target-deps/libsurvive/bindings/python/**", ext.target_dir .. "/pysurvive" }, +} diff --git a/source/extensions/isaacsim.xr.input_devices/python/__init__.py b/source/extensions/isaacsim.xr.input_devices/python/__init__.py new file mode 100644 index 00000000..d4e49660 --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/python/__init__.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .impl.extension import Extension + +__all__ = ["Extension"] diff --git a/source/extensions/isaacsim.xr.input_devices/python/impl/__init__.py b/source/extensions/isaacsim.xr.input_devices/python/impl/__init__.py new file mode 100644 index 00000000..cdbf4d2a --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/python/impl/__init__.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .extension import Extension +from .xr_device_integration import XRDeviceIntegration +from .manus_tracker import IsaacSimManusGloveTracker +from .vive_tracker import IsaacSimViveTracker + +__all__ = [ + "Extension", + "XRDeviceIntegration", + "IsaacSimManusGloveTracker", + "IsaacSimViveTracker" +] diff --git a/source/extensions/isaacsim.xr.input_devices/python/impl/extension.py b/source/extensions/isaacsim.xr.input_devices/python/impl/extension.py new file mode 100644 index 00000000..3414dd48 --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/python/impl/extension.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import omni.ext +import carb +from .xr_device_integration import XRDeviceIntegration + +class Extension(omni.ext.IExt): + _instance = None + + def on_startup(self, ext_id): + carb.log_info("IsaacSim XR Input Devices extension startup") + self.xr_integration = XRDeviceIntegration() + self._register_xr_devices() + Extension._instance = self + + def on_shutdown(self): + carb.log_info("IsaacSim XR Input Devices extension shutdown") + if hasattr(self, 'xr_integration'): + self.xr_integration.cleanup() + Extension._instance = None + + def _register_xr_devices(self): + self.xr_integration.register_devices() + + @classmethod + def get_instance(cls): + return cls._instance diff --git a/source/extensions/isaacsim.xr.input_devices/python/impl/manus_tracker.py b/source/extensions/isaacsim.xr.input_devices/python/impl/manus_tracker.py new file mode 100644 index 00000000..8c075fe2 --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/python/impl/manus_tracker.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import carb +from typing import Dict + +try: + from isaacsim.xr.input_devices._isaac_xr_input_devices import IsaacSimManusTracker + _manus_tracker_available = True +except ImportError: + carb.log_warn("IsaacSimManusTracker not available - using mock data for manus") + _manus_tracker_available = False + +class IsaacSimManusGloveTracker: + def __init__(self): + self.glove_data = {} + self.is_connected = False + + if _manus_tracker_available: + try: + self._manus_tracker = IsaacSimManusTracker() + success = self._manus_tracker.initialize() + if success: + self.is_connected = True + carb.log_info("Manus glove tracker initialized with SDK") + else: + carb.log_warn("Failed to initialize Manus SDK - using mock data") + self._manus_tracker = None + except Exception as e: + carb.log_warn(f"Failed to initialize Manus tracker: {e} - using mock data") + self._manus_tracker = None + else: + self._manus_tracker = None + carb.log_info("Manus glove tracker initialized (mock)") + + def update(self): + if self._manus_tracker and self.is_connected: + try: + raw_data = self._manus_tracker.get_glove_data() + for hand in ['left', 'right']: + self._populate_glove_data(raw_data, hand) + + except Exception as e: + carb.log_error(f"Failed to update Manus glove data: {e}") + else: + # Provide mock data + self.glove_data = { + 'left_0': { + 'position': [0.0, 0.0, 0.0], + 'orientation': [1.0, 0.0, 0.0, 0.0] + }, + 'right_0': { + 'position': [0.1, 0.0, 0.0], + 'orientation': [1.0, 0.0, 0.0, 0.0] + } + } + + def _populate_glove_data(self, raw_data: Dict, hand: str) -> None: + """ + Convert raw Manus data to Isaac Sim format with individual joint entries. + Adds to self.glove_data: {joint_name: {position: pos, orientation: ori}} + """ + if f'{hand}_position' not in raw_data or f'{hand}_orientation' not in raw_data: + return + + position = raw_data[f'{hand}_position'] + orientation = raw_data[f'{hand}_orientation'] + joint_count = len(position) // 3 # 3 values per position + + for joint_idx in range(joint_count): + joint_name = f"{hand}_{joint_idx}" + self.glove_data[joint_name] = { + 'position': [float(position[i]) for i in range(joint_idx * 3, joint_idx * 3 + 3)], + 'orientation': [float(orientation[i]) for i in range(joint_idx * 4, joint_idx * 4 + 4)] + } + + def get_data(self) -> Dict: + return self.glove_data + + def cleanup(self): + try: + if self._manus_tracker: + self._manus_tracker.cleanup() + self.is_connected = False + except Exception as e: + carb.log_error(f"Error during Manus tracker cleanup: {e}") diff --git a/source/extensions/isaacsim.xr.input_devices/python/impl/vive_tracker.py b/source/extensions/isaacsim.xr.input_devices/python/impl/vive_tracker.py new file mode 100644 index 00000000..3ea16592 --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/python/impl/vive_tracker.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import sys +import carb +import os +from typing import Dict +from pxr import Gf + +try: + if 'pysurvive' not in sys.modules: + ext_root = os.path.abspath(os.path.join(__file__, "../../../../..")) + vendor_path = os.path.join(ext_root, 'pysurvive') + if os.path.isdir(vendor_path) and vendor_path not in sys.path: + sys.path.insert(0, vendor_path) + carb.log_info(f"Using vendored pysurvive at {vendor_path}") + else: + carb.log_warn(f"Failed to add vendored pysurvive: {vendor_path}") + + import pysurvive + from pysurvive.pysurvive_generated import survive_simple_close + PYSURVIVE_AVAILABLE = True + carb.log_info("pysurvive imported successfully") +except ImportError as e: + carb.log_error(f"pysurvive not available") + PYSURVIVE_AVAILABLE = False + +class IsaacSimViveTracker: + def __init__(self): + self.device_data = {} + self.is_connected = False + + if PYSURVIVE_AVAILABLE: + try: + self._ctx = pysurvive.SimpleContext([sys.argv[0]]) + if self._ctx is None: + raise RuntimeError('Failed to initialize Survive context.') + self.is_connected = True + carb.log_info("Vive tracker initialized with pysurvive") + except Exception as e: + carb.log_warn(f"Failed to initialize Vive tracker: {e}") + else: + self._ctx = None + self.is_connected = True + carb.log_info("Vive tracker initialized (mock)") + + def update(self): + if not PYSURVIVE_AVAILABLE: + raise RuntimeError("pysurvive not available") + return + if not self.is_connected: + return + + try: + max_iterations = 10 # Prevent infinite loops + iteration = 0 + while iteration < max_iterations: + updated = self._ctx.NextUpdated() + if not updated: + break + iteration += 1 + + pose_obj, _ = updated.Pose() + pos = pose_obj.Pos # (x, y, z) + ori = pose_obj.Rot # (w, x, y, z) + device_id = updated.Name().decode('utf-8') + + self.device_data[device_id] = { + 'position': [float(pos[0]), float(-pos[2]), float(pos[1])], # x, z, -y + 'orientation': [float(ori[0]), float(ori[1]), float(-ori[3]), float(ori[2])] + } + + except Exception as e: + carb.log_error(f"Failed to update Vive tracker data: {e}") + + def get_data(self) -> Dict: + return self.device_data + + def cleanup(self): + try: + if PYSURVIVE_AVAILABLE and hasattr(self, '_ctx') and self._ctx is not None: + carb.log_info("Cleaning up Vive tracker context") + survive_simple_close(self._ctx.ptr) + self.is_connected = False + except Exception as e: + carb.log_error(f"Error during Vive tracker cleanup: {e}") diff --git a/source/extensions/isaacsim.xr.input_devices/python/impl/xr_device_integration.py b/source/extensions/isaacsim.xr.input_devices/python/impl/xr_device_integration.py new file mode 100644 index 00000000..3fdf603b --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/python/impl/xr_device_integration.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import carb +from .manus_tracker import IsaacSimManusGloveTracker +from .vive_tracker import IsaacSimViveTracker + + +def get_xr_device_integration(): + """Get the existing XRDeviceIntegration instance from the extension.""" + try: + import omni.ext + from .extension import Extension + ext_instance = Extension.get_instance() + if ext_instance and hasattr(ext_instance, 'xr_integration'): + return ext_instance.xr_integration + else: + carb.log_warn("Extension not loaded, creating new XRDeviceIntegration instance") + return XRDeviceIntegration() + except Exception as e: + carb.log_warn(f"Failed to get extension instance: {e}, creating new XRDeviceIntegration") + return XRDeviceIntegration() + +class XRDeviceIntegration: + def __init__(self): + self.manus_tracker = IsaacSimManusGloveTracker() + self.vive_tracker = IsaacSimViveTracker() + self.device_status = { + 'manus_gloves': {'connected': False, 'last_data_time': 0}, + 'vive_trackers': {'connected': False, 'last_data_time': 0}, + 'left_hand_connected': False, + 'right_hand_connected': False + } + + def register_devices(self): + try: + if self.manus_tracker.is_connected: + carb.log_info("Manus gloves registered successfully") + self.device_status['manus_gloves']['connected'] = True + else: + carb.log_warn("Failed to initialize Manus gloves") + + if self.vive_tracker.is_connected: + carb.log_info("Vive trackers registered successfully") + self.device_status['vive_trackers']['connected'] = True + else: + carb.log_warn("Failed to initialize Vive trackers") + + except Exception as e: + carb.log_error(f"Failed to register XR devices: {e}") + + def cleanup(self): + if hasattr(self, 'manus_tracker'): + self.manus_tracker.cleanup() + if hasattr(self, 'vive_tracker'): + self.vive_tracker.cleanup() diff --git a/source/extensions/isaacsim.xr.input_devices/python/tests/__init__.py b/source/extensions/isaacsim.xr.input_devices/python/tests/__init__.py new file mode 100644 index 00000000..a946756e --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/python/tests/__init__.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Tests for IsaacSim XR Input Devices extension. +""" diff --git a/source/extensions/isaacsim.xr.input_devices/python/tests/test_xr_input_devices.py b/source/extensions/isaacsim.xr.input_devices/python/tests/test_xr_input_devices.py new file mode 100644 index 00000000..18a51034 --- /dev/null +++ b/source/extensions/isaacsim.xr.input_devices/python/tests/test_xr_input_devices.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +""" +Basic tests for isaacsim.xr.input_devices +- Verifies modules import +- Exercises individual trackers (mock-safe) +- Exercises XRDeviceIntegration end-to-end structure +""" + +import os +import sys + +# Add the impl directory to Python path so we can import the modules directly +impl_path = os.path.join(os.path.dirname(__file__), '..', 'impl') +sys.path.insert(0, os.path.abspath(impl_path)) + +def test_extension_import(): + """Test that core modules can be imported.""" + try: + from xr_device_integration import XRDeviceIntegration, get_xr_device_integration # noqa: F401 + from manus_tracker import IsaacSimManusGloveTracker # noqa: F401 + from vive_tracker import IsaacSimViveTracker # noqa: F401 + print("Modules imported successfully") + return True + except Exception as e: + print(f"Import failed: {e}") + return False + +def test_individual_trackers(): + """Test individual tracker classes (mock-friendly).""" + try: + from manus_tracker import IsaacSimManusGloveTracker + from vive_tracker import IsaacSimViveTracker + + manus = IsaacSimManusGloveTracker() + manus.update() + manus_data = manus.get_data() + print(f"Manus tracker: {len(manus_data)} joint entries") + + vive = IsaacSimViveTracker() + vive.update() + vive_data = vive.get_data() + print(f"Vive tracker: {len(vive_data)} device entries") + + return True + except Exception as e: + print(f"Individual tracker test failed: {e}") + return False + +def test_basic_functionality(): + """Test XRDeviceIntegration orchestration and data schema.""" + try: + from xr_device_integration import XRDeviceIntegration + integration = XRDeviceIntegration() + + # Register and perform at least one update on both sources + integration.register_devices() + integration.update_manus() + integration.update_vive() + + all_data = integration.get_all_device_data() + assert isinstance(all_data, dict) + assert 'manus_gloves' in all_data + assert 'vive_trackers' in all_data + assert 'device_status' in all_data + + # Validate structure if entries are present + for group_key in ('manus_gloves', 'vive_trackers'): + for _, pose in all_data[group_key].items(): + pos = pose.get('position') + ori = pose.get('orientation') + assert isinstance(pos, (list, tuple)) and len(pos) == 3 + assert isinstance(ori, (list, tuple)) and len(ori) == 4 + + print("Integration produced structured device data") + integration.cleanup() + return True + except AssertionError as e: + print(f"Data shape assertion failed: {e}") + return False + except Exception as e: + print(f"Integration test failed: {e}") + return False + +if __name__ == "__main__": + print("Testing isaacsim.xr.input_devices") + print("=" * 40) + + tests = [ + ("Module Imports", test_extension_import), + ("Basic Functionality", test_basic_functionality), + ("Individual Trackers", test_individual_trackers), + ] + + passed = 0 + for name, fn in tests: + print(f"\n{name}:") + if fn(): + passed += 1 + + print(f"\nResults: {passed}/{len(tests)} tests passed") + if passed == len(tests): + print("All tests passed") + print("\nFor hardware testing: connect Manus gloves and Vive trackers, then run the sample.") + else: + print("Some tests failed") \ No newline at end of file diff --git a/source/scripts/build_libsurvive.sh b/source/scripts/build_libsurvive.sh new file mode 100755 index 00000000..1b264acf --- /dev/null +++ b/source/scripts/build_libsurvive.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Usage: build_libsurvive.sh +# If REPO_ROOT is not provided, infer it from this script's location. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="${1:-$(cd "$SCRIPT_DIR/../.." && pwd)}" + +TP_DIR="$REPO_ROOT/_build/target-deps/libsurvive" +SRC_DIR="$REPO_ROOT/_build/third_party/libsurvive" +BUILD_DIR="$SRC_DIR/build" + +# Skip if already built (lib + include present) +if [[ -f "$TP_DIR/lib/libsurvive.so" || -f "$TP_DIR/lib/libsurvive.dylib" || -f "$TP_DIR/lib/survive.lib" ]] && \ + [[ -f "$TP_DIR/include/survive.h" ]]; then + echo "[libsurvive] Found existing install under $TP_DIR, skipping build." + exit 0 +fi + +mkdir -p "$TP_DIR" "$SRC_DIR" + +# Clone source if missing +if [[ ! -d "$SRC_DIR/.git" ]]; then + echo "[libsurvive] Cloning libsurvive into $SRC_DIR" + git clone --depth=1 https://github.com/cntools/libsurvive.git "$SRC_DIR" +fi + +# Configure and build with RPATH so Python bindings can locate libsurvive without LD_LIBRARY_PATH +mkdir -p "$BUILD_DIR" +echo "[libsurvive] Configuring CMake (install prefix $TP_DIR)" +cmake -S "$SRC_DIR" -B "$BUILD_DIR" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$TP_DIR" \ + -DCMAKE_INSTALL_RPATH='$$ORIGIN/../../lib' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON + +echo "[libsurvive] Building and installing" +cmake --build "$BUILD_DIR" --target install -j"$(nproc || echo 4)" + +# Copy Python bindings for runtime import (pysurvive) +if [[ -d "$SRC_DIR/bindings/python" ]]; then + mkdir -p "$TP_DIR/bindings/python" + rsync -a --delete "$SRC_DIR/bindings/python/" "$TP_DIR/bindings/python/" + echo "[libsurvive] Copied Python bindings to $TP_DIR/bindings/python" +else + echo "[libsurvive] WARNING: Python bindings directory not found; pysurvive may not import" +fi + +# If compiled extension .so files exist under bindings/python, ensure their RUNPATH points to ../../lib +if command -v patchelf >/dev/null 2>&1; then + shopt -s nullglob + for so in "$TP_DIR"/bindings/python/**/*.so "$TP_DIR"/bindings/python/*.so; do + echo "[libsurvive] Setting RUNPATH on $(basename "$so") to \$ORIGIN/../../lib" + patchelf --set-rpath "\$ORIGIN/../../lib" "$so" || true + done + shopt -u nullglob +else + echo "[libsurvive] patchelf not found; assuming CMake RPATH on extension is sufficient" +fi + +# Print hints +echo "[libsurvive] Install complete at: $TP_DIR" +echo "[libsurvive] Python bindings at: $TP_DIR/bindings/python" diff --git a/source/standalone_examples/api/isaacsim.xr.input_devices/manus_vive_tracking_sample.py b/source/standalone_examples/api/isaacsim.xr.input_devices/manus_vive_tracking_sample.py new file mode 100644 index 00000000..43aaa84a --- /dev/null +++ b/source/standalone_examples/api/isaacsim.xr.input_devices/manus_vive_tracking_sample.py @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Manus Glove and Vive Tracker Visualization Sample + +This sample demonstrates simultaneous tracking of Manus gloves and Vive trackers +in Isaac Sim. The devices are visualized as colored cubes: +- Red cubes (0.01 size): Manus glove joints +- Blue cubes (0.02 size): Vive tracker positions + +DEPRECATED: This sample is deprecated. The coordinate transformation logic has +been moved to IsaacLab. +""" + +import os +import numpy as np +from isaacsim import SimulationApp + +# Initialize simulation app +simulation_app = SimulationApp( + {"headless": False}, experience=f'{os.environ["EXP_PATH"]}/isaacsim.exp.base.xr.openxr.kit' +) + +import carb +import omni.usd +from isaacsim.core.api import World +from isaacsim.core.api.objects import VisualCuboid +from isaacsim.core.utils.prims import create_prim, set_prim_visibility +from omni.isaac.core.prims import XFormPrim +from pxr import Gf, Sdf, UsdGeom, UsdLux + +# Import our XR device integration +try: + from isaacsim.xr.input_devices.impl.xr_device_integration import get_xr_device_integration + carb.log_info("Successfully imported XR device integration helper") +except ImportError as e: + carb.log_error(f"Failed to import XR device integration: {e}") + simulation_app.close() + exit(1) + +# Create world and lighting +my_world = World(stage_units_in_meters=1.0) + +# Add Light Source +stage = omni.usd.get_context().get_stage() +distantLight = UsdLux.DistantLight.Define(stage, Sdf.Path("/DistantLight")) +distantLight.CreateIntensityAttr(300) + +# Create cube prototypes +hidden_prim = create_prim("/Hidden/Prototypes", "Scope") + +# Red cube for Manus gloves (small) +manus_cube_path = "/Hidden/Prototypes/ManusCube" +VisualCuboid( + prim_path=manus_cube_path, + size=0.01, + color=np.array([255, 0, 0]), # Red color +) + +# Blue cube for Vive trackers (larger) +vive_cube_path = "/Hidden/Prototypes/ViveCube" +VisualCuboid( + prim_path=vive_cube_path, + size=0.02, + color=np.array([0, 0, 255]), # Blue color +) + +set_prim_visibility(hidden_prim, False) + +# Create point instancer for cubes +instancer_path = "/World/DeviceCubeInstancer" +point_instancer = UsdGeom.PointInstancer.Define(my_world.stage, instancer_path) +point_instancer.CreatePrototypesRel().SetTargets([Sdf.Path(manus_cube_path), Sdf.Path(vive_cube_path)]) + +max_devices = 60 + +# Initially hide all cubes until devices are tracked (index 2 = hidden, since we have 2 prototypes: 0=manus, 1=vive) +point_instancer.CreateProtoIndicesAttr().Set([2 for _ in range(max_devices)]) + +# Initialize positions and orientations +positions = [Gf.Vec3f(0.0, 0.0, 0.0) for i in range(max_devices)] +point_instancer.CreatePositionsAttr().Set(positions) + +orientations = [Gf.Quath(1.0, 0.0, 0.0, 0.0) for _ in range(max_devices)] +point_instancer.CreateOrientationsAttr().Set(orientations) + +# Add instancer to world scene +instancer_prim = XFormPrim(prim_path=instancer_path) +my_world.scene.add(instancer_prim) + +# Get XR device integration from extension (to avoid singleton conflicts) +xr_integration = get_xr_device_integration() +carb.log_info("Using XR device integration from extension") + +my_world.reset() +reset_needed = False +frame_count = 0 + +# Get attribute references for faster access +positions_attr = point_instancer.GetPositionsAttr() +orientations_attr = point_instancer.GetOrientationsAttr() +proto_idx_attr = point_instancer.GetProtoIndicesAttr() + +carb.log_info("Starting Manus Glove and Vive Tracker visualization with sequential updates") +carb.log_info("Red cubes (0.01) for Manus gloves, Blue cubes (0.02) for Vive trackers") + +# Main simulation loop +while simulation_app.is_running(): + my_world.step(render=True) + if my_world.is_stopped() and not reset_needed: + reset_needed = True + if my_world.is_playing(): + if reset_needed: + my_world.reset() + reset_needed = False + + # Get current cube arrays + current_positions = positions_attr.Get() + current_orientations = orientations_attr.Get() + proto_indices = proto_idx_attr.Get() + + # Initialize all cubes as hidden (index 2 = hidden, since we have 2 prototypes: 0=manus, 1=vive) + proto_indices = [2 for _ in range(max_devices)] + proto_idx_attr.Set(proto_indices) + cube_idx = 0 + + if frame_count % 2 == 0: + xr_integration.manus_tracker.update() + else: + xr_integration.vive_tracker.update() + frame_count += 1 + manus_data = xr_integration.manus_tracker.get_data() + vive_data = xr_integration.vive_tracker.get_data() + if frame_count % 100 == 0: + carb.log_warn(f"Manus data: {manus_data}") + carb.log_warn(f"Vive data: {vive_data}") + + # Process Manus gloves (red cubes, index 0) + for joint, joint_data in manus_data.items(): + if cube_idx >= max_devices: + break + pos, ori = joint_data['position'], joint_data['orientation'] + current_positions[cube_idx] = Gf.Vec3f(float(pos[0]), float(pos[1]), float(pos[2])) + current_orientations[cube_idx] = Gf.Quath(float(ori[0]), float(ori[1]), float(ori[2]), float(ori[3])) + proto_indices[cube_idx] = 0 # Red Manus cube + cube_idx += 1 + + # Process Vive trackers (blue cubes, index 1) + for joint, joint_data in vive_data.items(): + if cube_idx >= max_devices: + break + pos, ori = joint_data['position'], joint_data['orientation'] + current_positions[cube_idx] = Gf.Vec3f(float(pos[0]), float(pos[1]), float(pos[2])) + current_orientations[cube_idx] = Gf.Quath(float(ori[0]), float(ori[1]), float(ori[2]), float(ori[3])) + proto_indices[cube_idx] = 1 # Blue Vive cube + cube_idx += 1 + + # Update the instancer with new positions and orientations + positions_attr.Set(current_positions) + orientations_attr.Set(current_orientations) + proto_idx_attr.Set(proto_indices) + +# Cleanup +xr_integration.cleanup() +simulation_app.close()