Skip to content

Commit 9f7ce28

Browse files
jaiveersinghNVjaiveersinghNV
committed
Isaac ROS 3.0.0
1 parent d7fd0b3 commit 9f7ce28

36 files changed

+2725
-203
lines changed

‎.gitattributes‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Ignore Python files in linguist
2+
*.py linguist-detectable=false
3+
4+
# Images
5+
*.gif filter=lfs diff=lfs merge=lfs -text
6+
*.jpg filter=lfs diff=lfs merge=lfs -text
7+
*.png filter=lfs diff=lfs merge=lfs -text
8+
*.psd filter=lfs diff=lfs merge=lfs -text
9+
10+
# Archives
11+
*.gz filter=lfs diff=lfs merge=lfs -text
12+
*.tar filter=lfs diff=lfs merge=lfs -text
13+
*.zip filter=lfs diff=lfs merge=lfs -text
14+
15+
# Documents
16+
*.pdf filter=lfs diff=lfs merge=lfs -text
17+
18+
# Shared libraries
19+
*.so filter=lfs diff=lfs merge=lfs -text
20+
*.so.* filter=lfs diff=lfs merge=lfs -text
21+
22+
# ROS Bags
23+
**/resources/**/*.zstd filter=lfs diff=lfs merge=lfs -text
24+
**/resources/**/*.db3 filter=lfs diff=lfs merge=lfs -text
25+
**/resources/**/*.yaml filter=lfs diff=lfs merge=lfs -text
26+
**/resources/**/*.bag filter=lfs diff=lfs merge=lfs -text
27+
28+
# FIXME: Only for DNN packages
29+
# DNN Model files
30+
*.onnx filter=lfs diff=lfs merge=lfs -text

‎.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Ignore all pycache files
2+
**/__pycache__/**
3+
4+
# FIXME: Only for DNN-based packages
5+
# Ignore TensorRT plan files
6+
*.plan
7+
*.engine

‎CONTRIBUTING.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Isaac ROS Contribution Rules
2+
3+
Any contribution that you make to this repository will
4+
be under the Apache 2 License, as dictated by that
5+
[license](http://www.apache.org/licenses/LICENSE-2.0.html):
6+
7+
> **5. Submission of Contributions.** Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
8+
9+
Contributors must sign-off each commit by adding a `Signed-off-by: ...`
10+
line to commit messages to certify that they have the right to submit
11+
the code they are contributing to the project according to the
12+
[Developer Certificate of Origin (DCO)](https://developercertificate.org/).
13+
14+
[//]: # (202201002)

‎LICENSE‎

Lines changed: 66 additions & 201 deletions
Large diffs are not rendered by default.

‎README.md‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,14 @@
1-
# isaac_perceptor
2-
Perception workflows
1+
# Isaac Perceptor
2+
3+
This repository contains the integrated Isaac Perceptor launch files.
4+
5+
## Setup and Documentation
6+
7+
Visit [Isaac Perceptor](https://nvidia-isaac-ros.github.io/reference_workflows/isaac_perceptor/index.html) to learn how to use this repository.
8+
9+
## Performance
10+
11+
| Sample Graph<br/><br/> | Input Size<br/><br/> | Nova Carter<br/><br/> |
12+
|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
13+
| [DNN Stereo Disparity Live Graph](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/benchmarks/isaac_ros_ess_nova_benchmark/scripts/isaac_ros_hawk_1f_2lt_ess_depth_graph.py)<br/><br/><br/>3 Hawk Cameras<br/><br/><br/>1x Full ESS and 2x Throttled Light ESS<br/><br/> | 1200p<br/><br/><br/><br/><br/><br/> | Full: [30.2 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_hawk_1f_2lt_ess_depth_graph-carter_v2.4.json)<br/><br/><br/>Light: 15.2 fps (avg)<br/><br/><br/><br/> |
14+
| [Perceptor Graph](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/benchmarks/isaac_ros_perceptor_nova_benchmark/scripts/isaac_ros_perceptor_graph.py)<br/><br/><br/>3 Hawk Cameras<br/><br/><br/><br/> | 1200p<br/><br/><br/><br/><br/><br/> | Nvblox ESDF: [9.46 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_perceptor_graph-carter_v2.4.json)<br/><br/><br/>Nvblox Mesh: 9.77 fps<br/><br/><br/>Visual Odometry: 30.1 fps<br/><br/> |

‎isaac_ros_perceptor_bringup/CMakeLists.txt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES
2+
# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
# SPDX-License-Identifier: Apache-2.0
17+
18+
cmake_minimum_required(VERSION 3.22.1)
19+
project(isaac_ros_perceptor_bringup)
20+
21+
find_package(ament_cmake REQUIRED)
22+
find_package(ament_cmake_python REQUIRED)
23+
find_package(rclcpp REQUIRED)
24+
find_package(rclpy REQUIRED)
25+
find_package(ament_cmake_auto REQUIRED)
26+
ament_auto_find_build_dependencies()
27+
28+
ament_auto_package(INSTALL_TO_SHARE launch params)

‎isaac_ros_perceptor_bringup/launch/algorithms/hawks_processing.launch.py‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES
2+
# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
# SPDX-License-Identifier: Apache-2.0
17+
18+
import os
19+
20+
from isaac_ros_launch_utils.all_types import *
21+
import isaac_ros_launch_utils as lu
22+
import isaac_ros_perceptor_constants as pc
23+
24+
25+
def create_imager_pipeline(stereo_camera_name: str, identifier: str,
26+
args: lu.ArgumentContainer) -> list[Action]:
27+
stereo_camera_config = getattr(args, stereo_camera_name)
28+
29+
actions: list[Action] = []
30+
31+
rectify_composable_node = ComposableNode(
32+
name='rectify_node',
33+
package='isaac_ros_image_proc',
34+
plugin='nvidia::isaac_ros::image_proc::RectifyNode',
35+
namespace=f'{stereo_camera_name}/{identifier}',
36+
parameters=[{
37+
'output_width': pc.HAWK_IMAGE_WIDTH,
38+
'output_height': pc.HAWK_IMAGE_HEIGHT,
39+
}],
40+
)
41+
rectify_condition = IfCondition(lu.has_substring(stereo_camera_config, 'rectify'))
42+
actions.append(
43+
lu.load_composable_nodes(
44+
args.container_name,
45+
[rectify_composable_node],
46+
condition=rectify_condition,
47+
))
48+
49+
return actions
50+
51+
52+
def create_hawk_pipeline(stereo_camera_name: str, args: lu.ArgumentContainer) -> list[Action]:
53+
actions = []
54+
55+
# Load the configuration for the stereo camera
56+
stereo_camera_config = getattr(args, stereo_camera_name)
57+
run_ess_light = lu.has_substring(stereo_camera_config, 'ess_light')
58+
run_ess_full = lu.has_substring(stereo_camera_config, 'ess_full')
59+
run_ess = OrSubstitution(run_ess_light, run_ess_full)
60+
61+
actions.append(
62+
lu.assert_condition(
63+
'Camera config invalid. Can not run ess_light and ess_full at the same time.',
64+
IfCondition(AndSubstitution(run_ess_light, run_ess_full))),
65+
)
66+
engine_file_path = lu.if_else_substitution(run_ess_light, args.ess_light_engine_file_path,
67+
args.ess_full_engine_file_path)
68+
actions.append(lu.assert_path_exists(
69+
engine_file_path,
70+
condition=IfCondition(run_ess),
71+
))
72+
ess_skip_frames = lu.has_substring(stereo_camera_config, 'ess_skip_frames')
73+
throttler_skip = lu.if_else_substitution(ess_skip_frames, args.ess_number_of_frames_to_skip,
74+
'0')
75+
76+
# Run the left/right imager pipelines (rectify)
77+
actions.extend(create_imager_pipeline(stereo_camera_name, 'left', args))
78+
actions.extend(create_imager_pipeline(stereo_camera_name, 'right', args))
79+
80+
# Run the depth estimation
81+
ess_composable_node = ComposableNode(
82+
name='ess_node',
83+
package='isaac_ros_ess',
84+
plugin='nvidia::isaac_ros::dnn_stereo_depth::ESSDisparityNode',
85+
namespace=stereo_camera_name,
86+
parameters=[{
87+
'engine_file_path': engine_file_path,
88+
'threshold': 0.4,
89+
'throttler_skip': throttler_skip,
90+
}],
91+
remappings=[
92+
('left/camera_info', 'left/camera_info_rect'),
93+
('left/image_raw', 'left/image_rect'),
94+
('right/camera_info', 'right/camera_info_rect'),
95+
('right/image_raw', 'right/image_rect'),
96+
],
97+
)
98+
disparity_composable_node = ComposableNode(
99+
name='disparity_to_depth',
100+
package='isaac_ros_stereo_image_proc',
101+
plugin='nvidia::isaac_ros::stereo_image_proc::DisparityToDepthNode',
102+
namespace=stereo_camera_name,
103+
)
104+
depth_action = lu.load_composable_nodes(
105+
args.container_name, [ess_composable_node, disparity_composable_node],
106+
condition=IfCondition(run_ess))
107+
actions.append(depth_action)
108+
109+
return actions
110+
111+
112+
def generate_launch_description() -> LaunchDescription:
113+
args = lu.ArgumentContainer()
114+
115+
# Config strings for all stereo cameras.
116+
# The config string must be a subset of:
117+
# - driver,rectify,resize,ess_full,ess_light,ess_skip_frames,cuvslam,nvblox
118+
args.add_arg('front_stereo_camera')
119+
args.add_arg('back_stereo_camera')
120+
args.add_arg('left_stereo_camera')
121+
args.add_arg('right_stereo_camera')
122+
123+
# Additional arguments
124+
args.add_arg('container_name', 'nova_container')
125+
args.add_arg('ess_number_of_frames_to_skip', 1)
126+
args.add_arg('ess_full_engine_file_path',
127+
lu.get_isaac_ros_ws_path() +
128+
'/isaac_ros_assets/models/dnn_stereo_disparity/dnn_stereo_disparity_v4.0.0/ess.engine')
129+
args.add_arg('ess_light_engine_file_path',
130+
lu.get_isaac_ros_ws_path() +
131+
'/isaac_ros_assets/models/dnn_stereo_disparity/dnn_stereo_disparity_v4.0.0/light_ess.engine')
132+
133+
# Create pipelines for each camera according to the camera config
134+
actions = args.get_launch_actions()
135+
actions.extend(create_hawk_pipeline('front_stereo_camera', args))
136+
actions.extend(create_hawk_pipeline('back_stereo_camera', args))
137+
actions.extend(create_hawk_pipeline('left_stereo_camera', args))
138+
actions.extend(create_hawk_pipeline('right_stereo_camera', args))
139+
140+
return LaunchDescription(actions)

‎isaac_ros_perceptor_bringup/launch/algorithms/nvblox.launch.py‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES
2+
# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
# SPDX-License-Identifier: Apache-2.0
17+
18+
from isaac_ros_launch_utils.all_types import *
19+
import isaac_ros_launch_utils as lu
20+
import isaac_ros_perceptor_constants as pc
21+
22+
from nvblox_ros_python_utils.nvblox_launch_utils import NvbloxPeopleSegmentation
23+
import nvblox_ros_python_utils.nvblox_constants as nc
24+
25+
from typing import Any, List, Tuple
26+
27+
28+
def get_nvblox_remappings(camera_names: List[str],
29+
enable_people_segmentation: bool) -> List[Tuple[str, str]]:
30+
remappings = []
31+
for i, name in enumerate(camera_names):
32+
remappings.append((f'camera_{i}/depth/image', f'{name}/depth'))
33+
remappings.append((f'camera_{i}/depth/camera_info', f'{name}/camera_info'))
34+
# NOTE(alexmillane): If in people mapping mode the first camera subscribes to color
35+
# image coming from the segmentation graph.
36+
if enable_people_segmentation and name == 'front_stereo_camera':
37+
remappings.append((f'camera_{i}/color/image', f'/segmentation/image_resized'))
38+
remappings.append((f'camera_{i}/color/camera_info', f'/segmentation/camera_info_resized'))
39+
else:
40+
remappings.append((f'camera_{i}/color/image', f'{name}/left/image_rect'))
41+
remappings.append((f'camera_{i}/color/camera_info', f'{name}/left/camera_info_rect'))
42+
if enable_people_segmentation:
43+
remappings.append((f'mask/image', f'/unet/raw_segmentation_mask'))
44+
remappings.append((f'mask/camera_info', f'/segmentation/camera_info_resized'))
45+
return remappings
46+
47+
48+
def get_nvblox_params(camera_names: List[str], enable_people_segmentation: bool,
49+
global_frame: str) -> List[Any]:
50+
parameters = []
51+
parameters.append(lu.get_path('nvblox_examples_bringup', 'config/nvblox/nvblox_base.yaml'))
52+
parameters.append(lu.get_path('isaac_ros_perceptor_bringup', 'params/nvblox_perceptor.yaml'))
53+
parameters.append({'num_cameras': len(camera_names)})
54+
parameters.append({'global_frame': global_frame})
55+
if enable_people_segmentation:
56+
parameters.append(
57+
lu.get_path('nvblox_examples_bringup',
58+
'config/nvblox/specializations/nvblox_segmentation.yaml'))
59+
return parameters
60+
61+
62+
def check_valid_nvblox_configuration(args: lu.ArgumentContainer, enable_people_segmentation: bool):
63+
if enable_people_segmentation:
64+
# For now, we only allow people segmentation on the front camera.
65+
segmentation_camera_names = args.enabled_stereo_cameras_for_nvblox_people.split(',')
66+
assert len(
67+
segmentation_camera_names) <= 1, 'People segmentation is only possible for one camera.'
68+
assert segmentation_camera_names[0] == 'front_stereo_camera', \
69+
'People segmentation is only possible for the front stereo camera.'
70+
71+
72+
def add_nvblox(args: lu.ArgumentContainer) -> List[Action]:
73+
camera_names = args.enabled_stereo_cameras_for_nvblox.split(',')
74+
enable_people_segmentation = len(args.enabled_stereo_cameras_for_nvblox_people) > 0
75+
76+
remappings = get_nvblox_remappings(camera_names, enable_people_segmentation)
77+
parameters = get_nvblox_params(camera_names, enable_people_segmentation, args.global_frame)
78+
check_valid_nvblox_configuration(args, enable_people_segmentation)
79+
80+
# Add the nvblox node.
81+
if enable_people_segmentation:
82+
nvblox_node_name = 'nvblox_human_node'
83+
nvblox_plugin_name = 'nvblox::NvbloxHumanNode'
84+
else:
85+
nvblox_node_name = 'nvblox_node'
86+
nvblox_plugin_name = 'nvblox::NvbloxNode'
87+
88+
nvblox_node = ComposableNode(
89+
name=nvblox_node_name,
90+
package='nvblox_ros',
91+
plugin=nvblox_plugin_name,
92+
remappings=remappings,
93+
parameters=parameters,
94+
)
95+
96+
actions = []
97+
actions.append(lu.load_composable_nodes(args.container_name, [nvblox_node]))
98+
actions.append(
99+
lu.log_info(["Enabling nvblox for cameras '", args.enabled_stereo_cameras_for_nvblox, "'"]))
100+
101+
# Add people segmentation if enabled.
102+
if enable_people_segmentation:
103+
actions.append(
104+
lu.include(
105+
'nvblox_examples_bringup',
106+
'launch/perception/segmentation.launch.py',
107+
launch_arguments={
108+
'container_name': args.container_name,
109+
'input_topic': '/front_stereo_camera/left/image_rect',
110+
'input_camera_info_topic': '/front_stereo_camera/left/camera_info_rect',
111+
'people_segmentation': NvbloxPeopleSegmentation.peoplesemsegnet_shuffleseg,
112+
}))
113+
114+
return actions
115+
116+
117+
def generate_launch_description() -> LaunchDescription:
118+
args = lu.ArgumentContainer()
119+
args.add_arg('enabled_stereo_cameras_for_nvblox')
120+
args.add_arg('enabled_stereo_cameras_for_nvblox_people')
121+
args.add_arg('container_name', 'nova_container')
122+
args.add_arg('global_frame', 'odom')
123+
args.add_opaque_function(add_nvblox)
124+
return LaunchDescription(args.get_launch_actions())

0 commit comments

Comments
 (0)