Isaac ROS 0.9.2 (EA2.1)

release-ea2.1: aa1178dd2dccb9ce28807f750af87bdfc344856b

Author: hguillen

LICENSE

@@ -40,7 +40,7 @@ install(TARGETS dnn_image_encoder_node
 if(BUILD_TESTING)
   find_package(ament_lint_auto REQUIRED)
 
-  # Ignore copyright notices since we use custom JetPack EULA
+  # Ignore copyright notices since we use custom NVIDIA Isaac ROS Software License
   set(ament_cmake_copyright_FOUND TRUE)
 
   ament_lint_auto_find_test_dependencies()

README.md

@@ -13,6 +13,7 @@
 
 #include <memory>
 #include <string>
+#include <vector>
 
 #include "image_transport/image_transport.hpp"
 #include "isaac_ros_nvengine_interfaces/msg/tensor_list.hpp"
@@ -44,6 +45,10 @@ class DnnImageEncoderNode : public rclcpp::Node
   const int network_image_width_;
   const int network_image_height_;
   const std::string network_image_encoding_;
+  const bool maintain_aspect_ratio_;
+  const bool center_crop_;
+  const std::vector<double> image_mean_;
+  const std::vector<double> image_stddev_;
 
   // Name of the published Tensor message
   const std::string tensor_name_;

giistr-cla.md

@@ -1,12 +1,23 @@
 <?xml version="1.0"?>
+
+<!--
+Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+
+NVIDIA CORPORATION and its licensors retain all intellectual property
+and proprietary rights in and to this software, related documentation
+and any modifications thereto.  Any use, reproduction, disclosure or
+distribution of this software and related documentation without an express
+license agreement from NVIDIA CORPORATION is strictly prohibited.
+-->
+
 <?xml-model href="http://download.ros.org/schema/package_format3.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
 <package format="3">
   <name>isaac_ros_dnn_encoders</name>
   <version>0.9.0</version>
   <description>Encoders for preprocessing before running deep learning inference</description>
   <maintainer email="[email protected]">Hemal Shah</maintainer>
-  <license>Jetpack EULA</license>
-  <url type="website">https://developer.nvidia.com/blog/accelerating-ai-modules-for-ros-and-ros-2-on-jetson/</url>
+  <license>NVIDIA Isaac ROS Software License</license>
+  <url type="website">https://developer.nvidia.com/isaac-ros-gems/</url>
   <author>Ethan Yu</author>
   <author>Kajanan Chinniah</author>
 
@@ -22,4 +33,4 @@
   <export>
     <build_type>ament_cmake</build_type>
   </export>
-</package>
+</package>

isaac_ros_dnn_encoders/CMakeLists.txt

@@ -13,6 +13,7 @@
 #include <memory>
 #include <string>
 #include <unordered_map>
+#include <vector>
 
 #include "cv_bridge/cv_bridge.h"
 #include "opencv2/dnn.hpp"
@@ -24,13 +25,16 @@ enum NormalizationTypes
 {
   kNone,
   kUnitScaling,
-  kPositiveNegative
+  kPositiveNegative,
+  kImageNormalization
 };
 
 const std::unordered_map<std::string, int32_t> g_str_to_normalization_type({
     {"none", NormalizationTypes::kNone},
     {"unit_scaling", NormalizationTypes::kUnitScaling},
-    {"positive_negative", NormalizationTypes::kPositiveNegative}});
+    {"positive_negative", NormalizationTypes::kPositiveNegative},
+    {"image_normalization", NormalizationTypes::kImageNormalization}}
+);
 
 const std::unordered_map<std::string, std::string> g_str_to_image_encoding({
     {"rgb8", sensor_msgs::image_encodings::RGB8},
@@ -53,6 +57,10 @@ struct DnnImageEncoderNode::DnnImageEncoderImpl
   std::string image_encoding_;
   std::string normalization_type_;
   std::string tensor_name_;
+  bool maintain_aspect_ratio_;
+  bool center_crop_;
+  std::vector<double> image_mean_;
+  std::vector<double> image_stddev_;
 
   void Initialize(DnnImageEncoderNode * encoder_node)
   {
@@ -62,6 +70,10 @@ struct DnnImageEncoderNode::DnnImageEncoderImpl
     image_encoding_ = node->network_image_encoding_;
     normalization_type_ = node->network_normalization_type_;
     tensor_name_ = node->tensor_name_;
+    maintain_aspect_ratio_ = node->maintain_aspect_ratio_;
+    center_crop_ = node->center_crop_;
+    image_mean_ = node->image_mean_;
+    image_stddev_ = node->image_stddev_;
   }
 
   isaac_ros_nvengine_interfaces::msg::TensorList OnCallback(
@@ -73,6 +85,39 @@ struct DnnImageEncoderNode::DnnImageEncoderImpl
 
     // Resize the image to the user specified dimensions
     cv::Mat image_resized;
+
+    if (maintain_aspect_ratio_) {
+      const double width_ratio = static_cast<double>(image_msg->width) /
+        static_cast<double>(image_width_);
+      const double height_ratio = static_cast<double>(image_msg->height) /
+        static_cast<double>(image_height_);
+      cv::Size size;
+      if (height_ratio < width_ratio) {  // Cropping width
+        const double target_ratio = static_cast<double>(image_width_) /
+          static_cast<double>(image_height_);
+        const double crop_height = image_msg->height;
+        // Make sure the amount cropped is less than or equal to the current width of image
+        const bool cropped_less = target_ratio * image_msg->height < image_msg->width;
+        const double crop_width =
+          (cropped_less) ? target_ratio * image_msg->height : image_msg->width;
+        cv::Rect cropped_area(
+          (center_crop_) ? (static_cast<double>(image_msg->width) - crop_width) / 2.0 : 0,
+          0, crop_width, crop_height);
+        image_ptr->image = image_ptr->image(cropped_area);
+      } else {  // Cropping height
+        const double target_ratio = static_cast<double>(image_height_) /
+          static_cast<double>(image_width_);
+        const double crop_width = image_msg->width;
+        // Make sure the amount cropped is less than or equal to the current height of image
+        const bool cropped_less = target_ratio * image_msg->width < image_msg->height;
+        const double crop_height =
+          (cropped_less) ? target_ratio * image_msg->width : image_msg->height;
+        cv::Rect cropped_area(0,
+          (center_crop_) ? (static_cast<double>(image_msg->height) - crop_height) / 2.0 : 0,
+          crop_width, crop_height);
+        image_ptr->image = image_ptr->image(cropped_area);
+      }
+    }
     cv::resize(image_ptr->image, image_resized, cv::Size(image_width_, image_height_));
 
     // Normalize tensor depending on normalization type required
@@ -83,6 +128,16 @@ struct DnnImageEncoderNode::DnnImageEncoderImpl
       case NormalizationTypes::kPositiveNegative:
         image_resized.convertTo(image_resized, CV_32F, 2.0f / 255.0f, -1.0f);
         break;
+      case NormalizationTypes::kImageNormalization:
+        image_resized.convertTo(image_resized, CV_32F);
+        image_resized.forEach<cv::Vec3f>(
+          [this](cv::Vec3f & pixel, const int *) -> void
+          {
+            pixel[0] = (pixel[0] / 255.0f - image_mean_[0]) / image_stddev_[0];
+            pixel[1] = (pixel[1] / 255.0f - image_mean_[1]) / image_stddev_[1];
+            pixel[2] = (pixel[2] / 255.0f - image_mean_[2]) / image_stddev_[2];
+          });
+        break;
       default:
         image_resized.convertTo(image_resized, CV_32F);
     }
@@ -126,6 +181,10 @@ DnnImageEncoderNode::DnnImageEncoderNode(const rclcpp::NodeOptions options)
   network_image_width_(declare_parameter<int>("network_image_width", 224)),
   network_image_height_(declare_parameter<int>("network_image_height", 224)),
   network_image_encoding_(declare_parameter<std::string>("network_image_encoding", "rgb8")),
+  maintain_aspect_ratio_(declare_parameter<bool>("maintain_aspect_ratio", false)),
+  center_crop_(declare_parameter<bool>("center_crop", false)),
+  image_mean_(declare_parameter<std::vector<double>>("image_mean", {0.5, 0.5, 0.5})),
+  image_stddev_(declare_parameter<std::vector<double>>("image_stddev", {0.5, 0.5, 0.5})),
   tensor_name_(declare_parameter<std::string>("tensor_name", "input")),
   network_normalization_type_(declare_parameter<std::string>(
       "network_normalization_type", "unit_scaling")),
@@ -155,6 +214,13 @@ DnnImageEncoderNode::DnnImageEncoderNode(const rclcpp::NodeOptions options)
             network_normalization_type_);
   }
 
+  if (network_normalization_type_ == "image_normalization" &&
+    (image_mean_.size() != 3 || image_stddev_.size() != 3))
+  {
+    throw std::runtime_error(
+            "Error: if normalization type is set to Image Normalization, vectors image_mean "
+            "and image_stddev must have exactly 3 elements");
+  }
   impl_->Initialize(this);
 }
 

isaac_ros_dnn_encoders/include/isaac_ros_dnn_encoders/dnn_image_encoder_node.hpp

@@ -0,0 +1,130 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# NVIDIA CORPORATION and its licensors retain all intellectual property
+# and proprietary rights in and to this software, related documentation
+# and any modifications thereto.  Any use, reproduction, disclosure or
+# distribution of this software and related documentation without an express
+# license agreement from NVIDIA CORPORATION is strictly prohibited.
+
+from math import floor
+import os
+import pathlib
+import struct
+import time
+
+from cv_bridge import CvBridge
+from isaac_ros_nvengine_interfaces.msg import TensorList
+from isaac_ros_test import IsaacROSBaseTest
+from launch_ros.actions import ComposableNodeContainer
+from launch_ros.descriptions import ComposableNode
+import numpy as np
+
+import pytest
+import rclpy
+
+from sensor_msgs.msg import Image
+
+
+DIMENSION_WIDTH = 500
+DIMENSION_HEIGHT = 500
+
+
+@pytest.mark.rostest
+def generate_test_description():
+    encoder_node = ComposableNode(
+        name='encoder',
+        package='isaac_ros_dnn_encoders',
+        plugin='isaac_ros::dnn_inference::DnnImageEncoderNode',
+        namespace=IsaacROSDnnImageEncoderImageNormNodeTest.generate_namespace(),
+        parameters=[{
+            'network_image_width': DIMENSION_WIDTH,
+            'network_image_height': DIMENSION_HEIGHT,
+            'network_image_encoding': 'rgb8',
+            'network_normalization_type': 'image_normalization',
+            'image_mean': [0.5, 0.6, 0.25],
+            'image_stddev': [0.25, 0.8, 0.5]
+        }],
+        remappings=[('encoded_tensor', 'tensors')])
+
+    return IsaacROSDnnImageEncoderImageNormNodeTest.generate_test_description([
+        ComposableNodeContainer(
+            name='tensor_rt_container',
+            package='rclcpp_components',
+            executable='component_container',
+            composable_node_descriptions=[encoder_node],
+            namespace=IsaacROSDnnImageEncoderImageNormNodeTest.generate_namespace(),
+            output='screen'
+        )
+    ])
+
+
+class IsaacROSDnnImageEncoderImageNormNodeTest(IsaacROSBaseTest):
+    filepath = pathlib.Path(os.path.dirname(__file__))
+
+    def test_image_normalization(self):
+        """
+        Test Image Normalization feature.
+
+        Test that the DNN Image encoder is correctly normalizing the image based on
+        the given image mean and standard deviation vectors.
+        Given that the image mean vector is <0.5, 0.6, 0.25>, and the image standard
+        deviation vector is <0.25, 0.8, 0.5>, and that our input image is white
+        (each pixel value is 255), the value for each channel should be:
+        RED: ((255 / 255) - 0.5) / 0.25 = 2.0
+        GREEN: ((255 / 255) - 0.6) / 0.8 = 0.5
+        BLUE: ((255/ 255) - 0.25) / 0.5 = 1.5
+        This test verifies that each channel's values should be the calculated values
+        above.
+        """
+        TIMEOUT = 300
+        received_messages = {}
+        RED_EXPECTED_VAL = 2.0
+        GREEN_EXPECTED_VAL = 0.5
+        BLUE_EXPECTED_VAL = 1.5
+
+        self.generate_namespace_lookup(['image', 'tensors'])
+
+        image_pub = self.node.create_publisher(
+            Image, self.namespaces['image'], self.DEFAULT_QOS)
+
+        subs = self.create_logging_subscribers(
+            [('tensors', TensorList)], received_messages)
+
+        try:
+            # Create white image
+            cv_image = np.zeros((500, 500, 3), np.uint8)
+            cv_image[:] = (255, 255, 255)
+            image = CvBridge().cv2_to_imgmsg(cv_image)
+            image.encoding = 'rgb8'
+
+            end_time = time.time() + TIMEOUT
+            done = False
+
+            while time.time() < end_time:
+                image_pub.publish(image)
+                rclpy.spin_once(self.node, timeout_sec=(0.1))
+                if 'tensors' in received_messages:
+                    done = True
+                    break
+            self.assertTrue(done, 'Appropriate output not received')
+            tensor = received_messages['tensors'].tensors[0]
+
+            # This tensor has the format NCHW, so the stride for channel is
+            # calculated by VALUES_PER_CHANNEL. `tensor.data` is also storing
+            # raw bytes - the tensor values are floats, which are 4 bytes.
+            VALUES_PER_CHANNEL = DIMENSION_HEIGHT * DIMENSION_WIDTH
+            SIZEOF_FLOAT = 4
+            for i in range(0, floor(len(tensor.data) / SIZEOF_FLOAT)):
+                # struct.unpack returns a tuple with one element
+                result_val = struct.unpack(
+                    '<f', tensor.data[SIZEOF_FLOAT * i: SIZEOF_FLOAT * i + SIZEOF_FLOAT])[0]
+                if i // VALUES_PER_CHANNEL == 0:  # Red
+                    self.assertTrue(result_val == RED_EXPECTED_VAL)
+                elif i // VALUES_PER_CHANNEL == 1:  # Green
+                    self.assertTrue(result_val == GREEN_EXPECTED_VAL)
+                else:  # Blue
+                    self.assertTrue(result_val == BLUE_EXPECTED_VAL)
+
+        finally:
+            self.node.destroy_subscription(subs)
+            self.node.destroy_publisher(image_pub)