Isaac ROS 4.0 (#39)

Author: jaiveersinghNV

README.md

@@ -0,0 +1,24 @@
+## Security
+
+NVIDIA is dedicated to the security and trust of our software products and services, including all source code repositories managed through our organization.
+
+If you need to report a security issue, please use the appropriate contact points outlined below. **Please do not report security vulnerabilities through GitHub.**
+
+## Reporting Potential Security Vulnerability in an NVIDIA Product
+
+To report a potential security vulnerability in any NVIDIA product:
+- Web: [Security Vulnerability Submission Form](https://www.nvidia.com/object/submit-security-vulnerability.html)
+- E-Mail: [email protected]
+    - We encourage you to use the following PGP key for secure email communication: [NVIDIA public PGP Key for communication](https://www.nvidia.com/en-us/security/pgp-key)
+    - Please include the following information:
+      - Product/Driver name and version/branch that contains the vulnerability
+      - Type of vulnerability (code execution, denial of service, buffer overflow, etc.)
+      - Instructions to reproduce the vulnerability
+      - Proof-of-concept or exploit code
+      - Potential impact of the vulnerability, including how an attacker could exploit the vulnerability
+
+While NVIDIA currently does not have a bug bounty program, we do offer acknowledgement when an externally reported security issue is addressed under our coordinated vulnerability disclosure policy. Please visit our [Product Security Incident Response Team (PSIRT)](https://www.nvidia.com/en-us/security/psirt-policies/) policies page for more information.
+
+## NVIDIA Product Security
+
+For all security-related concerns, please visit NVIDIA's Product Security portal at https://www.nvidia.com/en-us/security

SECURITY.md

@@ -45,19 +45,19 @@ gxf::Expected<gxf::Handle<gxf::Tensor>> GetTensor(gxf::Entity entity, const char
 
 gxf::Expected<Shape> GetShape(gxf::Handle<gxf::Tensor> in_tensor) {
   Shape shape{};
-  
+
   shape.batch_size = in_tensor->shape().dimension(0);
   shape.channels = in_tensor->shape().dimension(1);
   shape.height = in_tensor->shape().dimension(2);
   shape.width = in_tensor->shape().dimension(3);
-    
-  // Only single channel 
+
+  // Only single channel
   if (shape.channels != kExpectedChannelCount) {
     GXF_LOG_ERROR("Received %d input channels, which is larger than the maximum allowable %d",
                   shape.channels, kExpectedChannelCount);
     return gxf::Unexpected{GXF_FAILURE};
   }
-  
+
   return shape;
 }
 
@@ -160,7 +160,7 @@ gxf_result_t SegmentAnythingPostprocessor::tick() {
   Shape shape = maybe_shape.value();
 
   auto out_message = gxf::Entity::New(context());
-  
+
   if (!out_message) {
     GXF_LOG_ERROR("Failed to allocate message");
     return gxf::ToResultCode(out_message);
@@ -180,8 +180,7 @@ gxf_result_t SegmentAnythingPostprocessor::tick() {
   cudaStream_t cuda_stream = maybe_cuda_stream.value();
 
   gxf::Expected<void> maybe_kernel_result;
-  
-   
+
   auto raw_tensor = out_message.value().add<gxf::Tensor>();
 
   if (!raw_tensor) {
@@ -202,14 +201,23 @@ gxf_result_t SegmentAnythingPostprocessor::tick() {
 
   maybe_kernel_result =
       GenerateSegmentationMask(shape, in_tensor, output_video_buffer_data, cuda_stream);
-  
+
   auto maybe_added_timestamp = AddInputTimestampToOutput(out_message.value(), in_message.value());
   if (!maybe_added_timestamp) { return gxf::ToResultCode(maybe_added_timestamp); }
 
 
   auto maybe_added_stream = AddStreamToOutput(out_message.value(), stream_);
   if (!maybe_added_stream) { return gxf::ToResultCode(maybe_added_stream); }
 
+  // Sync the stream, before sending it across the framework.
+  // This is important to do since Isaac does not support non synced work across
+  // codelets and multiple nodes.
+  cudaError_t sync_result = cudaStreamSynchronize(cuda_stream);
+  if (sync_result != cudaSuccess) {
+    GXF_LOG_ERROR("Error while synchronizing CUDA stream: %s", cudaGetErrorString(sync_result));
+    return GXF_FAILURE;
+  }
+
   return gxf::ToResultCode(out_->publish(std::move(out_message.value())));
 }
 

isaac_ros_gxf_extensions/gxf_isaac_ros_segment_anything/gxf/segment_anything/segment_anything_postprocessor.cpp

@@ -1,5 +1,5 @@
 // SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES
-// Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -66,18 +66,33 @@ gxf_result_t SegmentAnythingPromptProcessor::registerInterface(gxf::Registrar* r
   result &= registrar->parameter(max_batch_size_, "max_batch_size");
   result &= registrar->parameter(prompt_type_name_, "prompt_type_name");
   result &= registrar->parameter(has_input_mask_, "has_input_mask");
+  result &= registrar->parameter(cuda_stream_pool_, "stream_pool", "Cuda Stream Pool",
+                                 "Instance of gxf::CudaStreamPool to allocate CUDA stream.");
   return gxf::ToResultCode(result);
 }
 
 gxf_result_t SegmentAnythingPromptProcessor::start() {
 
+  // Get cuda stream from stream pool
+  auto maybe_stream = cuda_stream_pool_.get()->allocateStream();
+  if (!maybe_stream) { return gxf::ToResultCode(maybe_stream); }
+
+  cuda_stream_handle_ = std::move(maybe_stream.value());
+  if (!cuda_stream_handle_->stream()) {
+    GXF_LOG_ERROR("Allocated stream is not initialized!");
+    return GXF_FAILURE;
+  }
+  if (!cuda_stream_handle_.is_null()) {
+    cuda_stream_ = cuda_stream_handle_->stream().value();
+  }
+
   uint32_t orig_width = orig_img_dim_.get()[1];
   uint32_t orig_height = orig_img_dim_.get()[0];
 
   if (orig_width > orig_height) {
     resized_width_ = IMAGE_WIDTH_;
     resized_height_ = int((float(resized_width_) / orig_width) * orig_height);
-  } else { 
+  } else {
     resized_height_ = IMAGE_HEIGHT_;
     resized_width_ = int((float(resized_height_) / orig_height) * orig_width);
   }
@@ -96,21 +111,21 @@ gxf_result_t SegmentAnythingPromptProcessor::tick() {
   Detection2DParts parts;
   auto detection2_d_parts_expected = nvidia::isaac_ros::GetDetection2DList(
     in_message.value());
-  
+
   if (!detection2_d_parts_expected) { return gxf::ToResultCode(detection2_d_parts_expected); }
-  
+
   auto detection2_d_parts = detection2_d_parts_expected.value();
 
   // Extract detection2_d array to a struct type defined in detection2_d.hpp
   std::vector<nvidia::isaac_ros::Detection2D> detections =
     *(detection2_d_parts.detection2_d_array);
-  
+
   auto maybe_added_timestamp = AddInputTimestampToOutput(out_message.value(), in_message.value());
   if (!maybe_added_timestamp) {
     GXF_LOG_ERROR("Failed to add timestamp to output msg");
     return gxf::ToResultCode(maybe_added_timestamp);
   }
-  
+
   // Add invalid_frame tensor and publish it.
   // invalid_frame tensor is used by msg compositor to disqualify this input.
   if(detections.size() == 0){
@@ -126,7 +141,7 @@ gxf_result_t SegmentAnythingPromptProcessor::tick() {
 
   uint32_t batch_size = 1;
   uint32_t num_points = num_points_;
-  if (prompt_type_value_ == PromptType::kBbox) { 
+  if (prompt_type_value_ == PromptType::kBbox) {
     batch_size = max_batch_size_ < detections.size() ? max_batch_size_ : detections.size();
   }
 
@@ -139,7 +154,7 @@ gxf_result_t SegmentAnythingPromptProcessor::tick() {
   std::vector<float> label_vec;
 
   detectionToSAMPrompt(detections, prompt_vec, label_vec);
-  
+
   if (!out_message) {
     GXF_LOG_ERROR("Failed to allocate message");
     return gxf::ToResultCode(out_message);
@@ -175,13 +190,15 @@ gxf_result_t SegmentAnythingPromptProcessor::tick() {
   auto result = bbox_tensor.value()->reshapeCustom(
   prompt_tensor_shape, gxf::PrimitiveType::kFloat32,
       gxf::PrimitiveTypeSize(gxf::PrimitiveType::kFloat32),
-  gxf::Unexpected{GXF_UNINITIALIZED_VALUE}, gxf::MemoryStorageType::kDevice, allocator_.get());  
+  gxf::Unexpected{GXF_UNINITIALIZED_VALUE}, gxf::MemoryStorageType::kDevice, allocator_.get());
   if(!result){
     GXF_LOG_ERROR("Error allocating tensor for points tensor.");
     return gxf::ToResultCode(result);
   }
-  cudaMemcpy(bbox_tensor.value()->pointer(), prompt_vec.data(), buffer_size,cudaMemcpyHostToDevice);
-  
+  cudaMemcpyAsync(
+      bbox_tensor.value()->pointer(), prompt_vec.data(), buffer_size,cudaMemcpyHostToDevice,
+      cuda_stream_);
+
   // Create Tensor for labels corresponding to each point
   gxf::Shape label_tensor_shape({batch_size,num_points});
   uint32_t buffer_size_label{batch_size*num_points*sizeof(float)};
@@ -193,7 +210,9 @@ gxf_result_t SegmentAnythingPromptProcessor::tick() {
     GXF_LOG_ERROR("Error allocating tensor for label tensor.");
     return gxf::ToResultCode(label_tensor_result);
   }
-  cudaMemcpy(labels_tensor.value()->pointer(), label_vec.data(), buffer_size_label,cudaMemcpyHostToDevice); 
+  cudaMemcpyAsync(
+      labels_tensor.value()->pointer(), label_vec.data(), buffer_size_label,cudaMemcpyHostToDevice,
+      cuda_stream_);
 
   // Create Tensor to put has_input_mask data
   gxf::Shape has_in_mask({1});
@@ -207,8 +226,10 @@ gxf_result_t SegmentAnythingPromptProcessor::tick() {
     GXF_LOG_ERROR("Error allocating tensor for has input mask tensor.");
     return gxf::ToResultCode(has_input_mask_result);
   }
-  cudaMemcpy(has_input_mask_tensor.value()->pointer(), has_input_mask.data(), sizeof(float), cudaMemcpyHostToDevice);
-  
+  cudaMemcpyAsync(
+      has_input_mask_tensor.value()->pointer(), has_input_mask.data(), sizeof(float),
+      cudaMemcpyHostToDevice, cuda_stream_);
+
   // Create Tensor to put original image shape data
   gxf::Shape img_size_tensor_shape({2});
   std::vector<float> img_size(2);
@@ -217,12 +238,19 @@ gxf_result_t SegmentAnythingPromptProcessor::tick() {
   auto img_size_tensor_result = img_size_tensor.value()->reshapeCustom(
   img_size_tensor_shape, gxf::PrimitiveType::kFloat32,
       gxf::PrimitiveTypeSize(gxf::PrimitiveType::kFloat32),
-  gxf::Unexpected{GXF_UNINITIALIZED_VALUE}, gxf::MemoryStorageType::kDevice, allocator_.get());  
+  gxf::Unexpected{GXF_UNINITIALIZED_VALUE}, gxf::MemoryStorageType::kDevice, allocator_.get());
   if(!img_size_tensor_result){
     GXF_LOG_ERROR("Error allocating tensor for image size tensor.");
     return gxf::ToResultCode(img_size_tensor_result);
   }
-  cudaMemcpy(img_size_tensor.value()->pointer(),img_size.data(), 2*sizeof(float),cudaMemcpyHostToDevice);
+  cudaMemcpyAsync(
+      img_size_tensor.value()->pointer(),img_size.data(), 2*sizeof(float),
+      cudaMemcpyHostToDevice, cuda_stream_);
+
+  if (cudaStreamSynchronize(cuda_stream_) != cudaSuccess) {
+    GXF_LOG_ERROR("Failed to synchronize stream");
+    return GXF_FAILURE;
+  }
 
   out_points_->publish(std::move(out_message.value()));
   return GXF_SUCCESS;

isaac_ros_gxf_extensions/gxf_isaac_ros_segment_anything/gxf/segment_anything/segment_anything_prompt_processor.cpp

@@ -1,5 +1,5 @@
 // SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES
-// Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -27,6 +27,9 @@
 #include "detection2_d_array_message.hpp"
 #include "isaac_ros_nitros_detection2_d_array_type/nitros_detection2_d_array.hpp"
 
+#include "gxf/cuda/cuda_stream.hpp"
+#include "gxf/cuda/cuda_stream_pool.hpp"
+
 namespace nvidia {
 namespace isaac_ros {
 
@@ -47,15 +50,18 @@ class SegmentAnythingPromptProcessor : public gxf::Codelet {
     std::vector<float>& label_vec
     );
  private:
-
   gxf::Parameter<gxf::Handle<gxf::Receiver>> in_;
   gxf::Parameter<gxf::Handle<gxf::Transmitter>> out_points_;
   gxf::Parameter<gxf::Handle<gxf::Allocator>> allocator_;
   gxf::Parameter<std::vector<int32_t>> orig_img_dim_;
   gxf::Parameter<int32_t> max_batch_size_;
   gxf::Parameter<std::string> prompt_type_name_;
   gxf::Parameter<bool> has_input_mask_;
-  
+  gxf::Parameter<gxf::Handle<gxf::CudaStreamPool>> cuda_stream_pool_;
+  // CUDA stream variables
+  gxf::Handle<gxf::CudaStream> cuda_stream_handle_;
+  cudaStream_t cuda_stream_ = 0;
+
   const uint16_t IMAGE_WIDTH_ = 1024;
   const uint16_t IMAGE_HEIGHT_ = 1024;
   uint16_t resized_width_;

isaac_ros_gxf_extensions/gxf_isaac_ros_segment_anything/gxf/segment_anything/segment_anything_prompt_processor.hpp

@@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0
 <?xml-model href="http://download.ros.org/schema/package_format3.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
 <package format="3">
   <name>gxf_isaac_ros_segment_anything</name>
-  <version>3.2.5</version>
+  <version>4.0.0</version>
   <description>Segmentation Anything extension.</description>
 
   <maintainer email="[email protected]">Isaac ROS Maintainers</maintainer>

isaac_ros_gxf_extensions/gxf_isaac_ros_segment_anything/package.xml

@@ -1,5 +1,5 @@
 // SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES
-// Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -114,27 +114,46 @@ gxf_result_t SegmentationMaskColorizer::registerInterface(gxf::Registrar * regis
     color_segmentation_mask_encoding_str_, "color_segmentation_mask_encoding",
     "Color Segmentation Mask Encoding",
     "The encoding of the colored segmentation mask. This should be either rgb8 or bgr8");
+  result &= registrar->parameter(cuda_stream_pool_, "stream_pool", "Cuda Stream Pool",
+                                 "Instance of gxf::CudaStreamPool to allocate CUDA stream.");
   return gxf::ToResultCode(result);
 }
 
 gxf_result_t SegmentationMaskColorizer::start()
 {
+  // Get cuda stream from stream pool
+  auto maybe_stream = cuda_stream_pool_.get()->allocateStream();
+  if (!maybe_stream) { return gxf::ToResultCode(maybe_stream); }
+
+  cuda_stream_handle_ = std::move(maybe_stream.value());
+  if (!cuda_stream_handle_->stream()) {
+    GXF_LOG_ERROR("Allocated stream is not initialized!");
+    return GXF_FAILURE;
+  }
+  if (!cuda_stream_handle_.is_null()) {
+    cuda_stream_ = cuda_stream_handle_->stream().value();
+  }
+
   if (color_palette_vec_.get().empty()) {
     GXF_LOG_ERROR("Error: received empty color palette!");
     return GXF_FAILURE;
   }
 
   int64_t * data{nullptr};
   cudaError_t result =
-    cudaMalloc(&data, sizeof(int64_t) * color_palette_vec_.get().size());
+    cudaMallocAsync(&data, sizeof(int64_t) * color_palette_vec_.get().size(), cuda_stream_);
   if (result != cudaSuccess) {return GXF_FAILURE;}
   color_palette_.data.reset(data);
-
-  result = cudaMemcpy(
+  result = cudaMemcpyAsync(
     color_palette_.data.get(), color_palette_vec_.get().data(),
-    color_palette_vec_.get().size() * sizeof(int64_t), cudaMemcpyHostToDevice);
+    color_palette_vec_.get().size() * sizeof(int64_t), cudaMemcpyHostToDevice, cuda_stream_);
   if (result != cudaSuccess) {return GXF_FAILURE;}
 
+  if (cudaStreamSynchronize(cuda_stream_) != cudaSuccess) {
+    GXF_LOG_ERROR("Failed to synchronize stream");
+    return GXF_FAILURE;
+  }
+
   if (color_segmentation_mask_encoding_str_.get() == std::string{"rgb8"}) {
     color_segmentation_mask_encoding_ = ColorImageEncodings::kRGB8;
   } else if (color_segmentation_mask_encoding_str_.get() == std::string{"bgr8"}) {

isaac_ros_gxf_extensions/gxf_isaac_ros_unet/gxf/image_segmentation/segmentation_mask_colorizer.cpp

@@ -1,5 +1,5 @@
 // SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES
-// Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -27,6 +27,8 @@
 #include "gxf/std/receiver.hpp"
 #include "gxf/std/transmitter.hpp"
 #include "segmentation_mask_colorizer.cu.hpp"
+#include "gxf/cuda/cuda_stream.hpp"
+#include "gxf/cuda/cuda_stream_pool.hpp"
 
 namespace nvidia {
 namespace isaac_ros {
@@ -45,6 +47,10 @@ class SegmentationMaskColorizer : public gxf::Codelet {
   gxf::Parameter<gxf::Handle<gxf::Allocator>> allocator_;
   gxf::Parameter<std::vector<int64_t>> color_palette_vec_;
   gxf::Parameter<std::string> color_segmentation_mask_encoding_str_;
+  gxf::Parameter<gxf::Handle<gxf::CudaStreamPool>> cuda_stream_pool_;
+  // CUDA stream variables
+  gxf::Handle<gxf::CudaStream> cuda_stream_handle_;
+  cudaStream_t cuda_stream_ = 0;
 
   ArrayView<int64_t> color_palette_;
   ColorImageEncodings color_segmentation_mask_encoding_;

isaac_ros_gxf_extensions/gxf_isaac_ros_unet/gxf/image_segmentation/segmentation_mask_colorizer.hpp

@@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0
 <?xml-model href="http://download.ros.org/schema/package_format3.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
 <package format="3">
   <name>gxf_isaac_ros_unet</name>
-  <version>3.2.5</version>
+  <version>4.0.0</version>
   <description>Segmentation post-processor extension.</description>
 
   <maintainer email="[email protected]">Isaac ROS Maintainers</maintainer>

isaac_ros_gxf_extensions/gxf_isaac_ros_unet/package.xml

@@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0
 <?xml-model href="http://download.ros.org/schema/package_format3.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
 <package format="3">
   <name>isaac_ros_peoplesemseg_models_install</name>
-  <version>3.2.5</version>
+  <version>4.0.0</version>
   <description>Scripts for installing people segmentation models</description>
 
   <maintainer email="[email protected]">Isaac ROS Maintainers</maintainer>