fix: missing model code for CV-CUDA Release 0.3.0 Beta

Author: milesp-nvidia

.gitattributes

@@ -19,3 +19,4 @@
 *.onnx filter=lfs diff=lfs merge=lfs -text
 *.bmp filter=lfs diff=lfs merge=lfs -text
 *.mp4 filter=lfs diff=lfs merge=lfs -text
+*.a filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -50,5 +50,4 @@ _exhale_api
 
 # Samples
 # -------------
-models
 *.engine

.pre-commit-config.yaml

@@ -0,0 +1,91 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+
+default_stages:
+  - merge-commit
+  - commit
+  - post-rewrite
+
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.3.0
+  hooks:
+  - id: check-merge-conflict
+  - id: trailing-whitespace
+  - id: end-of-file-fixer
+  - id: mixed-line-ending
+    args: ['--fix=lf']
+  - id: check-executables-have-shebangs
+  - id: check-shebang-scripts-are-executable
+  - id: detect-private-key
+  - id: check-added-large-files
+  - id: check-case-conflict
+  - id: destroyed-symlinks
+  - id: check-yaml
+    exclude: '.clang-format'
+
+- repo: https://github.com/detailyang/pre-commit-shell
+  rev: 1.0.5
+  hooks:
+  - id: shell-lint
+    args: ['-x', '--severity=warning']
+
+- repo: https://github.com/pre-commit/mirrors-clang-format
+  rev: v14.0.6
+  hooks:
+  - id: clang-format
+
+- repo: https://github.com/pryorda/dockerfilelint-precommit-hooks
+  rev: v0.1.0
+  hooks:
+  - id: dockerfilelint
+
+- repo: https://github.com/psf/black
+  rev: 22.10.0
+  hooks:
+  - id: black
+
+- repo: https://github.com/pycqa/flake8
+  rev: 5.0.4
+  hooks:
+  - id: flake8
+    args:
+      - "--max-line-length=110"
+
+- repo: local
+  hooks:
+  - id: copyright_check
+    name: 'check copyright message'
+    language: system
+    types: ['file', 'text']
+    exclude_types: ['markdown', 'xml', 'json', 'csv']
+    entry: ./lint/copyright_check.sh
+    exclude: 'models/.*'
+  - id: lfs_check
+    name: 'check LFS objects'
+    language: system
+    entry: ./lint/lfs_check.sh
+    require_serial: true
+
+- repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook
+  rev: v9.0.0
+  hooks:
+  - id: commitlint
+    stages: [commit-msg]
+    args: ['--config','lint/commitlint.config.js']
+    additional_dependencies: ['@commitlint/config-conventional']

samples/object_detection/models/download_models.sh

@@ -0,0 +1,45 @@
+#!/bin/bash -e
+
+# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Object Detection
+# PeopleNet model
+# This model is based on NVIDIA DetectNet_v2 detector with ResNet34 as feature extractor.
+
+OUT_DIR='/tmp'
+
+if [[ $# -ge 1 ]]; then
+   OUT_DIR=$1
+fi
+
+# Download the etlt model and the labels files from NGC
+
+if [ ! -f $OUT_DIR/resnet34_peoplenet_int8.etlt ]
+then
+	wget 'https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet/versions/deployable_quantized_v2.6.1/files/resnet34_peoplenet_int8.etlt' -P $OUT_DIR
+fi
+
+if [ ! -f $OUT_DIR/labels.txt ]
+then
+	wget 'https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet/versions/deployable_quantized_v2.6.1/files/labels.txt' -P $OUT_DIR
+fi
+
+# Use tao-converter which parses the .etlt model file, and generates an optimized TensorRT engine
+# The model supports implicit batch dimension which requires the max batch size, input layer dimensions and ordering to be specified.
+if [ ! -f ${OUT_DIR}/peoplenet.engine ]
+then
+	/tmp/tao_binaries/tao-converter -e $OUT_DIR/peoplenet.engine -k tlt_encode -d 3,544,960 -m 32 -i nchw $OUT_DIR/resnet34_peoplenet_int8.etlt
+fi

samples/segmentation_triton/python/models/fcn_resnet101/1/model.py

@@ -0,0 +1,119 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# docs_tag: begin_python_imports
+# NOTE: One must import PyCuda driver first, before CVCUDA or VPF otherwise
+# things may throw unexpected errors.
+import pycuda.driver as cuda
+import json
+import torch
+import cvcuda
+from model_inference import SegmentationPyTorch
+from pipelines import PreprocessorCvcuda, PostprocessorCvcuda
+
+# Import Triton modules
+import triton_python_backend_utils as pb_utils
+
+# docs_tag: end_python_imports
+
+
+# Triton Python Model
+class TritonPythonModel:
+    def initialize(self, args):
+        # docs_tag: begin_init_model
+        self.model_config = model_config = json.loads(args["model_config"])
+        params = model_config["parameters"]
+        self.device_id = int(params["device_id"]["string_value"])
+        self.network_width = int(params["network_width"]["string_value"])
+        self.network_height = int(params["network_height"]["string_value"])
+        self.visualization_class_name = params["visualization_class_name"][
+            "string_value"
+        ]
+        cuda_device = cuda.Device(self.device_id)
+        self.cuda_ctx = cuda_device.retain_primary_context()
+        self.cuda_ctx.push()
+        self.cvcuda_stream = cvcuda.Stream()
+        self.torch_stream = torch.cuda.ExternalStream(self.cvcuda_stream.handle)
+
+        self.inference = SegmentationPyTorch(
+            output_dir="/tmp",
+            seg_class_name=self.visualization_class_name,
+            batch_size=1,
+            image_size=(self.network_width, self.network_height),
+            device_id=self.device_id,
+        )
+
+        self.input_tensor_name = "inputrgb"
+        self.output_tensor_name = "outputrgb"
+
+        self.preprocess = PreprocessorCvcuda(self.device_id)
+        self.postprocess = PostprocessorCvcuda(
+            "NCHW",
+            gpu_output=True,
+            device_id=self.device_id,
+        )
+        # docs_tag: end_init_model
+
+    # docs_tag: begin_execute_model
+    def execute(self, requests):
+        responses = []
+        # Every Python backend must iterate over everyone of the requests
+        # and create a pb_utils.InferenceResponse for each of them.
+        try:
+            with self.cvcuda_stream, torch.cuda.stream(self.torch_stream):
+                for request in requests:
+                    in_0 = pb_utils.get_input_tensor_by_name(
+                        request, self.input_tensor_name
+                    )
+                    in_0_numpy = in_0.as_numpy()
+                    image_tensors = torch.from_numpy(in_0_numpy)
+
+                    orig_tensor, resized_tensor, normalized_tensor = self.preprocess(
+                        image_tensors.cuda(),
+                        out_size=(self.network_width, self.network_height),
+                    )
+
+                    probabilities = self.inference(normalized_tensor)
+
+                    blurred_frame = self.postprocess(
+                        probabilities,
+                        orig_tensor,
+                        resized_tensor,
+                        self.inference.class_index,
+                    )
+
+                    # Get Triton output tensor
+                    out_tensor_0 = pb_utils.Tensor(
+                        "outputrgb", blurred_frame.cpu().numpy()
+                    )
+                    # Create inference response
+                    inference_response = pb_utils.InferenceResponse(
+                        output_tensors=[out_tensor_0]
+                    )
+                    responses.append(inference_response)
+
+            # You should return a list of pb_utils.InferenceResponse. Length
+            # of this list must match the length of `requests` list.
+
+            return responses
+        except Exception as e:
+            print(e)
+        # docs_tag: end_execute_model
+
+    # docs_tag: begin_finalize_model
+    def finalize(self):
+        self.cuda_ctx.pop()
+
+    # docs_tag: end_finalize_model

samples/segmentation_triton/python/models/fcn_resnet101/1/model_inference.py

@@ -0,0 +1,92 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Bring the commons folder from the samples directory into our path so that
+# we can import modules from it.
+import logging
+import torch
+import nvtx
+from torchvision.models import segmentation as segmentation_models
+
+
+# docs_tag: begin_init_segmentationpytorch
+class SegmentationPyTorch:  # noqa: E302
+    def __init__(
+        self,
+        output_dir,
+        seg_class_name,
+        batch_size,
+        image_size,
+        device_id,
+    ):
+        self.logger = logging.getLogger(__name__)
+        self.output_dir = output_dir
+        self.device_id = device_id
+        # Fetch the segmentation index to class name information from the weights
+        # meta properties.
+        # The underlying pytorch model that we use for inference is the FCN model
+        # from torchvision.
+        torch_model = segmentation_models.fcn_resnet101
+        weights = segmentation_models.FCN_ResNet101_Weights.DEFAULT
+
+        try:
+            self.class_index = weights.meta["categories"].index(seg_class_name)
+        except ValueError:
+            raise ValueError(
+                "Requested segmentation class '%s' is not supported by the "
+                "fcn_resnet101 model. All supported class names are: %s"
+                % (seg_class_name, ", ".join(weights.meta["categories"]))
+            )
+
+        # Inference uses PyTorch to run a segmentation model on the pre-processed
+        # input and outputs the segmentation masks.
+        class FCN_Softmax(torch.nn.Module):
+            def __init__(self, fcn):
+                super(FCN_Softmax, self).__init__()
+                self.fcn = fcn
+
+            def forward(self, x):
+                infer_output = self.fcn(x)["out"]
+                return torch.nn.functional.softmax(infer_output, dim=1)
+
+        fcn_base = torch_model(weights=weights)
+        fcn_base.eval()
+        self.model = FCN_Softmax(fcn_base).cuda(self.device_id)
+        self.model.eval()
+
+        self.logger.info("Using PyTorch as the inference engine.")
+        # docs_tag: end_init_segmentationpytorch
+
+    # docs_tag: begin_call_segmentationpytorch
+    def __call__(self, tensor):
+        nvtx.push_range("inference.torch")
+
+        with torch.no_grad():
+
+            if isinstance(tensor, torch.Tensor):
+                # We are all good here. Nothing needs to be done.
+                pass
+            else:
+                # Convert CVCUDA tensor to Torch tensor.
+                tensor = torch.as_tensor(
+                    tensor.cuda(), device="cuda:%d" % self.device_id
+                )
+
+            segmented = self.model(tensor)
+
+        nvtx.pop_range()
+        return segmented
+
+    # docs_tag: end_call_segmentationpytorch