Merge branch 'main' into target-device

Author: dtrawins

CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -24,7 +24,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-cmake_minimum_required(VERSION 3.17)
+cmake_minimum_required(VERSION 3.31.8)
 
 project(tritonopenvinobackend LANGUAGES C CXX)
 

README.md

@@ -57,7 +57,7 @@ Follow the steps below to build the backend shared library.
 ```
 $ mkdir build
 $ cd build
-$ cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install -DTRITON_BUILD_OPENVINO_VERSION=2025.0.0 -DTRITON_BUILD_CONTAINER_VERSION=25.01  ..
+$ cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install -DTRITON_BUILD_OPENVINO_VERSION=2025.2.0 -DTRITON_BUILD_CONTAINER_VERSION=25.06  ..
 $ make install
 ```
 The compiled backend will be added to `build/install/backends/openvino` folder.

src/openvino.cc

@@ -1,4 +1,4 @@
-// Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
@@ -557,7 +557,7 @@ ModelState::ValidateInputs(const size_t expected_input_cnt)
     }
 
     auto openvino_element = ModelConfigDataTypeToOpenVINOElement(io_dtype);
-    if (openvino_element == ov::element::undefined) {
+    if (openvino_element == ov::element::dynamic) {
       return TRITONSERVER_ErrorNew(
           TRITONSERVER_ERROR_INTERNAL,
           (std::string("unsupported datatype ") + io_dtype + " for input '" +
@@ -658,7 +658,7 @@ ModelState::ValidateOutputs()
     }
 
     auto openvino_element = ModelConfigDataTypeToOpenVINOElement(io_dtype);
-    if (openvino_element == ov::element::undefined) {
+    if (openvino_element == ov::element::dynamic) {
       return TRITONSERVER_ErrorNew(
           TRITONSERVER_ERROR_INTERNAL,
           (std::string("unsupported datatype ") + io_dtype + " for output '" +
@@ -703,7 +703,7 @@ ModelState::AutoCompleteConfig()
   RETURN_IF_ERROR(
       ModelConfig().MemberAsString("default_model_filename", &artifact_name));
   std::string model_path;
-  THROW_IF_BACKEND_INSTANCE_ERROR(ReadModel(artifact_name, &model_path));
+  RETURN_IF_ERROR(ReadModel(artifact_name, &model_path));
   model_read_ = false;  // Re-read model after autocomplete
 
   // Get OV model inputs and outputs
@@ -1279,7 +1279,9 @@ ModelInstanceState::SetInputTensors(
       batchn_shape[0] = total_batch_size;
     }
 
-    const int64_t batchn_byte_size = GetByteSize(input_datatype, batchn_shape);
+    int64_t batchn_byte_size = 0;
+    RETURN_IF_ERROR(
+        GetByteSize(input_datatype, batchn_shape, &batchn_byte_size));
 
     if (batch_pad_size_ != 0) {
       ov::Tensor input_tensor =
@@ -1293,7 +1295,7 @@ ModelInstanceState::SetInputTensors(
              "batch_size equal to max_batch_size for better performance.")
                 .c_str());
       }
-      char* dest = (char*)input_tensor.data(ov::element::undefined);
+      char* dest = (char*)input_tensor.data(ov::element::dynamic);
       memset(dest, 0, input_tensor.get_byte_size());
       collector.ProcessTensor(
           input_name, dest, input_tensor.get_byte_size(),
@@ -1361,7 +1363,7 @@ ModelInstanceState::ReadOutputTensors(
 
     responder.ProcessTensor(
         name, ConvertFromOpenVINOElement(output_tensor.get_element_type()),
-        output_shape, (const char*)output_tensor.data(ov::element::undefined),
+        output_shape, (const char*)output_tensor.data(ov::element::dynamic),
         TRITONSERVER_MEMORY_CPU, 0);
   }
 

src/openvino_utils.cc

@@ -1,4 +1,4 @@
-// Copyright 2021-2022, NVIDIA CORPORATION. All rights reserved.
+// Copyright 2021-2025, NVIDIA CORPORATION. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
@@ -68,8 +68,6 @@ ConvertFromOpenVINOElement(ov::element::Type openvino_element)
     case ov::element::u64:
       return TRITONSERVER_TYPE_UINT64;
     // The following types are not supported:
-    // Unspecified value. Used by default
-    case ov::element::undefined:
     // Dynamic value
     case ov::element::dynamic:
     // 16bit floating point value, 8 bit for exponent, 7 bit for mantisa
@@ -119,7 +117,7 @@ ConvertToOpenVINOElement(TRITONSERVER_DataType data_type)
       break;
   }
 
-  return ov::element::undefined;
+  return ov::element::dynamic;
 }
 
 ov::element::Type
@@ -162,7 +160,7 @@ ModelConfigDataTypeToOpenVINOElement(const std::string& data_type_str)
     }
   }
 
-  return ov::element::undefined;
+  return ov::element::dynamic;
 }
 
 std::string

tools/gen_openvino_dockerfile.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -145,13 +145,13 @@ def dockerfile_for_windows(output_file):
 SHELL ["cmd", "/S", "/C"]
 
 # Install instructions:
-# https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-archive-windows.html
+# https://docs.openvino.ai/2025/get-started/install-openvino/install-openvino-archive-windows.html
 
 # The windows part is using pre-build archive, while the linux part is building
 # from source.
 # TODO: Unify build steps between windows and linux.
 
-ARG OPENVINO_VERSION=2025.0.0
+ARG OPENVINO_VERSION=2025.2.0
 ARG OPENVINO_BUILD_TYPE
 
 WORKDIR /workspace
@@ -161,7 +161,10 @@ def dockerfile_for_windows(output_file):
 RUN IF "%OPENVINO_VERSION%"=="2024.4.0" curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/windows/w_openvino_toolkit_windows_2024.4.0.16579.c3152d32c9c_x86_64.zip --output ov.zip
 RUN IF "%OPENVINO_VERSION%"=="2024.5.0" curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/windows/w_openvino_toolkit_windows_2024.5.0.17288.7975fa5da0c_x86_64.zip --output ov.zip
 RUN IF "%OPENVINO_VERSION%"=="2025.0.0" curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2025.0/windows/openvino_toolkit_windows_2025.0.0.17942.1f68be9f594_x86_64.zip --output ov.zip
+RUN IF "%OPENVINO_VERSION%"=="2025.1.0" curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2025.1/windows/openvino_toolkit_windows_2025.1.0.18503.6fec06580ab_x86_64.zip --output ov.zip
+RUN IF "%OPENVINO_VERSION%"=="2025.2.0" curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2025.2/windows/openvino_toolkit_windows_2025.2.0.19140.c01cd93e24d_x86_64.zip --output ov.zip
 RUN IF not exist ov.zip ( echo "OpenVINO version %OPENVINO_VERSION% not supported" && exit 1 )
+
 RUN tar -xf ov.zip
 RUN powershell.exe "Get-ChildItem w_openvino_toolkit_windows_* | foreach { ren $_.fullname install }"