Add initial test infra

Author: chilo-ms

plugin_execution_providers/test/tensorrt/CMakeLists.txt

@@ -0,0 +1,136 @@
+# usage:
+# cd build/
+# cmake -S ../ -B ./ -DCMAKE_BUILD_TYPE=Debug -DORT_HOME=/path/to/ort_package/onnxruntime-win-x64-gpu-1.23.0 -DCMAKE_CUDA_ARCHITECTURES=80 -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DTENSORRT_HOME=/path/to/tensorrt/TensorRT-10.3.0.26 -DCMAKE_POSITION_INDEPENDENT_CODE=ON (see the result of "nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits")
+# cmake --build ./ --config Debug
+cmake_minimum_required(VERSION 3.26)
+project(tensorrt_ep_test VERSION 1.0)
+set(CMAKE_CXX_STANDARD 17)
+
+# CMake config to force dynamic debug CRT or dynamic release CRT globally for all dependencies.
+# This is to address the issue of:
+# libprotobufd.lib(common.obj) : error LNK2038: mismatch detected for 'RuntimeLibrary': value 'MTd_StaticDebug' doesn't match value 'MDd_DynamicDebug' in unary_elementwise_ops_impl.obj
+if (WIN32)
+  if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+    set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDebugDLL" CACHE STRING "" FORCE)  # /MDd
+    set(BUILD_SHARED_LIBS OFF)  # Build protobuf as static .lib, but using dynamic runtime
+  endif()
+
+  if(CMAKE_BUILD_TYPE STREQUAL "Release")
+    set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDLL" CACHE STRING "" FORCE)
+    set(BUILD_SHARED_LIBS OFF)  # Build protobuf as static .lib, but using dynamic runtime
+  endif()
+endif()
+
+add_definitions(-DONNX_NAMESPACE=onnx)
+add_definitions(-DONNX_ML)
+add_definitions(-DNOMINMAX)
+file(GLOB tensorrt_ep_test_src "./*.cc" "./*.h")
+add_executable(tensorrt_ep_test ${tensorrt_ep_test_src})
+
+if (NOT ORT_HOME)
+  message(FATAL_ERROR "Please specify ORT_HOME, e.g. -DORT_HOME=/path/to/ort/")
+endif()
+
+# Use release mode if not specified
+if (NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE "Release")
+endif()
+
+# Add dependencies
+include(FetchContent)
+
+# Add protobuf
+FetchContent_Declare(
+  protobuf
+  GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git
+  GIT_TAG        v21.12  # Use a specific tag or commit
+)
+
+if (WIN32)
+  # Sometimes, protobuf ignores CMAKE_MSVC_RUNTIME_LIBRARY. To ensure it works:
+  set(protobuf_MSVC_STATIC_RUNTIME OFF CACHE BOOL "" FORCE)
+endif()
+
+set(protobuf_BUILD_TESTS OFF CACHE BOOL "" FORCE)
+
+FetchContent_MakeAvailable(protobuf)
+
+# Add ONNX
+FetchContent_Declare(
+  onnx
+  GIT_REPOSITORY https://github.com/onnx/onnx.git
+  GIT_TAG        v1.18.0  # Use a specific tag or commit
+)
+
+FetchContent_MakeAvailable(onnx)
+
+# Add GSL
+FetchContent_Declare(
+  gsl
+  GIT_REPOSITORY https://github.com/microsoft/GSL.git
+  GIT_TAG        v4.0.0  # Use a specific tag or commit
+)
+
+FetchContent_MakeAvailable(gsl)
+
+# Add GoogleTest
+FetchContent_Declare(
+  googletest
+  URL https://github.com/google/googletest/archive/refs/heads/main.zip
+)
+# For Windows: prevents overriding parent project's runtime library settings
+set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
+FetchContent_MakeAvailable(googletest)
+
+set(DEPS_PATH "${CMAKE_BINARY_DIR}/_deps")
+
+if (WIN32) # Windows
+  set(ORT_LIB "${ORT_HOME}/lib/onnxruntime.lib")
+  
+  if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+    set(DEPS_LIBS ${DEPS_LIBS}
+                  "${DEPS_PATH}/protobuf-build/${CMAKE_BUILD_TYPE}/libprotobufd.lib"
+                  "${DEPS_PATH}/protobuf-build/${CMAKE_BUILD_TYPE}/libprotocd.lib")
+  else()
+    set(DEPS_LIBS ${DEPS_LIBS} 
+                  "${DEPS_PATH}/protobuf-build/${CMAKE_BUILD_TYPE}/libprotobuf.lib"
+                  "${DEPS_PATH}/protobuf-build/${CMAKE_BUILD_TYPE}/libprotoc.lib")
+  endif()
+
+  set(DEPS_LIBS "${DEPS_PATH}/onnx-build/${CMAKE_BUILD_TYPE}/onnx.lib"
+                "${DEPS_PATH}/onnx-build/${CMAKE_BUILD_TYPE}/onnx_proto.lib")
+
+else() # Linux
+  set(ORT_LIB "${ORT_HOME}/lib/libonnxruntime.so")
+
+  set(DEPS_LIBS "${DEPS_PATH}/onnx-build/libonnx.a"
+               "${DEPS_PATH}/onnx-build/libonnx_proto.a")
+
+  if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+    set(DEPS_LIBS ${DEPS_LIBS}
+                 "${DEPS_PATH}/protobuf-build/libprotobufd.a"
+                 "${DEPS_PATH}/protobuf-build/libprotocd.a")
+  else()
+    set(DEPS_LIBS ${DEPS_LIBS}
+                 "${DEPS_PATH}/protobuf-build/libprotobuf.a"
+                 "${DEPS_PATH}/protobuf-build/libprotoc.a")
+  endif()
+endif()
+
+MESSAGE(STATUS "Looking for following dependencies ...")
+MESSAGE(STATUS "ORT lib  : ${ORT_LIB}")
+MESSAGE(STATUS "Deps libs: ${DEPS_LIBS}")
+
+
+target_include_directories(tensorrt_ep_test PUBLIC "${ORT_HOME}/include"
+                                             "${DEPS_PATH}/gsl-src/include" # GSL is header-only
+                                             "${DEPS_PATH}/onnx-src"
+                                             "${DEPS_PATH}/onnx-build"
+                                             "${DEPS_PATH}/protobuf-src/src"
+)
+
+target_link_libraries(tensorrt_ep_test PUBLIC #${DEPS_LIBS}
+                                        GTest::gtest GTest::gtest_main
+                                        protobuf::libprotobuf onnx
+                                        ${ORT_LIB}
+)

plugin_execution_providers/test/tensorrt/helper.cc

@@ -0,0 +1,67 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#ifdef _WIN32
+#include <Windows.h>
+#include <assert.h>
+#endif
+
+#include <stdexcept>
+
+#ifdef ORT_NO_EXCEPTIONS
+#if defined(__ANDROID__)
+#include <android/log.h>
+#else
+#include <iostream>
+#endif
+#endif
+
+#include <string>
+
+#define THROW(...) throw std::runtime_error(std::string(__VA_ARGS__));
+
+#ifdef _WIN32
+std::string ToUTF8String(std::wstring_view s) {
+  if (s.size() >= static_cast<size_t>(std::numeric_limits<int>::max()))
+    THROW("length overflow");
+
+  const int src_len = static_cast<int>(s.size() + 1);
+  const int len = WideCharToMultiByte(CP_UTF8, 0, s.data(), src_len, nullptr, 0, nullptr, nullptr);
+  assert(len > 0);
+  std::string ret(static_cast<size_t>(len) - 1, '\0');
+#pragma warning(disable : 4189)
+  const int r = WideCharToMultiByte(CP_UTF8, 0, s.data(), src_len, (char*)ret.data(), len, nullptr, nullptr);
+  assert(len == r);
+#pragma warning(default : 4189)
+  return ret;
+}
+
+std::wstring ToWideString(std::string_view s) {
+  if (s.size() >= static_cast<size_t>(std::numeric_limits<int>::max()))
+    THROW("length overflow");
+
+  const int src_len = static_cast<int>(s.size() + 1);
+  const int len = MultiByteToWideChar(CP_UTF8, 0, s.data(), src_len, nullptr, 0);
+  assert(len > 0);
+  std::wstring ret(static_cast<size_t>(len) - 1, '\0');
+#pragma warning(disable : 4189)
+  const int r = MultiByteToWideChar(CP_UTF8, 0, s.data(), src_len, (wchar_t*)ret.data(), len);
+  assert(len == r);
+#pragma warning(default : 4189)
+  return ret;
+}
+#endif  // #ifdef _WIN32
+
+#ifdef NO_EXCEPTIONS
+void PrintFinalMessage(const char* msg) {
+#if defined(__ANDROID__)
+  __android_log_print(ANDROID_LOG_ERROR, "onnxruntime", "%s", msg);
+#else
+  // TODO, consider changing the output of the error message from std::cerr to logging when the
+  // exceptions are disabled, since using std::cerr might increase binary size, and std::cerr output
+  // might not be easily accessible on some systems such as mobile
+  // TODO, see if we need to change the output of the error message from std::cerr to NSLog for iOS
+  std::cerr << msg << std::endl;
+#endif
+}
+#endif  // #ifdef NO_EXCEPTIONS

plugin_execution_providers/test/tensorrt/path_string.h

@@ -0,0 +1,96 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <string>
+#include <type_traits>
+
+// for std::tolower or std::towlower
+#ifdef _WIN32
+#include <cwctype>
+#else
+#include <cctype>
+#endif
+
+#include "onnxruntime_c_api.h"
+
+// char type for filesystem paths
+using PathChar = ORTCHAR_T;
+// string type for filesystem paths
+using PathString = std::basic_string<PathChar>;
+
+inline std::string ToUTF8String(const std::string& s) { return s; }
+#ifdef _WIN32
+/**
+ * Convert a wide character string to a UTF-8 string
+ */
+std::string ToUTF8String(std::wstring_view s);
+inline std::string ToUTF8String(const wchar_t* s) {
+  return ToUTF8String(std::wstring_view{s});
+}
+inline std::string ToUTF8String(const std::wstring& s) {
+  return ToUTF8String(std::wstring_view{s});
+}
+std::wstring ToWideString(std::string_view s);
+inline std::wstring ToWideString(const char* s) {
+  return ToWideString(std::string_view{s});
+}
+inline std::wstring ToWideString(const std::string& s) {
+  return ToWideString(std::string_view{s});
+}
+inline std::wstring ToWideString(const std::wstring& s) { return s; }
+inline std::wstring ToWideString(std::wstring_view s) { return std::wstring{s}; }
+#else
+inline std::string ToWideString(const std::string& s) { return s; }
+inline std::string ToWideString(const char* s) { return s; }
+inline std::string ToWideString(std::string_view s) { return std::string{s}; }
+#endif
+
+inline PathString ToPathString(const PathString& s) {
+  return s;
+}
+
+#ifdef _WIN32
+
+static_assert(std::is_same<PathString, std::wstring>::value, "PathString is not std::wstring!");
+
+inline PathString ToPathString(std::string_view s) {
+  return ToWideString(s);
+}
+inline PathString ToPathString(const char* s) {
+  return ToWideString(s);
+}
+inline PathString ToPathString(const std::string& s) {
+  return ToWideString(s);
+}
+
+inline PathChar ToLowerPathChar(PathChar c) {
+  return std::towlower(c);
+}
+
+inline std::string PathToUTF8String(const PathString& s) {
+  return ToUTF8String(s);
+}
+
+#else
+
+static_assert(std::is_same<PathString, std::string>::value, "PathString is not std::string!");
+
+inline PathString ToPathString(const char* s) {
+  return s;
+}
+
+inline PathString ToPathString(std::string_view s) {
+  return PathString{s};
+}
+
+inline PathChar ToLowerPathChar(PathChar c) {
+  return std::tolower(c);
+}
+
+inline std::string PathToUTF8String(const PathString& s) {
+  return s;
+}
+
+#endif

plugin_execution_providers/test/tensorrt/tensorrt_basic_test.cc

@@ -0,0 +1,87 @@
+#include <gtest/gtest.h>
+#include <string>
+#include <gsl/gsl>
+
+#include "onnxruntime_cxx_api.h"
+#include "test_trt_ep_utils.h"
+#include "path_string.h"
+
+namespace test {
+namespace trt_ep {
+// char type for filesystem paths
+using PathChar = ORTCHAR_T;
+// string type for filesystem paths
+using PathString = std::basic_string<PathChar>;
+
+class TensorrtExecutionProviderCacheTest : public testing::TestWithParam<std::string> {};
+
+OrtStatus* CreateOrtSession(PathString model_name,
+                            std::string lib_registration_name,
+                            PathString lib_path) {
+  const OrtApi* ort_api = OrtGetApiBase()->GetApi(ORT_API_VERSION);
+  Ort::Env env;
+
+  // Register plugin TRT EP library with ONNX Runtime.
+  env.RegisterExecutionProviderLibrary(
+      lib_registration_name.c_str(),  // Registration name can be anything the application chooses.
+      lib_path                        // Path to the plugin TRT EP library.
+  );
+
+  // Unregister the library using the application-specified registration name.
+  // Must only unregister a library after all sessions that use the library have been released.
+  auto unregister_plugin_eps_at_scope_exit =
+      gsl::finally([&]() { env.UnregisterExecutionProviderLibrary(lib_registration_name.c_str()); });
+
+  {
+    std::vector<Ort::ConstEpDevice> ep_devices = env.GetEpDevices();
+    // EP name should match the name assigned by the EP factory when creating the EP (i.e., in the implementation of
+    // OrtEP::CreateEp())
+    std::string ep_name = lib_registration_name;
+
+    // Find the Ort::EpDevice for "TensorRTEp".
+    std::vector<Ort::ConstEpDevice> selected_ep_devices = {};
+    for (Ort::ConstEpDevice ep_device : ep_devices) {
+      if (std::string(ep_device.EpName()) == ep_name) {
+        selected_ep_devices.push_back(ep_device);
+        break;
+      }
+    }
+
+    if (selected_ep_devices[0] == nullptr) {
+      // Did not find EP. Report application error ...
+      std::string message = "Did not find EP: " + ep_name;
+      return ort_api->CreateStatus(ORT_EP_FAIL, message.c_str());
+    }
+
+    std::unordered_map<std::string, std::string> ep_options;  // Optional EP options.
+    Ort::SessionOptions session_options;
+    session_options.AppendExecutionProvider_V2(env, selected_ep_devices, ep_options);
+
+    Ort::Session session(env, model_name.c_str(), session_options);
+
+    // Get default ORT allocator
+    Ort::AllocatorWithDefaultOptions allocator;
+
+    // Get input name
+    Ort::AllocatedStringPtr input_name_ptr =
+        session.GetInputNameAllocated(0, allocator);  // Keep the smart pointer alive to avoid dangling pointer
+    const char* input_name = input_name_ptr.get();
+
+  }
+
+
+}
+
+TEST(TensorrtExecutionProviderTest, SessionCreationWithMultiThreadsAndInferenceWithMultiThreads) {
+  std::vector<std::thread> threads;
+  std::string model_name = "basic_model_for_test.onnx";
+  std::string graph_name = "basic_model";
+  std::string lib_registration_name = "TensorRTEp";
+  PathString lib_path = ORT_TSTR("TensorRTEp.dll");
+  std::vector<int64_t> dims = {1, 3, 2};
+  CreateBaseModel(model_name, graph_name, dims);
+  CreateOrtSession(ToPathString(model_name), lib_registration_name, lib_path);
+}
+
+}  // namespace trt_ep
+}  // namespace test