[Tensorrt] update features and bug fixes (#8935)

#8738
#8764
#8756
#8761
#8800
#8813
#8845
#8859
#8885
#8882
#8865
#8912

Author: zhupengyang

lite/api/cxx_api_impl.cc

@@ -91,6 +91,8 @@ void CxxPaddleApiImpl::Init(const lite_api::CxxConfig &config) {
         raw_predictor_->scope(), config.nnadapter_device_names());
     Context<TargetType::kNNAdapter>::SetNNAdapterContextProperties(
         raw_predictor_->scope(), config.nnadapter_context_properties());
+    Context<TargetType::kNNAdapter>::SetNNAdapterContextCallback(
+        raw_predictor_->scope(), config.nnadapter_context_callback());
     Context<TargetType::kNNAdapter>::SetNNAdapterModelCacheDir(
         raw_predictor_->scope(), config.nnadapter_model_cache_dir());
     Context<TargetType::kNNAdapter>::SetNNAdapterModelCacheBuffers(

lite/api/light_api_impl.cc

@@ -72,6 +72,8 @@ void LightPredictorImpl::Init(const lite_api::MobileConfig& config) {
       raw_predictor_->scope(), config.nnadapter_device_names());
   Context<TargetType::kNNAdapter>::SetNNAdapterContextProperties(
       raw_predictor_->scope(), config.nnadapter_context_properties());
+  Context<TargetType::kNNAdapter>::SetNNAdapterContextCallback(
+      raw_predictor_->scope(), config.nnadapter_context_callback());
   Context<TargetType::kNNAdapter>::SetNNAdapterModelCacheDir(
       raw_predictor_->scope(), config.nnadapter_model_cache_dir());
   Context<TargetType::kNNAdapter>::SetNNAdapterModelCacheBuffers(

lite/api/paddle_api.h

@@ -19,6 +19,7 @@
 
 #ifndef PADDLE_LITE_API_H_  // NOLINT
 #define PADDLE_LITE_API_H_
+#include <functional>
 #include <map>
 #include <memory>
 #include <string>
@@ -165,6 +166,8 @@ class LITE_API ConfigBase {
   // The NNAdapter context properties for device configuration, model
   // compilation and execution
   std::string nnadapter_context_properties_{};
+  int (*nnadapter_context_callback_)(int event_id,
+                                     void* user_data){nullptr};  // NOLINT
   // The directory to find and store the compiled NNAdapter models.
   std::string nnadapter_model_cache_dir_{""};
   // Dynamic shapes of the NNAdapter model
@@ -274,6 +277,26 @@ class LITE_API ConfigBase {
   const std::string& nnadapter_context_properties() const {
     return nnadapter_context_properties_;
   }
+  // Set nnadapter_context_callback for NNAdapter device to get runtime
+  // parameters.
+  // For example:
+  // cudaStream_t cuda_stream;
+  // cudaStreamCreate(&cuda_stream);
+  // int nnadapter_context_callback(int event_id, void* user_data) {
+  //   if (event_id == 0x0100) {
+  //     *(std::reinterpret_cast<cudaStream_t*>(user_data)) = cuda_stream;
+  //   }
+  //   return 0;
+  // }
+  void set_nnadapter_context_callback(
+      int (*nnadapter_context_callback)(int event_id, void* user_data)) {
+    nnadapter_context_callback_ = nnadapter_context_callback;
+  }
+  int (*nnadapter_context_callback() const)(int event_id,  // NOLINT
+                                            void* user_data) {
+    return nnadapter_context_callback_;
+  }
+
   // Enable caching and set the directory to search and store the compiled
   // NNAdapter models in the file system.
   void set_nnadapter_model_cache_dir(const std::string& model_cache_dir) {

lite/backends/nnadapter/nnadapter/include/nnadapter/core/types.h

@@ -38,7 +38,9 @@ typedef struct Operand {
 typedef struct Argument {
   int index;
   void* memory;
-  void* (*access)(void* memory, NNAdapterOperandType* type);
+  void* (*access)(void* memory,
+                  NNAdapterOperandType* type,
+                  void* device_buffer);
 } Argument;
 
 typedef struct Operation {

lite/backends/nnadapter/nnadapter/include/nnadapter/driver/device.h

@@ -28,7 +28,10 @@ typedef struct Device {
   // Interfaces
   int (*open_device)(void** device);
   void (*close_device)(void* device);
-  int (*create_context)(void* device, const char* properties, void** context);
+  int (*create_context)(void* device,
+                        const char* properties,
+                        int (*callback)(int event_id, void* user_data),
+                        void** context);
   void (*destroy_context)(void* context);
   int (*validate_program)(void* context,
                           const core::Model* model,

lite/backends/nnadapter/nnadapter/include/nnadapter/nnadapter.h

@@ -2316,6 +2316,7 @@ int NNAdapterDevice_getVersion(const NNAdapterDevice* device, int32_t* version);
 int NNAdapterContext_create(NNAdapterDevice** devices,
                             uint32_t num_devices,
                             const char* properties,
+                            int (*callback)(int event_id, void* user_data),
                             NNAdapterContext** context);
 /**
  * Release the context.
@@ -2489,7 +2490,8 @@ int NNAdapterExecution_setInput(NNAdapterExecution* execution,
                                 int32_t index,
                                 void* memory,
                                 void* (*access)(void* memory,
-                                                NNAdapterOperandType* type));
+                                                NNAdapterOperandType* type,
+                                                void* device_buffer));
 /**
  * Set the output memory and the function used to access it.
  *
@@ -2518,7 +2520,8 @@ int NNAdapterExecution_setOutput(NNAdapterExecution* execution,
                                  int32_t index,
                                  void* memory,
                                  void* (*access)(void* memory,
-                                                 NNAdapterOperandType* type));
+                                                 NNAdapterOperandType* type,
+                                                 void* device_buffer));
 /**
  * Start to run the execution synchronously.
  *

lite/backends/nnadapter/nnadapter/include/nnadapter/operation/reduce.h

@@ -14,6 +14,7 @@
 
 #pragma once
 
+#include <iostream>
 namespace nnadapter {
 namespace operation {
 
@@ -26,12 +27,15 @@ namespace operation {
   NNADAPTER_CHECK_EQ(output_count, 1);                                      \
   /* Input */                                                               \
   auto input_operand = input_operands[0];                                   \
+  auto input_dimension_count = input_operand->type.dimensions.count;        \
   NNADAPTER_VLOG(5) << "input_operand: " << OperandToString(input_operand); \
   /* Axes */                                                                \
   auto axes_operand = input_operands[1];                                    \
   int axes_size = axes_operand->length / sizeof(int32_t);                   \
   auto axes_data = reinterpret_cast<int32_t*>(axes_operand->buffer);        \
   for (int i = 0; i < axes_size; i++) {                                     \
+    axes_data[i] = axes_data[i] < 0 ? axes_data[i] + input_dimension_count  \
+                                    : axes_data[i];                         \
     NNADAPTER_VLOG(5) << "axes[" << i << "]: " << axes_data[i];             \
   }                                                                         \
   /* Keep_dim */                                                            \

lite/backends/nnadapter/nnadapter/src/driver/amlogic_npu/driver.cc

@@ -38,7 +38,10 @@ void CloseDevice(void* device) {
   }
 }
 
-int CreateContext(void* device, const char* properties, void** context) {
+int CreateContext(void* device,
+                  const char* properties,
+                  int (*callback)(int event_id, void* user_data),
+                  void** context) {
   if (!device || !context) {
     return NNADAPTER_INVALID_PARAMETER;
   }

lite/backends/nnadapter/nnadapter/src/driver/amlogic_npu/engine.cc

@@ -173,7 +173,7 @@ int Program::CheckInputsAndOutputs(uint32_t input_count,
     // Get actual type
     auto& arg = input_arguments[i];
     NNAdapterOperandType type;
-    arg.access(arg.memory, &type);
+    arg.access(arg.memory, &type, nullptr);
     // Check dimensions count
     uint32_t count = type.dimensions.count;
     int32_t* data = type.dimensions.data;
@@ -210,7 +210,7 @@ int Program::Execute(uint32_t input_count,
     NNADAPTER_CHECK(arg.memory);
     NNADAPTER_CHECK(arg.access);
     auto type = input_types_[arg.index];
-    auto buffer = arg.access(arg.memory, &type);
+    auto buffer = arg.access(arg.memory, &type, nullptr);
     NNADAPTER_CHECK(buffer);
     auto length = GetOperandTypeBufferLength(type);
     if (IsUInt8AsymmPerLayerQuantType(type.precision)) {
@@ -243,7 +243,7 @@ int Program::Execute(uint32_t input_count,
     // TODO(hong19860320) Get the dimensions of the outputs from amlnpu_ddk
     // according to the dynamic dimensions of the inputs, fill them to 'type'
     // and call the 'access' function to re-allocate the host output memory
-    auto buffer = arg.access(arg.memory, type);
+    auto buffer = arg.access(arg.memory, type, nullptr);
     NNADAPTER_CHECK(buffer);
     auto length = GetOperandTypeBufferLength(*type);
     // Initialize the output info for the execution

lite/backends/nnadapter/nnadapter/src/driver/android_nnapi/driver.cc

@@ -38,7 +38,10 @@ void CloseDevice(void* device) {
   }
 }
 
-int CreateContext(void* device, const char* properties, void** context) {
+int CreateContext(void* device,
+                  const char* properties,
+                  int (*callback)(int event_id, void* user_data),
+                  void** context) {
   if (!device || !context) {
     return NNADAPTER_INVALID_PARAMETER;
   }