BREAKING CHANGE: Removing deprecated APIs like setNumMinTimingIters and added DLA memory size configurations

Signed-off-by: Dheeraj Peri <[email protected]>

Author: peri044

core/conversion/conversionctx/ConversionCtx.h

@@ -33,7 +33,6 @@ struct BuilderSettings {
   Device device;
   nvinfer1::EngineCapability capability = TRT_ENGINE_CAPABILITY_STANDARD;
   nvinfer1::IInt8Calibrator* calibrator = nullptr;
-  uint64_t num_min_timing_iters = 2;
   uint64_t num_avg_timing_iters = 1;
   uint64_t workspace_size = 0;
   uint64_t dla_sram_size = 1048576;

cpp/bin/torchtrtc/README.md

@@ -82,13 +82,17 @@ torchtrtc [input_file_path] [output_file_path]
                                         serialized TensorRT engine and embed it
                                         into a TorchScript module (device spec
                                         must be provided)
-      --num-min-timing-iter=[num_iters] Number of minimization timing iterations
-                                        used to select kernels
       --num-avg-timing-iters=[num_iters]
                                         Number of averaging timing iterations
                                         used to select kernels
       --workspace-size=[workspace_size] Maximum size of workspace given to
                                         TensorRT
+      --dla-sram-size=[dla_sram_size]   Fast software managed RAM used by DLA
+                                        to communicate within a layer.
+      --dla-local-dram-size=[dla_local_dram_size]  Host RAM used by DLA to share
+                                        intermediate tensor data across operations.
+      --dla-global-dram-size=[dla_global_dram_size] Host RAM used by DLA to store
+                                        weights and metadata for execution
       --atol=[atol]                     Absolute tolerance threshold for acceptable
                                         numerical deviation from standard torchscript
                                         output (default 1e-8)

cpp/bin/torchtrtc/main.cpp

@@ -365,10 +365,6 @@ int main(int argc, char** argv) {
     }
   }
 
-  if (num_min_timing_iters) {
-    compile_settings.num_min_timing_iters = args::get(num_min_timing_iters);
-  }
-
   if (num_avg_timing_iters) {
     compile_settings.num_avg_timing_iters = args::get(num_avg_timing_iters);
   }

cpp/include/torch_tensorrt/torch_tensorrt.h

@@ -636,10 +636,6 @@ struct TORCHTRT_API CompileSpec {
    */
   EngineCapability capability = EngineCapability::kSTANDARD;
 
-  /**
-   * Number of minimization timing iterations used to select kernels
-   */
-  uint64_t num_min_timing_iters = 2;
   /**
    * Number of averaging timing iterations used to select kernels
    */

docsrc/tutorials/ptq.rst

@@ -130,8 +130,6 @@ Then all thats required to setup the module for INT8 calibration is to set the f
     compile_spec.enabled_precisions.insert(torch::kI8);
     /// Use the TensorRT Entropy Calibrator
     compile_spec.ptq_calibrator = calibrator;
-    /// Set a larger workspace (you may get better performace from doing so)
-    compile_spec.workspace_size = 1 << 28;
 
     auto trt_mod = torch_tensorrt::CompileGraph(mod, compile_spec);
 

docsrc/tutorials/torchtrtc.rst

@@ -85,13 +85,17 @@ to standard TorchScript. Load with ``torch.jit.load()`` and run like you would r
                                           serialized TensorRT engine and embed it
                                           into a TorchScript module (device spec
                                           must be provided)
-        --num-min-timing-iter=[num_iters] Number of minimization timing iterations
-                                          used to select kernels
         --num-avg-timing-iters=[num_iters]
                                           Number of averaging timing iterations
                                           used to select kernels
         --workspace-size=[workspace_size] Maximum size of workspace given to
                                           TensorRT
+        --dla-sram-size=[dla_sram_size]   Fast software managed RAM used by DLA
+                                          to communicate within a layer.
+        --dla-local-dram-size=[dla_local_dram_size]  Host RAM used by DLA to share
+                                          intermediate tensor data across operations.
+        --dla-global-dram-size=[dla_global_dram_size] Host RAM used by DLA to store
+                                          weights and metadata for execution
         --atol=[atol]                     Absolute tolerance threshold for acceptable
                                           numerical deviation from standard torchscript
                                           output (default 1e-8)

docsrc/tutorials/using_dla.rst

@@ -33,9 +33,6 @@ Using DLA in a C++ application
     # If a layer fails to run on DLA it will fallback to GPU
     compile_spec.device.allow_gpu_fallback = true;
 
-    # Set the workspace size
-    compile_spec.workspace_size = 1 << 28;
-
 
 Using DLA in a python application
 

py/torch_tensorrt/csrc/register_tensorrt_classes.cpp

@@ -60,8 +60,6 @@ void RegisterTRTCompileSpec() {
   ADD_FIELD_GET_SET_REGISTRATION(TRTCompileSpecTSRegistration, torch_tensorrt::pyapi::CompileSpec, refit);
   ADD_FIELD_GET_SET_REGISTRATION(TRTCompileSpecTSRegistration, torch_tensorrt::pyapi::CompileSpec, debug);
   ADD_FIELD_GET_SET_REGISTRATION(TRTCompileSpecTSRegistration, torch_tensorrt::pyapi::CompileSpec, capability);
-  ADD_FIELD_GET_SET_REGISTRATION(
-      TRTCompileSpecTSRegistration, torch_tensorrt::pyapi::CompileSpec, num_min_timing_iters);
   ADD_FIELD_GET_SET_REGISTRATION(
       TRTCompileSpecTSRegistration, torch_tensorrt::pyapi::CompileSpec, num_avg_timing_iters);
   ADD_FIELD_GET_SET_REGISTRATION(TRTCompileSpecTSRegistration, torch_tensorrt::pyapi::CompileSpec, workspace_size);

py/torch_tensorrt/csrc/torch_tensorrt_py.cpp

@@ -300,7 +300,6 @@ PYBIND11_MODULE(_C, m) {
       .def_readwrite("debug", &CompileSpec::debug)
       .def_readwrite("device", &CompileSpec::device)
       .def_readwrite("capability", &CompileSpec::capability)
-      .def_readwrite("num_min_timing_iters", &CompileSpec::num_min_timing_iters)
       .def_readwrite("num_avg_timing_iters", &CompileSpec::num_avg_timing_iters)
       .def_readwrite("workspace_size", &CompileSpec::workspace_size)
       .def_readwrite("dla_sram_size", &CompileSpec::dla_sram_size)

py/torch_tensorrt/ts/_compile_spec.py

@@ -211,6 +211,18 @@ def _parse_compile_spec(compile_spec: Dict[str, Any]) -> _ts_C.CompileSpec:
         assert type(compile_spec["workspace_size"]) is int
         info.workspace_size = compile_spec["workspace_size"]
 
+    if "dla_sram_size" in compile_spec:
+        assert type(compile_spec["dla_sram_size"]) is int
+        info.dla_sram_size = compile_spec["dla_sram_size"]
+
+    if "dla_local_dram_size" in compile_spec:
+        assert type(compile_spec["dla_local_dram_size"]) is int
+        info.dla_local_dram_size = compile_spec["dla_local_dram_size"]
+
+    if "dla_global_dram_size" in compile_spec:
+        assert type(compile_spec["dla_global_dram_size"]) is int
+        info.dla_global_dram_size = compile_spec["dla_global_dram_size"]
+
     if "truncate_long_and_double" in compile_spec:
         assert type(compile_spec["truncate_long_and_double"]) is bool
         info.truncate_long_and_double = compile_spec["truncate_long_and_double"]
@@ -229,9 +241,11 @@ def TensorRTCompileSpec(inputs=[],
                         refit=False,
                         debug=False,
                         capability=_enums.EngineCapability.default,
-                        num_min_timing_iters=2,
                         num_avg_timing_iters=1,
                         workspace_size=0,
+                        dla_sram_size=1048576,
+                        dla_local_dram_size=1073741824,
+                        dla_global_dram_size=536870912,
                         truncate_long_and_double=False,
                         calibrator=None) -> torch.classes.tensorrt.CompileSpec:
     """Utility to create a formated spec dictionary for using the PyTorch TensorRT backend
@@ -263,7 +277,6 @@ def TensorRTCompileSpec(inputs=[],
         refit (bool): Enable refitting
         debug (bool): Enable debuggable engine
         capability (torch_tensorrt.EngineCapability): Restrict kernel selection to safe gpu kernels or safe dla kernels
-        num_min_timing_iters (int): Number of minimization timing iterations used to select kernels
         num_avg_timing_iters (int): Number of averaging timing iterations used to select kernels
         workspace_size (int): Maximum size of workspace given to TensorRT
         truncate_long_and_double (bool): Truncate weights provided in int64 or double (float64) to int32 and float32
@@ -283,9 +296,11 @@ def TensorRTCompileSpec(inputs=[],
         "refit": refit,  # enable refit
         "debug": debug,  # enable debuggable engine
         "capability": capability,  # Restrict kernel selection to safe gpu kernels or safe dla kernels
-        "num_min_timing_iters": num_min_timing_iters,  # Number of minimization timing iterations used to select kernels
         "num_avg_timing_iters": num_avg_timing_iters,  # Number of averaging timing iterations used to select kernels
         "workspace_size": workspace_size,  # Maximum size of workspace given to TensorRT
+        "dla_sram_size": dla_sram_size,  # Fast software managed RAM used by DLA to communicate within a layer.
+        "dla_local_dram_size": dla_local_dram_size,  # Host RAM used by DLA to share intermediate tensor data across operations
+        "dla_global_dram_size": dla_global_dram_size,  # Host RAM used by DLA to store weights and metadata for execution
         "calibrator": calibrator,
         "truncate_long_and_double": truncate_long_and_double
     }
@@ -331,9 +346,11 @@ def TensorRTCompileSpec(inputs=[],
     backend_spec._set_debug(parsed_spec.debug)
     backend_spec._set_refit(parsed_spec.refit)
     backend_spec._set_capability(int(parsed_spec.capability))
-    backend_spec._set_num_min_timing_iters(parsed_spec.num_min_timing_iters)
     backend_spec._set_num_avg_timing_iters(parsed_spec.num_avg_timing_iters)
     backend_spec._set_workspace_size(parsed_spec.workspace_size)
+    backend_spec._set_dla_sram_size(parsed_spec.dla_sram_size)
+    backend_spec._set_dla_local_dram_size(parsed_spec.dla_local_dram_size)
+    backend_spec._set_dla_global_dram_size(parsed_spec._set_dla_global_dram_size)
     backend_spec._set_truncate_long_and_double(parsed_spec.truncate_long_and_double)
     backend_spec._set_ptq_calibrator(parsed_spec._get_calibrator_handle())