Merge pull request #1975 from pytorch/torch_version_upgrade_jun23_nightly

chore: Upgrade Torch nightly to `2.1.0.dev20230605` [4 / x]

Author: gs-olive

.bazelrc

@@ -22,7 +22,7 @@
 # +------------------------------------------------------------+
 # Enable colorful output of GCC
 build --cxxopt="-fdiagnostics-color=always"
-build --cxxopt='-std=c++14'
+build --cxxopt='-std=c++17'
 #build --linkopt="-Wl,--no-as-needed"
 
 

.circleci/config.yml

@@ -269,10 +269,10 @@ commands:
     parameters:
       torch-build:
         type: string
-        default: "2.1.0.dev20230419+cu118"
+        default: "2.1.0.dev20230605+cu118"
       torchvision-build:
         type: string
-        default: "0.16.0.dev20230419+cu118"
+        default: "0.16.0.dev20230605+cu118"
       torch-build-index:
         type: string
         default: "https://download.pytorch.org/whl/nightly/cu118"
@@ -1352,10 +1352,10 @@ parameters:
   # Nightly platform config
   torch-build:
     type: string
-    default: "2.1.0.dev20230419+cu118"
+    default: "2.1.0.dev20230605+cu118"
   torchvision-build:
     type: string
-    default: "0.16.0.dev20230419+cu118"
+    default: "0.16.0.dev20230605+cu118"
   torch-build-index:
     type: string
     default: "https://download.pytorch.org/whl/nightly/cu118"

CMakeLists.txt

@@ -2,8 +2,8 @@
 cmake_minimum_required(VERSION 3.17)
 project(Torch-TensorRT LANGUAGES CXX)
 
-# use c++14 like PyTorch
-set(CMAKE_CXX_STANDARD 14)
+# use c++17 like PyTorch
+set(CMAKE_CXX_STANDARD 17)
 
 # Build the libraries with -fPIC
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)

README.md

@@ -116,7 +116,7 @@ torch.jit.save(trt_ts_module, "trt_torchscript_module.ts") # save the TRT embedd
 These are the following dependencies used to verify the testcases. Torch-TensorRT can work with other versions, but the tests are not guaranteed to pass.
 
 - Bazel 5.2.0
-- Libtorch 2.1.0.dev20230419 (built with CUDA 11.8)
+- Libtorch 2.1.0.dev20230605 (built with CUDA 11.8)
 - CUDA 11.8
 - cuDNN 8.8.0
 - TensorRT 8.6.1

WORKSPACE

@@ -51,17 +51,17 @@ new_local_repository(
 http_archive(
     name = "libtorch",
     build_file = "@//third_party/libtorch:BUILD",
-    sha256 = "1a526a9cd19c1015674d26921dbb94bcd2d632a6f9c431a21c43f4e24768d834",
+    sha256 = "999becce82b73e566d0ffe010cd21fea8cf3a33f90f09dcc6b01150b820ae063",
     strip_prefix = "libtorch",
-    urls = ["https://download.pytorch.org/libtorch/nightly/cu118/libtorch-cxx11-abi-shared-with-deps-2.1.0.dev20230419%2Bcu118.zip"],
+    urls = ["https://download.pytorch.org/libtorch/nightly/cu118/libtorch-cxx11-abi-shared-with-deps-2.1.0.dev20230605%2Bcu118.zip"],
 )
 
 http_archive(
     name = "libtorch_pre_cxx11_abi",
     build_file = "@//third_party/libtorch:BUILD",
-    sha256 = "60c5912a5085a6a7073b3804b10d41d6cc054693bbeb7a45e0247050c2837bac",
+    sha256 = "786cc728c63ea69c40bd8fb535cf8e5e1dfff1d43eaad3eb5256b9ed89c1b268",
     strip_prefix = "libtorch",
-    urls = ["https://download.pytorch.org/libtorch/nightly/cu118/libtorch-shared-with-deps-2.1.0.dev20230419%2Bcu118.zip"],
+    urls = ["https://download.pytorch.org/libtorch/nightly/cu118/libtorch-shared-with-deps-2.1.0.dev20230605%2Bcu118.zip"],
 )
 
 # Download these tarballs manually from the NVIDIA website

py/requirements.txt

@@ -2,7 +2,7 @@ numpy
 packaging
 pybind11==2.6.2
 --extra-index-url https://download.pytorch.org/whl/nightly/cu118
-torch==2.1.0.dev20230419+cu118
-torchvision==0.16.0.dev20230419+cu118
+torch==2.1.0.dev20230605+cu118
+torchvision==0.16.0.dev20230605+cu118
 --extra-index-url https://pypi.ngc.nvidia.com
 tensorrt==8.6.1

py/setup.py

@@ -427,6 +427,8 @@ def run(self):
     ext_modules=ext_modules,
     install_requires=[
         "torch >=2.1.dev,<2.2" if not LEGACY else "torch >=1.13.0,<2.0",
+        "pyyaml",
+        "packaging",
     ],
     setup_requires=[],
     cmdclass={

py/torch_tensorrt/dynamo/backend/backends.py

@@ -12,6 +12,7 @@
     partition,
     get_submod_inputs,
 )
+from torch_tensorrt.dynamo.backend.utils import parse_dynamo_kwargs
 from torch_tensorrt.dynamo.backend.conversion import convert_module
 
 from torch._dynamo.backends.common import fake_tensor_unsupported
@@ -25,22 +26,20 @@
 @td.register_backend(name="torch_tensorrt")
 @fake_tensor_unsupported
 def torch_tensorrt_backend(
-    gm: torch.fx.GraphModule,
-    sample_inputs: Sequence[torch.Tensor],
-    settings: CompilationSettings = CompilationSettings(),
+    gm: torch.fx.GraphModule, sample_inputs: Sequence[torch.Tensor], **kwargs
 ):
     DEFAULT_BACKEND = aot_torch_tensorrt_aten_backend
 
-    return DEFAULT_BACKEND(gm, sample_inputs, settings=settings)
+    return DEFAULT_BACKEND(gm, sample_inputs, **kwargs)
 
 
 @td.register_backend(name="aot_torch_tensorrt_aten")
 @fake_tensor_unsupported
 def aot_torch_tensorrt_aten_backend(
-    gm: torch.fx.GraphModule,
-    sample_inputs: Sequence[torch.Tensor],
-    settings: CompilationSettings = CompilationSettings(),
+    gm: torch.fx.GraphModule, sample_inputs: Sequence[torch.Tensor], **kwargs
 ):
+    settings = parse_dynamo_kwargs(kwargs)
+
     custom_backend = partial(
         _pretraced_backend,
         settings=settings,

py/torch_tensorrt/dynamo/backend/conversion.py

@@ -24,11 +24,21 @@ def convert_module(
     Returns:
         TRTModule or TRTModuleNext
     """
+    # Specify module output data types to ensure TRT output types agree with
+    # that of the equivalent Torch module
+    module_outputs = module(*inputs)
+
+    if not isinstance(module_outputs, (list, tuple)):
+        module_outputs = [module_outputs]
+
+    output_dtypes = list(output.dtype for output in module_outputs)
+
     interpreter = TRTInterpreter(
         module,
         InputTensorSpec.from_tensors(inputs),
         explicit_batch_dimension=True,
         logger_level=(trt.Logger.VERBOSE if settings.debug else trt.Logger.WARNING),
+        output_dtypes=output_dtypes,
     )
 
     interpreter_result = interpreter.run(

py/torch_tensorrt/dynamo/backend/utils.py

@@ -1,9 +1,15 @@
 import torch
+import logging
+from dataclasses import replace, fields
 
+from torch_tensorrt.dynamo.backend._settings import CompilationSettings
 from typing import Any, Union, Sequence, Dict
 from torch_tensorrt import _Input, Device
 
 
+logger = logging.getLogger(__name__)
+
+
 def prepare_inputs(
     inputs: Union[_Input.Input, torch.Tensor, Sequence, Dict],
     device: torch.device = torch.device("cuda"),
@@ -66,3 +72,36 @@ def prepare_device(device: Union[Device, torch.device]) -> torch.device:
         )
 
     return device
+
+
+def parse_dynamo_kwargs(kwargs: Dict) -> CompilationSettings:
+    """Parses the kwargs field of a Dynamo backend
+
+    Args:
+        kwargs: Keyword arguments dictionary provided to the backend
+    Returns:
+        CompilationSettings object with relevant kwargs
+    """
+
+    # Initialize an empty CompilationSettings object
+    settings = CompilationSettings()
+
+    # If the user specifies keyword args, overwrite those fields in settings
+    # Validate all specified kwargs to ensure they are true fields of the dataclass
+    #
+    # Note: kwargs provided by torch.compile are wrapped in the "options" key
+    if kwargs:
+        if "options" in kwargs and len(kwargs) == 1:
+            kwargs = kwargs["options"]
+
+        valid_attrs = {attr.name for attr in fields(settings)}
+        valid_kwargs = {k: v for k, v in kwargs.items() if k in valid_attrs}
+        settings = replace(settings, **valid_kwargs)
+
+    # Enable debug/verbose mode if requested
+    if settings.debug:
+        logger.setLevel(logging.DEBUG)
+
+    logger.debug(f"Compiling with Settings:\n{settings}")
+
+    return settings