From d0ccf253bdbcfef509e2e707cec27a97482d5825 Mon Sep 17 00:00:00 2001 From: Dheeraj Peri Date: Wed, 23 Apr 2025 22:56:57 -0700 Subject: [PATCH 01/12] chore: bug fixes --- tests/py/dynamo/models/test_models_export.py | 2 +- tests/py/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/py/dynamo/models/test_models_export.py b/tests/py/dynamo/models/test_models_export.py index 19fdeaa9ab..1b82ba9ba6 100644 --- a/tests/py/dynamo/models/test_models_export.py +++ b/tests/py/dynamo/models/test_models_export.py @@ -290,7 +290,7 @@ def calibrate_loop(model): with torch.no_grad(): with export_torch_mode(): - exp_program = torch.export.export(model, (input_tensor,)) + exp_program = torch.export.export(model, (input_tensor,), strict=False) trt_model = torchtrt.dynamo.compile( exp_program, inputs=[input_tensor], diff --git a/tests/py/requirements.txt b/tests/py/requirements.txt index 011ed01e35..bea33bd5ae 100644 --- a/tests/py/requirements.txt +++ b/tests/py/requirements.txt @@ -10,5 +10,5 @@ pyyaml timm>=1.0.3 flashinfer-python; python_version < "3.13" transformers==4.49.0 -nvidia-modelopt[deploy,hf,torch]~=0.17.0; python_version < "3.13" +nvidia-modelopt[deploy,hf,torch]~=0.27.0; python_version < "3.13" --extra-index-url https://pypi.nvidia.com From fea44e45542a8474ae7c569c994a2150f41619b1 Mon Sep 17 00:00:00 2001 From: Dheeraj Peri Date: Wed, 23 Apr 2025 23:38:03 -0700 Subject: [PATCH 02/12] chore: revert modelopt change --- tests/py/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/py/requirements.txt b/tests/py/requirements.txt index bea33bd5ae..011ed01e35 100644 --- a/tests/py/requirements.txt +++ b/tests/py/requirements.txt @@ -10,5 +10,5 @@ pyyaml timm>=1.0.3 flashinfer-python; python_version < "3.13" transformers==4.49.0 -nvidia-modelopt[deploy,hf,torch]~=0.27.0; python_version < "3.13" +nvidia-modelopt[deploy,hf,torch]~=0.17.0; python_version < "3.13" --extra-index-url https://pypi.nvidia.com From adcbc8c7e431bbeba15b175e870fb206f77bd911 Mon Sep 17 00:00:00 2001 From: Dheeraj Peri Date: Thu, 24 Apr 2025 00:54:23 -0700 Subject: [PATCH 03/12] chore: updates --- tests/py/dynamo/models/test_models_export.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/py/dynamo/models/test_models_export.py b/tests/py/dynamo/models/test_models_export.py index 1b82ba9ba6..0c28b23bba 100644 --- a/tests/py/dynamo/models/test_models_export.py +++ b/tests/py/dynamo/models/test_models_export.py @@ -254,7 +254,6 @@ def calibrate_loop(model): @unittest.skipIf( platform.system() != "Linux" - or torch.cuda.get_device_capability() < (8, 9) or not importlib.util.find_spec("modelopt") or Version(metadata.version("nvidia-modelopt")) < Version("0.17.0"), "modelopt 0.17.0 or later is required, Int8 quantization is supported in modelopt since 0.17.0 or later for linux", From 0778ac4248bab5cb98740f929c3b89cb7956e1fa Mon Sep 17 00:00:00 2001 From: Dheeraj Peri Date: Thu, 24 Apr 2025 05:04:44 -0700 Subject: [PATCH 04/12] chore: update modelopt version --- tests/py/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/py/requirements.txt b/tests/py/requirements.txt index 011ed01e35..aab37183e7 100644 --- a/tests/py/requirements.txt +++ b/tests/py/requirements.txt @@ -10,5 +10,5 @@ pyyaml timm>=1.0.3 flashinfer-python; python_version < "3.13" transformers==4.49.0 -nvidia-modelopt[deploy,hf,torch]~=0.17.0; python_version < "3.13" +nvidia-modelopt~=0.27.0; python_version < "3.13" --extra-index-url https://pypi.nvidia.com From 51f4e07fc453bf269be33d6393daf33d30465a94 Mon Sep 17 00:00:00 2001 From: Dheeraj Peri Date: Thu, 24 Apr 2025 11:57:47 -0700 Subject: [PATCH 05/12] chore: revert modelopt to 0.17 --- tests/py/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/py/requirements.txt b/tests/py/requirements.txt index aab37183e7..f7fc568eb9 100644 --- a/tests/py/requirements.txt +++ b/tests/py/requirements.txt @@ -10,5 +10,5 @@ pyyaml timm>=1.0.3 flashinfer-python; python_version < "3.13" transformers==4.49.0 -nvidia-modelopt~=0.27.0; python_version < "3.13" +nvidia-modelopt~=0.17.0; python_version < "3.13" --extra-index-url https://pypi.nvidia.com From 01771d87b610ea8e98098981df2b0658453a88e8 Mon Sep 17 00:00:00 2001 From: Dheeraj Peri Date: Fri, 25 Apr 2025 12:41:04 -0700 Subject: [PATCH 06/12] chore: update modelopt version to 0.27 --- tests/py/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/py/requirements.txt b/tests/py/requirements.txt index f7fc568eb9..aab37183e7 100644 --- a/tests/py/requirements.txt +++ b/tests/py/requirements.txt @@ -10,5 +10,5 @@ pyyaml timm>=1.0.3 flashinfer-python; python_version < "3.13" transformers==4.49.0 -nvidia-modelopt~=0.17.0; python_version < "3.13" +nvidia-modelopt~=0.27.0; python_version < "3.13" --extra-index-url https://pypi.nvidia.com From 213921645ad634c79efe57ce4ac12fb86d446982 Mon Sep 17 00:00:00 2001 From: Dheeraj Peri Date: Fri, 25 Apr 2025 14:19:44 -0700 Subject: [PATCH 07/12] chore: updates --- tests/py/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/py/requirements.txt b/tests/py/requirements.txt index aab37183e7..c677a49155 100644 --- a/tests/py/requirements.txt +++ b/tests/py/requirements.txt @@ -10,5 +10,5 @@ pyyaml timm>=1.0.3 flashinfer-python; python_version < "3.13" transformers==4.49.0 -nvidia-modelopt~=0.27.0; python_version < "3.13" +nvidia-modelopt[all]~=0.27.0; python_version < "3.13" --extra-index-url https://pypi.nvidia.com From b1303a0b18083f351f2ab7a27a8e472cba3af9ee Mon Sep 17 00:00:00 2001 From: Dheeraj Peri Date: Sat, 26 Apr 2025 14:04:20 -0700 Subject: [PATCH 08/12] chore: remove modelopt version --- tests/py/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/py/requirements.txt b/tests/py/requirements.txt index c677a49155..6321824458 100644 --- a/tests/py/requirements.txt +++ b/tests/py/requirements.txt @@ -10,5 +10,5 @@ pyyaml timm>=1.0.3 flashinfer-python; python_version < "3.13" transformers==4.49.0 -nvidia-modelopt[all]~=0.27.0; python_version < "3.13" +nvidia-modelopt[all]; python_version < "3.13" --extra-index-url https://pypi.nvidia.com From 7502837e9f62476061a1ff6a8810973bf47cae13 Mon Sep 17 00:00:00 2001 From: Dheeraj Peri Date: Mon, 28 Apr 2025 11:49:03 -0700 Subject: [PATCH 09/12] chore: restrict modelopt installation to certain tests --- .github/workflows/build-test-linux.yml | 2 ++ tests/py/requirements.txt | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test-linux.yml b/.github/workflows/build-test-linux.yml index d0fabf9993..814061900f 100644 --- a/.github/workflows/build-test-linux.yml +++ b/.github/workflows/build-test-linux.yml @@ -138,6 +138,7 @@ jobs: pushd . cd tests/py python -m pip install -r requirements.txt + python -m pip install nvidia-modelopt[all]; python_version < "3.13" --extra-index-url https://pypi.nvidia.com cd dynamo python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml -n 4 conversion/ python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml automatic_plugin/test_automatic_plugin.py @@ -172,6 +173,7 @@ jobs: pushd . cd tests/py python -m pip install -r requirements.txt + python -m pip install nvidia-modelopt[all]; python_version < "3.13" --extra-index-url https://pypi.nvidia.com cd dynamo python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dyn_models_export.xml --ir dynamo models/ popd diff --git a/tests/py/requirements.txt b/tests/py/requirements.txt index 6321824458..29a6dfb382 100644 --- a/tests/py/requirements.txt +++ b/tests/py/requirements.txt @@ -10,5 +10,4 @@ pyyaml timm>=1.0.3 flashinfer-python; python_version < "3.13" transformers==4.49.0 -nvidia-modelopt[all]; python_version < "3.13" --extra-index-url https://pypi.nvidia.com From a893fdc0b4522db6dd479228d80ebb2d7b8b6fb9 Mon Sep 17 00:00:00 2001 From: Dheeraj Peri Date: Mon, 28 Apr 2025 12:35:58 -0700 Subject: [PATCH 10/12] chore: updates --- .github/workflows/build-test-linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test-linux.yml b/.github/workflows/build-test-linux.yml index 814061900f..a8e90a3222 100644 --- a/.github/workflows/build-test-linux.yml +++ b/.github/workflows/build-test-linux.yml @@ -138,7 +138,7 @@ jobs: pushd . cd tests/py python -m pip install -r requirements.txt - python -m pip install nvidia-modelopt[all]; python_version < "3.13" --extra-index-url https://pypi.nvidia.com + python -m pip install nvidia-modelopt[all] --extra-index-url https://pypi.nvidia.com cd dynamo python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml -n 4 conversion/ python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml automatic_plugin/test_automatic_plugin.py @@ -173,7 +173,7 @@ jobs: pushd . cd tests/py python -m pip install -r requirements.txt - python -m pip install nvidia-modelopt[all]; python_version < "3.13" --extra-index-url https://pypi.nvidia.com + python -m pip install nvidia-modelopt[all] --extra-index-url https://pypi.nvidia.com cd dynamo python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dyn_models_export.xml --ir dynamo models/ popd From cd99f5884dd0bf327ce00273847d98d70e1c0dad Mon Sep 17 00:00:00 2001 From: Dheeraj Peri Date: Mon, 28 Apr 2025 14:39:07 -0700 Subject: [PATCH 11/12] chore: updates --- tests/py/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/py/requirements.txt b/tests/py/requirements.txt index 29a6dfb382..f247fa5696 100644 --- a/tests/py/requirements.txt +++ b/tests/py/requirements.txt @@ -10,4 +10,3 @@ pyyaml timm>=1.0.3 flashinfer-python; python_version < "3.13" transformers==4.49.0 ---extra-index-url https://pypi.nvidia.com From 8fbf8f855008723d8e7929c662907f4db79de88a Mon Sep 17 00:00:00 2001 From: Dheeraj Peri Date: Tue, 29 Apr 2025 09:54:09 -0700 Subject: [PATCH 12/12] chore: minor updates related to logging --- .../dynamo/conversion/_TRTInterpreter.py | 2 +- .../dynamo/conversion/converter_utils.py | 63 ++++++++++--------- .../dynamo/backend/test_backend_compiler.py | 8 +-- tests/py/dynamo/conversion/harness.py | 2 - tests/py/dynamo/models/test_dtype_support.py | 1 - 5 files changed, 35 insertions(+), 41 deletions(-) diff --git a/py/torch_tensorrt/dynamo/conversion/_TRTInterpreter.py b/py/torch_tensorrt/dynamo/conversion/_TRTInterpreter.py index a3444c025d..da5f3b36c9 100644 --- a/py/torch_tensorrt/dynamo/conversion/_TRTInterpreter.py +++ b/py/torch_tensorrt/dynamo/conversion/_TRTInterpreter.py @@ -893,7 +893,7 @@ def get_attr(self, target: str, args: Any, kwargs: Any) -> np.ndarray: else: constant_tensor = frozen_attr - return to_torch(constant_tensor) + return to_torch(constant_tensor) def call_method(self, target: str, args: Any, kwargs: Any) -> Any: assert isinstance(target, str) diff --git a/py/torch_tensorrt/dynamo/conversion/converter_utils.py b/py/torch_tensorrt/dynamo/conversion/converter_utils.py index 0808db2ed8..685f40b254 100644 --- a/py/torch_tensorrt/dynamo/conversion/converter_utils.py +++ b/py/torch_tensorrt/dynamo/conversion/converter_utils.py @@ -590,42 +590,45 @@ def to_numpy( Returns: A Numpy array or None, if the input was None. """ - output = None + with unset_fake_temporarily(): + output = None - if value is None or isinstance(value, np.ndarray): - output = value + if value is None or isinstance(value, np.ndarray): + output = value - elif isinstance(value, torch.Tensor): - if value.is_quantized: - value = value.dequantize() - elif value.dtype == torch.bfloat16: - # TODO: Remove when numpy has a BF16 type - _LOGGER.warning( - "Requested a conversion of bfloat16 tensor from torch to numpy which isn't supported. Casting this tensor to FP32 precision currently. Please use to_torch() API for better data representation", - ) - value = value.to(torch.float) - - output = value.cpu().detach().contiguous().numpy() + elif isinstance(value, torch.Tensor): + if value.is_quantized: + value = value.dequantize() + elif value.dtype == torch.bfloat16: + # TODO: Remove when numpy has a BF16 type + _LOGGER.warning( + "Requested a conversion of bfloat16 tensor from torch to numpy which isn't supported. Casting this tensor to FP32 precision currently. Please use to_torch() API for better data representation", + ) + value = value.to(torch.float) - elif isinstance(value, int): - output = np.array([value], dtype=np.int32) + output = value.cpu().detach().contiguous().numpy() - elif isinstance(value, float): - output = np.array([value], dtype=np.float32) + elif isinstance(value, int): + output = np.array([value], dtype=np.int32) - elif isinstance(value, bool): - output = np.array([value], dtype=np.bool_) + elif isinstance(value, float): + output = np.array([value], dtype=np.float32) - if isinstance(output, np.ndarray) or output is None: - return ( - output - if (dtype is None or output is None) - else output.astype(_enums.dtype._from(dtype).to(np.dtype, use_default=True)) - ) - else: - raise AssertionError( - f"to_numpy can only be called on None, bool, int, float, np.ndarray, or torch.Tensor, got: {value}" - ) + elif isinstance(value, bool): + output = np.array([value], dtype=np.bool_) + + if isinstance(output, np.ndarray) or output is None: + return ( + output + if (dtype is None or output is None) + else output.astype( + _enums.dtype._from(dtype).to(np.dtype, use_default=True) + ) + ) + else: + raise AssertionError( + f"to_numpy can only be called on None, bool, int, float, np.ndarray, or torch.Tensor, got: {value}" + ) def to_torch( diff --git a/tests/py/dynamo/backend/test_backend_compiler.py b/tests/py/dynamo/backend/test_backend_compiler.py index 4c65800f05..6369d3805c 100644 --- a/tests/py/dynamo/backend/test_backend_compiler.py +++ b/tests/py/dynamo/backend/test_backend_compiler.py @@ -2,11 +2,10 @@ from copy import deepcopy import torch +import torch_tensorrt from torch.testing._internal.common_utils import TestCase, run_tests from torch_tensorrt.dynamo.partitioning import fast_partition -import torch_tensorrt - from ..testing_utilities import DECIMALS_OF_AGREEMENT, lower_graph_testing @@ -51,7 +50,6 @@ def forward(self, x, y): pass_through_build_failures=True, torch_executed_ops={"torch.ops.aten.add.Tensor"}, use_python_runtime=False, - debug=True, ) optimized_model_results = optimized_model(*inputs).detach().cpu() torch_model_results = fx_graph(*inputs).detach().cpu() @@ -132,7 +130,6 @@ def forward(self, x, y): pass_through_build_failures=True, torch_executed_ops={"torch.ops.aten.add.Tensor"}, use_python_runtime=False, - debug=True, ) optimized_model_results = optimized_model(*inputs).detach().cpu() torch_model_results = model(*inputs).detach().cpu() @@ -177,7 +174,6 @@ def forward(self, x, y): optimization_level=4, version_compatible=True, max_aux_streams=5, - debug=True, ) optimized_model_results = optimized_model(*inputs).detach().cpu() torch_model_results = fx_graph(*inputs).detach().cpu() @@ -225,7 +221,6 @@ def forward(self, x, y): min_block_size=1, pass_through_build_failures=True, truncate_double=True, - debug=True, ) optimized_model_results = optimized_model(*inputs).detach().cpu() torch_model_results = fx_graph(*inputs).detach().cpu() @@ -298,7 +293,6 @@ def forward(self, x, y): min_block_size=1, pass_through_build_failures=True, truncate_double=False, - debug=True, torch_executed_ops={"torch.ops.aten.add.Tensor"}, ) optimized_model_results = optimized_model(*inputs).detach().cpu() diff --git a/tests/py/dynamo/conversion/harness.py b/tests/py/dynamo/conversion/harness.py index 6ff45507a0..aa22a74fc0 100644 --- a/tests/py/dynamo/conversion/harness.py +++ b/tests/py/dynamo/conversion/harness.py @@ -415,7 +415,6 @@ def run_test( compilation_settings = CompilationSettings( enabled_precisions={dtype._from(precision)}, truncate_double=True, - debug=True, immutable_weights=immutable_weights, ) @@ -507,7 +506,6 @@ def run_test_compare_tensor_attributes_only( compilation_settings = CompilationSettings( enabled_precisions={dtype._from(precision)}, truncate_double=True, - debug=True, immutable_weights=immutable_weights, ) diff --git a/tests/py/dynamo/models/test_dtype_support.py b/tests/py/dynamo/models/test_dtype_support.py index 146f7fdb7d..37b40574a1 100644 --- a/tests/py/dynamo/models/test_dtype_support.py +++ b/tests/py/dynamo/models/test_dtype_support.py @@ -297,7 +297,6 @@ def forward(self, x): ir="torch_compile", inputs=inputs, enabled_precisions={torch.bfloat16}, - debug=True, min_block_size=1, device=device, cache_built_engines=False,