Skip to content

Commit 097037d

Browse files
authored
[5680954,5620660@2][ONNX][Autocast] Update value info in converted graph (NVIDIA#611)
## What does this PR do? **Type of change:** Bug fix **Overview:** Some tensor's precision weren't set in the ONNX graph (`value_info`) when converting the model with Autocast. This caused ONNX-Graphsurgeon to incorrectly interpret those precisions in `gs.import_onnx(ModelProto)`, which caused quantization of the converted model to fail. ## Usage ```python $ python -m modelopt.onnx.autocast --onnx_path=$MODEL_NAME.onnx --keep_io_types $ python -m modelopt.onnx.quantization --onnx_path=$MODEL_NAME.fp16.onnx --calibration_eps cpu ``` ## Testing See bug 5680954@12. ## Before your PR is "*Ready for review*" <!-- If you haven't finished some of the above items you can still open `Draft` PR. --> - **Make sure you read and follow [Contributor guidelines](https://github.com/NVIDIA/TensorRT-Model-Optimizer/blob/main/CONTRIBUTING.md)** and your commits are signed. - **Is this change backward compatible?**: Yes - **Did you write any new necessary tests?**: No - **Did you add or update any necessary documentation?**: No - **Did you update [Changelog](https://github.com/NVIDIA/TensorRT-Model-Optimizer/blob/main/CHANGELOG.rst)?**: No --------- Signed-off-by: gcunhase <[email protected]>
1 parent 7edf59c commit 097037d

File tree

2 files changed

+77
-0
lines changed

2 files changed

+77
-0
lines changed

modelopt/onnx/autocast/precisionconverter.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1114,6 +1114,13 @@ def _fix_network_output_names(self):
11141114
break
11151115
cast_node.input[0] = pre_cast_name
11161116
cast_node.output[0] = original_name
1117+
# Ensure correct output tensor type
1118+
cast_to_precision = next(
1119+
attr.i for attr in cast_node.attribute if attr.name == "to"
1120+
)
1121+
self.value_info_map[
1122+
cast_node.output[0]
1123+
].type.tensor_type.elem_type = cast_to_precision
11171124

11181125
modified = True
11191126
logger.debug(f"Fixed network output names: {post_cast_name} -> {output.name}")
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
import os
17+
18+
import onnx
19+
import onnx_graphsurgeon as gs
20+
import pytest
21+
import torch
22+
from _test_utils.onnx.lib_test_models import SimpleMLP, export_as_onnx
23+
24+
from modelopt.onnx.autocast import convert_to_mixed_precision
25+
from modelopt.onnx.quantization import quantize
26+
27+
28+
def assert_nodes_are_quantized(nodes):
29+
for node in nodes:
30+
for inp_idx, inp in enumerate(node.inputs):
31+
if isinstance(inp, gs.Variable):
32+
assert node.i(inp_idx).op == "DequantizeLinear", (
33+
f"Input '{inp.name}' of node '{node.name}' is not quantized but should be!"
34+
)
35+
return True
36+
37+
38+
@pytest.mark.parametrize("keep_io_types", [True, False])
39+
def test_autocast_quantize_int8(tmp_path, keep_io_types):
40+
model_torch = SimpleMLP()
41+
input_tensor = torch.randn(2, 16, 16)
42+
low_precision_type = "fp16"
43+
44+
onnx_path = os.path.join(tmp_path, "model.onnx")
45+
export_as_onnx(model_torch, input_tensor, onnx_filename=onnx_path)
46+
47+
# Convert model to low precision
48+
converted_model = convert_to_mixed_precision(
49+
onnx_path, keep_io_types=keep_io_types, low_precision_type=low_precision_type
50+
)
51+
converted_model_path = onnx_path.replace(
52+
".onnx", f".{low_precision_type}.{'keepIOTypes' if keep_io_types else ''}.onnx"
53+
)
54+
onnx.save(converted_model, converted_model_path)
55+
56+
# Quantize converted model
57+
quantize(converted_model_path, quantize_mode="int8", high_precision_dtype=low_precision_type)
58+
59+
# Output model should be produced in the same tmp_path
60+
output_onnx_path = converted_model_path.replace(".onnx", ".quant.onnx")
61+
62+
# Check that quantized explicit model is generated
63+
assert os.path.isfile(output_onnx_path)
64+
65+
# Load the output model and check QDQ node placements
66+
graph = gs.import_onnx(onnx.load(output_onnx_path))
67+
68+
# Check that all MatMul nodes are quantized
69+
mm_nodes = [n for n in graph.nodes if n.op == "MatMul"]
70+
assert assert_nodes_are_quantized(mm_nodes)

0 commit comments

Comments
 (0)