Precision loss in TensorRT inference

[Huntersdeng]

Env

• NVIDIA GeForce RTX 4060 Ti
• cuda 12.6
• tensorrt 10.13.0.35
• torch 2.7.0+cu126
• transformers 5.0.0rc1

Description

I find there are some gaps between the inference results from onnx model and tensorrt model.
So I modify the onnxexport.py to export some middle layers' output:

import torch
from pathlib import Path
from transformers.models.sam3 import Sam3Processor, Sam3Model
from transformers.models.sam3.modeling_sam3 import (
    inverse_sigmoid,
    box_cxcywh_to_xyxy,
    Sam3ImageSegmentationOutput,
)
from PIL import Image
import requests

device = "cpu"  # for onnx export we use CPU for maximum compatibility

# 1. Load model & processor
model = Sam3Model.from_pretrained("/home/zy/weights/sam3").to(device)
processor = Sam3Processor.from_pretrained("/home/zy/weights/sam3")

model.eval()

prompt = "dog"

# 2. Build a sample batch (same as your example)
image_url = "http://images.cocodataset.org/val2017/000000077595.jpg"
image = Image.open(requests.get(image_url, stream=True).raw).convert("RGB")

inputs = processor(images=image, text=prompt, return_tensors="pt").to(device)

pixel_values = inputs["pixel_values"]
input_ids = inputs["input_ids"]
attention_mask = inputs["attention_mask"]
print(input_ids.shape)

# 3. Wrap Sam3Model so the ONNX graph has clean inputs/outputs
# class Sam3ONNXWrapper(torch.nn.Module):
#     def __init__(self, sam3, input_ids, attention_mask):
#         super().__init__()
#         self.sam3 = sam3
#         self.register_buffer("const_input_ids", input_ids.to(torch.int64).cpu())
#         self.register_buffer("const_attention_mask", attention_mask.to(torch.int64).cpu())

#     def forward(self, pixel_values):
#         outputs = self.sam3(
#             pixel_values=pixel_values,
#             input_ids=self.const_input_ids,
#             attention_mask=self.const_attention_mask,
#         )
#         print(outputs.keys())
#         # Typical useful outputs
#         pred_boxes = outputs.pred_boxes
#         pred_logits = outputs.pred_logits
#         pred_masks = outputs.pred_masks
#         presence_score = outputs.presence_logits
#         return (pred_boxes, pred_logits, pred_masks, presence_score)


class Sam3ONNXWrapper(torch.nn.Module):
    def __init__(self, sam3, input_ids, attention_mask):
        super().__init__()
        self.sam3 = sam3
        self.register_buffer("const_input_ids", input_ids.to(torch.int64).cpu())
        self.register_buffer(
            "const_attention_mask", attention_mask.to(torch.int64).cpu()
        )

    def forward(self, pixel_values):
        vision_outputs = self.sam3.vision_encoder(pixel_values)
        fpn_hidden_states = vision_outputs.fpn_hidden_states[:-1]
        fpn_position_encoding = vision_outputs.fpn_position_encoding[:-1]
        text_features = self.sam3.get_text_features(
            input_ids=self.const_input_ids, attention_mask=self.const_attention_mask
        )
        text_mask = attention_mask.bool() if attention_mask is not None else None
        combined_prompt_features = text_features
        combined_prompt_mask = text_mask
        encoder_outputs = self.sam3.detr_encoder(
            vision_features=[fpn_hidden_states[-1]],
            text_features=combined_prompt_features,
            vision_pos_embeds=[fpn_position_encoding[-1]],
            text_mask=combined_prompt_mask,
        )
        decoder_outputs = self.sam3.detr_decoder(
            vision_features=encoder_outputs.last_hidden_state,
            text_features=encoder_outputs.text_features,
            vision_pos_encoding=encoder_outputs.pos_embeds_flattened,
            text_mask=combined_prompt_mask,
            spatial_shapes=encoder_outputs.spatial_shapes,
        )

        all_box_offsets = self.sam3.detr_decoder.box_head(
            decoder_outputs.intermediate_hidden_states
        )
        reference_boxes_inv_sig = inverse_sigmoid(decoder_outputs.reference_boxes)
        all_pred_boxes_cxcywh = (reference_boxes_inv_sig + all_box_offsets).sigmoid()
        all_pred_boxes = box_cxcywh_to_xyxy(all_pred_boxes_cxcywh)

        all_pred_logits = self.sam3.dot_product_scoring(
            decoder_hidden_states=decoder_outputs.intermediate_hidden_states,
            text_features=encoder_outputs.text_features,
            text_mask=combined_prompt_mask,
        ).squeeze(-1)

        pred_logits = all_pred_logits[-1]
        pred_boxes = all_pred_boxes[-1]
        decoder_hidden_states = decoder_outputs.intermediate_hidden_states[-1]
        presence_logits = decoder_outputs.presence_logits[-1]

        mask_outputs = self.sam3.mask_decoder(
            decoder_queries=decoder_hidden_states,
            backbone_features=list(fpn_hidden_states),
            encoder_hidden_states=encoder_outputs.last_hidden_state,
            prompt_features=combined_prompt_features,
            prompt_mask=combined_prompt_mask,
        )

        pred_masks = mask_outputs.pred_masks
        semantic_seg = mask_outputs.semantic_seg

        vision_feature_0 = fpn_hidden_states[0]
        vision_feature_1 = fpn_hidden_states[1]
        vision_feature_2 = fpn_hidden_states[2]

        return (
            vision_feature_0,
            vision_feature_1,
            vision_feature_2,
            text_features,
            encoder_outputs.last_hidden_state,
            decoder_hidden_states,
            pred_boxes,
            pred_logits,
            presence_logits,
            pred_masks,
            semantic_seg,
        )


wrapper = Sam3ONNXWrapper(model, input_ids, attention_mask).to(device).eval()
# wrapper(pixel_values)/

# 5. Export to ONNX
output_dir = Path(f"onnx_weights_dbg")
output_dir.mkdir(exist_ok=True)
onnx_path = str(output_dir / f"sam3_dbg.onnx")
# output_names = ["pred_boxes", "pred_logits", "pred_masks", "presence_score"]
output_names = [
    "vision_feature_0",
    "vision_feature_1",
    "vision_feature_2",
    "text_features",
    "encoder_hidden_state",
    "decoder_hidden_states",
    "pred_boxes",
    "pred_logits",
    "presence_logits",
    "pred_masks",
    "semantic_seg",
]


torch.onnx.export(
    wrapper,
    (pixel_values),
    onnx_path,
    input_names=["pixel_values"],
    output_names=output_names,
    dynamo=False,
    opset_version=17,
)

print(f"Exported to {onnx_path}")

And I use polygraphy to check. Here's the result:

[I] TF32 is disabled by default. Turn on TF32 for better performance with minor accuracy differences.
[I] trt-runner-N0-01/05/26-15:07:18     | Activating and starting inference
[I] Configuring with profiles:[
        Profile 0:
            {pixel_values [min=[1, 3, 1008, 1008], opt=[1, 3, 1008, 1008], max=[1, 3, 1008, 1008]]}
    ]
[W] profileSharing0806 is on by default in TensorRT 10.0. This flag is deprecated and has no effect.
[I] Building engine with configuration:
    Flags                  | []
    Engine Capability      | EngineCapability.STANDARD
    Memory Pools           | [WORKSPACE: 15936.19 MiB, TACTIC_DRAM: 15936.19 MiB, TACTIC_SHARED_MEMORY: 1024.00 MiB]
    Tactic Sources         | [EDGE_MASK_CONVOLUTIONS, JIT_CONVOLUTIONS]
    Profiling Verbosity    | ProfilingVerbosity.DETAILED
    Preview Features       | [PROFILE_SHARING_0806]
[I] Finished engine building in 58.691 seconds
[I] trt-runner-N0-01/05/26-15:07:18    
    ---- Inference Input(s) ----
    {pixel_values [dtype=float32, shape=(1, 3, 1008, 1008)]}
[I] trt-runner-N0-01/05/26-15:07:18    
    ---- Inference Output(s) ----
    {vision_feature_0 [dtype=float32, shape=(1, 256, 288, 288)],
     vision_feature_1 [dtype=float32, shape=(1, 256, 144, 144)],
     vision_feature_2 [dtype=float32, shape=(1, 256, 72, 72)],
     text_features [dtype=float32, shape=(1, 32, 256)],
     encoder_hidden_state [dtype=float32, shape=(1, 5184, 256)],
     decoder_hidden_states [dtype=float32, shape=(1, 200, 256)],
     pred_boxes [dtype=float32, shape=(1, 200, 4)],
     pred_logits [dtype=float32, shape=(1, 200)],
     presence_logits [dtype=float32, shape=(1, 1)],
     pred_masks [dtype=float32, shape=(1, 200, 288, 288)],
     semantic_seg [dtype=float32, shape=(1, 1, 288, 288)]}
[I] trt-runner-N0-01/05/26-15:07:18     | Completed 1 iteration(s) in 1306 ms | Average inference time: 1306 ms.
[I] onnxrt-runner-N0-01/05/26-15:07:18  | Activating and starting inference
[I] Creating ONNX-Runtime Inference Session with providers: ['CPUExecutionProvider']
2026-01-05 15:08:22.030170384 [W:onnxruntime:, graph.cc:120 MergeShapeInfo] Error merging shape info for output. '/vision_encoder/backbone/embeddings/Concat_output_0' source:{4} target:{5}. Falling back to lenient merge.
[I] onnxrt-runner-N0-01/05/26-15:07:18 
    ---- Inference Input(s) ----
    {pixel_values [dtype=float32, shape=(1, 3, 1008, 1008)]}
[I] onnxrt-runner-N0-01/05/26-15:07:18 
    ---- Inference Output(s) ----
    {vision_feature_0 [dtype=float32, shape=(1, 256, 288, 288)],
     vision_feature_1 [dtype=float32, shape=(1, 256, 144, 144)],
     vision_feature_2 [dtype=float32, shape=(1, 256, 72, 72)],
     text_features [dtype=float32, shape=(1, 32, 256)],
     encoder_hidden_state [dtype=float32, shape=(1, 5184, 256)],
     decoder_hidden_states [dtype=float32, shape=(1, 200, 256)],
     pred_boxes [dtype=float32, shape=(1, 200, 4)],
     pred_logits [dtype=float32, shape=(1, 200)],
     presence_logits [dtype=float32, shape=(1, 1)],
     pred_masks [dtype=float32, shape=(1, 200, 288, 288)],
     semantic_seg [dtype=float32, shape=(1, 1, 288, 288)]}
[I] onnxrt-runner-N0-01/05/26-15:07:18  | Completed 1 iteration(s) in 8450 ms | Average inference time: 8450 ms.
[I] Accuracy Comparison | trt-runner-N0-01/05/26-15:07:18 vs. onnxrt-runner-N0-01/05/26-15:07:18
[I]     Comparing Output: 'vision_feature_0' (dtype=float32, shape=(1, 256, 288, 288)) with 'vision_feature_0' (dtype=float32, shape=(1, 256, 288, 288))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[W]         onnxrt-runner-N0-01/05/26-15:07:18  | Output: vision_feature_0: Some values are 0. Will add a small epsilon quantity to these when computing relative difference. Note that this may cause some relative differences to be extremely high. 
[I]         trt-runner-N0-01/05/26-15:07:18: vision_feature_0 | Stats: mean=-0.01514, std-dev=0.016879, var=0.00028488, median=-0.01227, min=-0.14353 at (0, 17, 285, 11), max=0.07911 at (0, 111, 281, 214), avg-magnitude=0.017047, p90=0.0032417, p95=0.0072221, p99=0.015082
[I]             ---- Histogram ----
                Bin Range            |  Num Elems | Visualization
                (-0.144  , -0.121  ) |         47 | 
                (-0.121  , -0.099  ) |       5634 | 
                (-0.099  , -0.0767 ) |     119782 | 
                (-0.0767 , -0.0545 ) |     408355 | #
                (-0.0545 , -0.0322 ) |    2606431 | ###########
                (-0.0322 , -0.00995) |    8920261 | ########################################
                (-0.00995, 0.0123  ) |    8790913 | #######################################
                (0.0123  , 0.0346  ) |     378221 | #
                (0.0346  , 0.0568  ) |       3978 | 
                (0.0568  , 0.0791  ) |         42 | 
[I]         onnxrt-runner-N0-01/05/26-15:07:18: vision_feature_0 | Stats: mean=-0.01514, std-dev=0.016879, var=0.00028488, median=-0.01227, min=-0.14353 at (0, 17, 285, 11), max=0.07911 at (0, 111, 281, 214), avg-magnitude=0.017047, p90=0.0032417, p95=0.007222, p99=0.015082
[I]             ---- Histogram ----
                Bin Range            |  Num Elems | Visualization
                (-0.144  , -0.121  ) |         47 | 
                (-0.121  , -0.099  ) |       5634 | 
                (-0.099  , -0.0767 ) |     119783 | 
                (-0.0767 , -0.0545 ) |     408353 | #
                (-0.0545 , -0.0322 ) |    2606436 | ###########
                (-0.0322 , -0.00995) |    8920265 | ########################################
                (-0.00995, 0.0123  ) |    8790908 | #######################################
                (0.0123  , 0.0346  ) |     378218 | #
                (0.0346  , 0.0568  ) |       3978 | 
                (0.0568  , 0.0791  ) |         42 | 
[I]         Error Metrics: vision_feature_0
[I]             Minimum Required Tolerance: elemwise error | [abs=6.754e-06] OR [rel=1.0737e+09] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=5.7006e-08, std-dev=6.4222e-08, var=4.1245e-15, median=4.2375e-08, min=0 at (0, 0, 0, 8), max=6.754e-06 at (0, 211, 257, 30), avg-magnitude=5.7006e-08, p90=1.1921e-07, p95=1.546e-07, p99=2.645e-07
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (0       , 6.75e-07) |   21216916 | ########################################
                    (6.75e-07, 1.35e-06) |      13286 | 
                    (1.35e-06, 2.03e-06) |       2264 | 
                    (2.03e-06, 2.7e-06 ) |        702 | 
                    (2.7e-06 , 3.38e-06) |        299 | 
                    (3.38e-06, 4.05e-06) |        107 | 
                    (4.05e-06, 4.73e-06) |         60 | 
                    (4.73e-06, 5.4e-06 ) |         22 | 
                    (5.4e-06 , 6.08e-06) |          6 | 
                    (6.08e-06, 6.75e-06) |          2 | 
[I]             Relative Difference | Stats: mean=54.914, std-dev=2.3388e+05, var=5.4698e+10, median=3.2489e-06, min=0 at (0, 0, 0, 8), max=1.0737e+09 at (0, 251, 32, 46), avg-magnitude=54.914, p90=2.2915e-05, p95=4.6669e-05, p99=0.00023475
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (0       , 1.07e+08) |   21233663 | ########################################
                    (1.07e+08, 2.15e+08) |          0 | 
                    (2.15e+08, 3.22e+08) |          0 | 
                    (3.22e+08, 4.29e+08) |          0 | 
                    (4.29e+08, 5.37e+08) |          0 | 
                    (5.37e+08, 6.44e+08) |          0 | 
                    (6.44e+08, 7.52e+08) |          0 | 
                    (7.52e+08, 8.59e+08) |          0 | 
                    (8.59e+08, 9.66e+08) |          0 | 
                    (9.66e+08, 1.07e+09) |          1 | 
[E]         FAILED | Output: 'vision_feature_0' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'vision_feature_1' (dtype=float32, shape=(1, 256, 144, 144)) with 'vision_feature_1' (dtype=float32, shape=(1, 256, 144, 144))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:07:18: vision_feature_1 | Stats: mean=0.0019535, std-dev=0.99889, var=0.99777, median=-0.002932, min=-5.9977 at (0, 139, 51, 57), max=6.2735 at (0, 44, 20, 101), avg-magnitude=0.7847, p90=1.2633, p95=1.6437, p99=2.3871
[I]             ---- Histogram ----
                Bin Range      |  Num Elems | Visualization
                (-6   , -4.77) |        148 | 
                (-4.77, -3.54) |       4602 | 
                (-3.54, -2.32) |      60173 | #
                (-2.32, -1.09) |     623058 | ##########
                (-1.09, 0.138) |    2276178 | ########################################
                (0.138, 1.37 ) |    1898847 | #################################
                (1.37 , 2.59 ) |     412856 | #######
                (2.59 , 3.82 ) |      30715 | 
                (3.82 , 5.05 ) |       1750 | 
                (5.05 , 6.27 ) |         89 | 
[I]         onnxrt-runner-N0-01/05/26-15:07:18: vision_feature_1 | Stats: mean=0.0019535, std-dev=0.99889, var=0.99778, median=-0.0029335, min=-5.9977 at (0, 139, 51, 57), max=6.2735 at (0, 44, 20, 101), avg-magnitude=0.7847, p90=1.2633, p95=1.6437, p99=2.3871
[I]             ---- Histogram ----
                Bin Range      |  Num Elems | Visualization
                (-6   , -4.77) |        148 | 
                (-4.77, -3.54) |       4602 | 
                (-3.54, -2.32) |      60174 | #
                (-2.32, -1.09) |     623058 | ##########
                (-1.09, 0.138) |    2276180 | ########################################
                (0.138, 1.37 ) |    1898843 | #################################
                (1.37 , 2.59 ) |     412857 | #######
                (2.59 , 3.82 ) |      30715 | 
                (3.82 , 5.05 ) |       1750 | 
                (5.05 , 6.27 ) |         89 | 
[I]         Error Metrics: vision_feature_1
[I]             Minimum Required Tolerance: elemwise error | [abs=0.00055704] OR [rel=78.247] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=5.5588e-06, std-dev=5.9869e-06, var=3.5843e-11, median=4.1723e-06, min=0 at (0, 0, 0, 78), max=0.00055704 at (0, 191, 114, 120), avg-magnitude=5.5588e-06, p90=1.1563e-05, p95=1.4782e-05, p99=2.4557e-05
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (0       , 5.57e-05) |    5302571 | ########################################
                    (5.57e-05, 0.000111) |       4649 | 
                    (0.000111, 0.000167) |        811 | 
                    (0.000167, 0.000223) |        260 | 
                    (0.000223, 0.000279) |         92 | 
                    (0.000279, 0.000334) |         23 | 
                    (0.000334, 0.00039 ) |          4 | 
                    (0.00039 , 0.000446) |          4 | 
                    (0.000446, 0.000501) |          1 | 
                    (0.000501, 0.000557) |          1 | 
[I]             Relative Difference | Stats: mean=9.6089e-05, std-dev=0.045692, var=0.0020878, median=6.5735e-06, min=0 at (0, 0, 0, 78), max=78.247 at (0, 205, 50, 50), avg-magnitude=9.6089e-05, p90=4.3977e-05, p95=8.9272e-05, p99=0.00044842
[I]                 ---- Histogram ----
                    Bin Range    |  Num Elems | Visualization
                    (0   , 7.82) |    5308410 | ########################################
                    (7.82, 15.6) |          4 | 
                    (15.6, 23.5) |          0 | 
                    (23.5, 31.3) |          0 | 
                    (31.3, 39.1) |          0 | 
                    (39.1, 46.9) |          0 | 
                    (46.9, 54.8) |          0 | 
                    (54.8, 62.6) |          0 | 
                    (62.6, 70.4) |          1 | 
                    (70.4, 78.2) |          1 | 
[E]         FAILED | Output: 'vision_feature_1' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'vision_feature_2' (dtype=float32, shape=(1, 256, 72, 72)) with 'vision_feature_2' (dtype=float32, shape=(1, 256, 72, 72))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:07:18: vision_feature_2 | Stats: mean=-0.02647, std-dev=0.68585, var=0.47039, median=-0.018099, min=-3.9931 at (0, 43, 66, 57), max=3.8522 at (0, 185, 17, 3), avg-magnitude=0.53415, p90=0.82253, p95=1.0764, p99=1.5906
[I]             ---- Histogram ----
                Bin Range          |  Num Elems | Visualization
                (-3.99  , -3.21  ) |        176 | 
                (-3.21  , -2.42  ) |       2074 | 
                (-2.42  , -1.64  ) |      16797 | #
                (-1.64  , -0.855 ) |     120900 | #########
                (-0.855 , -0.0704) |     480100 | ###################################
                (-0.0704, 0.714  ) |     533487 | ########################################
                (0.714  , 1.5    ) |     155587 | ###########
                (1.5    , 2.28   ) |      16819 | #
                (2.28   , 3.07   ) |       1131 | 
                (3.07   , 3.85   ) |         33 | 
[I]         onnxrt-runner-N0-01/05/26-15:07:18: vision_feature_2 | Stats: mean=-0.02647, std-dev=0.68585, var=0.47039, median=-0.018098, min=-3.9931 at (0, 43, 66, 57), max=3.8522 at (0, 185, 17, 3), avg-magnitude=0.53415, p90=0.82253, p95=1.0764, p99=1.5906
[I]             ---- Histogram ----
                Bin Range          |  Num Elems | Visualization
                (-3.99  , -3.21  ) |        176 | 
                (-3.21  , -2.42  ) |       2074 | 
                (-2.42  , -1.64  ) |      16797 | #
                (-1.64  , -0.855 ) |     120900 | #########
                (-0.855 , -0.0704) |     480098 | ###################################
                (-0.0704, 0.714  ) |     533488 | ########################################
                (0.714  , 1.5    ) |     155588 | ###########
                (1.5    , 2.28   ) |      16819 | #
                (2.28   , 3.07   ) |       1131 | 
                (3.07   , 3.85   ) |         33 | 
[I]         Error Metrics: vision_feature_2
[I]             Minimum Required Tolerance: elemwise error | [abs=0.00030649] OR [rel=18.878] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=4.0281e-06, std-dev=4.2446e-06, var=1.8017e-11, median=3.0398e-06, min=0 at (0, 0, 0, 40), max=0.00030649 at (0, 175, 64, 7), avg-magnitude=4.0281e-06, p90=8.4341e-06, p95=1.0818e-05, p99=1.7703e-05
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (0       , 3.06e-05) |    1324953 | ########################################
                    (3.06e-05, 6.13e-05) |       1739 | 
                    (6.13e-05, 9.19e-05) |        218 | 
                    (9.19e-05, 0.000123) |         97 | 
                    (0.000123, 0.000153) |         46 | 
                    (0.000153, 0.000184) |         27 | 
                    (0.000184, 0.000215) |         13 | 
                    (0.000215, 0.000245) |          6 | 
                    (0.000245, 0.000276) |          3 | 
                    (0.000276, 0.000306) |          2 | 
[I]             Relative Difference | Stats: mean=9.2484e-05, std-dev=0.018245, var=0.00033289, median=7.1053e-06, min=0 at (0, 0, 0, 40), max=18.878 at (0, 58, 67, 40), avg-magnitude=9.2484e-05, p90=4.8423e-05, p95=9.8349e-05, p99=0.00049433
[I]                 ---- Histogram ----
                    Bin Range    |  Num Elems | Visualization
                    (0   , 1.89) |    1327099 | ########################################
                    (1.89, 3.78) |          3 | 
                    (3.78, 5.66) |          0 | 
                    (5.66, 7.55) |          1 | 
                    (7.55, 9.44) |          0 | 
                    (9.44, 11.3) |          0 | 
                    (11.3, 13.2) |          0 | 
                    (13.2, 15.1) |          0 | 
                    (15.1, 17  ) |          0 | 
                    (17  , 18.9) |          1 | 
[E]         FAILED | Output: 'vision_feature_2' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'text_features' (dtype=float32, shape=(1, 32, 256)) with 'text_features' (dtype=float32, shape=(1, 32, 256))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:07:18: text_features | Stats: mean=-0.0045255, std-dev=0.69494, var=0.48294, median=0.0011168, min=-4.3921 at (0, 6, 54), max=4.1521 at (0, 1, 112), avg-magnitude=0.39579, p90=0.57612, p95=1.1946, p99=2.154
[I]             ---- Histogram ----
                Bin Range        |  Num Elems | Visualization
                (-4.39 , -3.54 ) |         24 | 
                (-3.54 , -2.68 ) |         36 | 
                (-2.68 , -1.83 ) |         91 | 
                (-1.83 , -0.974) |        318 | ##
                (-0.974, -0.12 ) |       2229 | ##################
                (-0.12 , 0.734 ) |       4806 | ########################################
                (0.734 , 1.59  ) |        458 | ###
                (1.59  , 2.44  ) |        190 | #
                (2.44  , 3.3   ) |         33 | 
                (3.3   , 4.15  ) |          7 | 
[I]         onnxrt-runner-N0-01/05/26-15:07:18: text_features | Stats: mean=-0.0045255, std-dev=0.69494, var=0.48294, median=0.0011169, min=-4.3921 at (0, 6, 54), max=4.1521 at (0, 1, 112), avg-magnitude=0.39579, p90=0.57612, p95=1.1946, p99=2.154
[I]             ---- Histogram ----
                Bin Range        |  Num Elems | Visualization
                (-4.39 , -3.54 ) |         24 | 
                (-3.54 , -2.68 ) |         36 | 
                (-2.68 , -1.83 ) |         91 | 
                (-1.83 , -0.974) |        318 | ##
                (-0.974, -0.12 ) |       2229 | ##################
                (-0.12 , 0.734 ) |       4806 | ########################################
                (0.734 , 1.59  ) |        458 | ###
                (1.59  , 2.44  ) |        190 | #
                (2.44  , 3.3   ) |         33 | 
                (3.3   , 4.15  ) |          7 | 
[I]         Error Metrics: text_features
[I]             Minimum Required Tolerance: elemwise error | [abs=4.1723e-06] OR [rel=0.0014184] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=3.3811e-07, std-dev=3.9789e-07, var=1.5831e-13, median=2.0862e-07, min=0 at (0, 0, 45), max=4.1723e-06 at (0, 9, 36), avg-magnitude=3.3811e-07, p90=7.7486e-07, p95=1.0729e-06, p99=1.9981e-06
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (0       , 4.17e-07) |       5984 | ########################################
                    (4.17e-07, 8.34e-07) |       1420 | #########
                    (8.34e-07, 1.25e-06) |        489 | ###
                    (1.25e-06, 1.67e-06) |        152 | #
                    (1.67e-06, 2.09e-06) |         73 | 
                    (2.09e-06, 2.5e-06 ) |         44 | 
                    (2.5e-06 , 2.92e-06) |         19 | 
                    (2.92e-06, 3.34e-06) |          3 | 
                    (3.34e-06, 3.76e-06) |          5 | 
                    (3.76e-06, 4.17e-06) |          3 | 
[I]             Relative Difference | Stats: mean=5.1803e-06, std-dev=3.0262e-05, var=9.1576e-10, median=1.1036e-06, min=0 at (0, 0, 45), max=0.0014184 at (0, 23, 201), avg-magnitude=5.1803e-06, p90=6.8691e-06, p95=1.4526e-05, p99=7.9079e-05
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (0       , 0.000142) |       8152 | ########################################
                    (0.000142, 0.000284) |         23 | 
                    (0.000284, 0.000426) |         10 | 
                    (0.000426, 0.000567) |          3 | 
                    (0.000567, 0.000709) |          0 | 
                    (0.000709, 0.000851) |          2 | 
                    (0.000851, 0.000993) |          1 | 
                    (0.000993, 0.00113 ) |          0 | 
                    (0.00113 , 0.00128 ) |          0 | 
                    (0.00128 , 0.00142 ) |          1 | 
[E]         FAILED | Output: 'text_features' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'encoder_hidden_state' (dtype=float32, shape=(1, 5184, 256)) with 'encoder_hidden_state' (dtype=float32, shape=(1, 5184, 256))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:07:18: encoder_hidden_state | Stats: mean=-0.029215, std-dev=2.0924, var=4.3781, median=0.04309, min=-16.037 at (0, 3240, 233), max=10.017 at (0, 4940, 222), avg-magnitude=1.5971, p90=2.4842, p95=3.2261, p99=4.6539
[I]             ---- Histogram ----
                Bin Range      |  Num Elems | Visualization
                (-18.5, -15.1) |         19 | 
                (-15.1, -11.7) |        263 | 
                (-11.7, -8.27) |       4140 | 
                (-8.27, -4.87) |      17348 | 
                (-4.87, -1.47) |     269728 | #############
                (-1.47, 1.94 ) |     826388 | ########################################
                (1.94 , 5.34 ) |     203948 | #########
                (5.34 , 8.74 ) |       5239 | 
                (8.74 , 12.1 ) |         31 | 
                (12.1 , 15.5 ) |          0 | 
[I]         onnxrt-runner-N0-01/05/26-15:07:18: encoder_hidden_state | Stats: mean=-0.054315, std-dev=2.3718, var=5.6254, median=0.014089, min=-18.478 at (0, 5121, 233), max=15.546 at (0, 4940, 222), avg-magnitude=1.819, p90=2.8103, p95=3.6918, p99=5.3783
[I]             ---- Histogram ----
                Bin Range      |  Num Elems | Visualization
                (-18.5, -15.1) |         89 | 
                (-15.1, -11.7) |        427 | 
                (-11.7, -8.27) |       4753 | 
                (-8.27, -4.87) |      29375 | #
                (-4.87, -1.47) |     297798 | ###############
                (-1.47, 1.94 ) |     749043 | ########################################
                (1.94 , 5.34 ) |     231736 | ############
                (5.34 , 8.74 ) |      13373 | 
                (8.74 , 12.1 ) |        487 | 
                (12.1 , 15.5 ) |         23 | 
[I]         Error Metrics: encoder_hidden_state
[I]             Minimum Required Tolerance: elemwise error | [abs=8.6287] OR [rel=4.8772e+05] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=1.0321, std-dev=0.84537, var=0.71465, median=0.83463, min=2.3842e-07 at (0, 723, 246), max=8.6287 at (0, 5104, 233), avg-magnitude=1.0321, p90=2.1743, p95=2.6776, p99=3.7953
[I]                 ---- Histogram ----
                    Bin Range         |  Num Elems | Visualization
                    (2.38e-07, 0.863) |     682372 | ########################################
                    (0.863   , 1.73 ) |     407910 | #######################
                    (1.73    , 2.59 ) |     161541 | #########
                    (2.59    , 3.45 ) |      52946 | ###
                    (3.45    , 4.31 ) |      16452 | 
                    (4.31    , 5.18 ) |       4393 | 
                    (5.18    , 6.04 ) |       1113 | 
                    (6.04    , 6.9  ) |        256 | 
                    (6.9     , 7.77 ) |         95 | 
                    (7.77    , 8.63 ) |         26 | 
[I]             Relative Difference | Stats: mean=4.9905, std-dev=582.27, var=3.3904e+05, median=0.57435, min=1.1797e-07 at (0, 723, 246), max=4.8772e+05 at (0, 3625, 33), avg-magnitude=4.9905, p90=3.2239, p95=6.4486, p99=32.176
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (1.18e-07, 4.88e+04) |    1327092 | ########################################
                    (4.88e+04, 9.75e+04) |          6 | 
                    (9.75e+04, 1.46e+05) |          3 | 
                    (1.46e+05, 1.95e+05) |          0 | 
                    (1.95e+05, 2.44e+05) |          0 | 
                    (2.44e+05, 2.93e+05) |          2 | 
                    (2.93e+05, 3.41e+05) |          0 | 
                    (3.41e+05, 3.9e+05 ) |          0 | 
                    (3.9e+05 , 4.39e+05) |          0 | 
                    (4.39e+05, 4.88e+05) |          1 | 
[E]         FAILED | Output: 'encoder_hidden_state' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'decoder_hidden_states' (dtype=float32, shape=(1, 200, 256)) with 'decoder_hidden_states' (dtype=float32, shape=(1, 200, 256))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:07:18: decoder_hidden_states | Stats: mean=-0.0075265, std-dev=0.97304, var=0.9468, median=0.023594, min=-5.9228 at (0, 44, 32), max=4.4711 at (0, 143, 171), avg-magnitude=0.71806, p90=1.0877, p95=1.469, p99=2.4004
[I]             ---- Histogram ----
                Bin Range        |  Num Elems | Visualization
                (-6.03 , -4.93 ) |         65 | 
                (-4.93 , -3.82 ) |        177 | 
                (-3.82 , -2.72 ) |        337 | 
                (-2.72 , -1.62 ) |       1634 | ##
                (-1.62 , -0.518) |      11169 | #################
                (-0.518, 0.584 ) |      25552 | ########################################
                (0.584 , 1.69  ) |      10529 | ################
                (1.69  , 2.79  ) |       1426 | ##
                (2.79  , 3.89  ) |        264 | 
                (3.89  , 4.99  ) |         47 | 
[I]         onnxrt-runner-N0-01/05/26-15:07:18: decoder_hidden_states | Stats: mean=-0.0072725, std-dev=0.96756, var=0.93617, median=0.022201, min=-6.0283 at (0, 10, 32), max=4.9917 at (0, 79, 171), avg-magnitude=0.713, p90=1.0802, p95=1.4609, p99=2.3643
[I]             ---- Histogram ----
                Bin Range        |  Num Elems | Visualization
                (-6.03 , -4.93 ) |         80 | 
                (-4.93 , -3.82 ) |        141 | 
                (-3.82 , -2.72 ) |        343 | 
                (-2.72 , -1.62 ) |       1553 | ##
                (-1.62 , -0.518) |      11210 | #################
                (-0.518, 0.584 ) |      25704 | ########################################
                (0.584 , 1.69  ) |      10501 | ################
                (1.69  , 2.79  ) |       1377 | ##
                (2.79  , 3.89  ) |        240 | 
                (3.89  , 4.99  ) |         51 | 
[I]         Error Metrics: decoder_hidden_states
[I]             Minimum Required Tolerance: elemwise error | [abs=3.905] OR [rel=30657] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=0.4291, std-dev=0.38742, var=0.15009, median=0.3242, min=0 at (0, 92, 155), max=3.905 at (0, 106, 159), avg-magnitude=0.4291, p90=0.94021, p95=1.1949, p99=1.7619
[I]                 ---- Histogram ----
                    Bin Range      |  Num Elems | Visualization
                    (0    , 0.39 ) |      29305 | ########################################
                    (0.39 , 0.781) |      13941 | ###################
                    (0.781, 1.17 ) |       5220 | #######
                    (1.17 , 1.56 ) |       1796 | ##
                    (1.56 , 1.95 ) |        649 | 
                    (1.95 , 2.34 ) |        192 | 
                    (2.34 , 2.73 ) |         71 | 
                    (2.73 , 3.12 ) |         23 | 
                    (3.12 , 3.51 ) |          2 | 
                    (3.51 , 3.9  ) |          1 | 
[I]             Relative Difference | Stats: mean=3.8535, std-dev=142.28, var=20243, median=0.586, min=0 at (0, 92, 155), max=30657 at (0, 85, 138), avg-magnitude=3.8535, p90=3.808, p95=7.4858, p99=36.234
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (0       , 3.07e+03) |      51197 | ########################################
                    (3.07e+03, 6.13e+03) |          1 | 
                    (6.13e+03, 9.2e+03 ) |          1 | 
                    (9.2e+03 , 1.23e+04) |          0 | 
                    (1.23e+04, 1.53e+04) |          0 | 
                    (1.53e+04, 1.84e+04) |          0 | 
                    (1.84e+04, 2.15e+04) |          0 | 
                    (2.15e+04, 2.45e+04) |          0 | 
                    (2.45e+04, 2.76e+04) |          0 | 
                    (2.76e+04, 3.07e+04) |          1 | 
[E]         FAILED | Output: 'decoder_hidden_states' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'pred_boxes' (dtype=float32, shape=(1, 200, 4)) with 'pred_boxes' (dtype=float32, shape=(1, 200, 4))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:07:18: pred_boxes | Stats: mean=0.52495, std-dev=0.20177, var=0.04071, median=0.52565, min=-0.097559 at (0, 89, 0), max=1.0239 at (0, 144, 3), avg-magnitude=0.52519, p90=0.77729, p95=0.8783, p99=0.98599
[I]             ---- Histogram ----
                Bin Range         |  Num Elems | Visualization
                (-0.0976, 0.0146) |          5 | #
                (0.0146 , 0.127 ) |         25 | #####
                (0.127  , 0.239 ) |         30 | ######
                (0.239  , 0.351 ) |         75 | ################
                (0.351  , 0.463 ) |        170 | #####################################
                (0.463  , 0.575 ) |        179 | ########################################
                (0.575  , 0.687 ) |        159 | ###################################
                (0.687  , 0.8   ) |         90 | ####################
                (0.8    , 0.912 ) |         36 | ########
                (0.912  , 1.02  ) |         31 | ######
[I]         onnxrt-runner-N0-01/05/26-15:07:18: pred_boxes | Stats: mean=0.51037, std-dev=0.25017, var=0.062584, median=0.50878, min=-0.0011068 at (0, 111, 0), max=1.0027 at (0, 26, 2), avg-magnitude=0.51038, p90=0.85718, p95=0.9608, p99=0.99875
[I]             ---- Histogram ----
                Bin Range         |  Num Elems | Visualization
                (-0.0976, 0.0146) |          9 | ##
                (0.0146 , 0.127 ) |         43 | ############
                (0.127  , 0.239 ) |         73 | #####################
                (0.239  , 0.351 ) |        102 | #############################
                (0.351  , 0.463 ) |        138 | ########################################
                (0.463  , 0.575 ) |        108 | ###############################
                (0.575  , 0.687 ) |        125 | ####################################
                (0.687  , 0.8   ) |         87 | #########################
                (0.8    , 0.912 ) |         55 | ###############
                (0.912  , 1.02  ) |         60 | #################
[I]         Error Metrics: pred_boxes
[I]             Minimum Required Tolerance: elemwise error | [abs=0.68987] OR [rel=151.79] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=0.077092, std-dev=0.087299, var=0.007621, median=0.046964, min=0.00022468 at (0, 70, 3), max=0.68987 at (0, 71, 0), avg-magnitude=0.077092, p90=0.19431, p95=0.2537, p99=0.38542
[I]                 ---- Histogram ----
                    Bin Range          |  Num Elems | Visualization
                    (0.000225, 0.0692) |        483 | ########################################
                    (0.0692  , 0.138 ) |        169 | #############
                    (0.138   , 0.207 ) |         78 | ######
                    (0.207   , 0.276 ) |         47 | ###
                    (0.276   , 0.345 ) |         14 | #
                    (0.345   , 0.414 ) |          3 | 
                    (0.414   , 0.483 ) |          3 | 
                    (0.483   , 0.552 ) |          0 | 
                    (0.552   , 0.621 ) |          0 | 
                    (0.621   , 0.69  ) |          3 | 
[I]             Relative Difference | Stats: mean=0.8091, std-dev=6.8001, var=46.241, median=0.099547, min=0.00048962 at (0, 70, 3), max=151.79 at (0, 40, 0), avg-magnitude=0.8091, p90=0.67171, p95=1.2416, p99=10.851
[I]                 ---- Histogram ----
                    Bin Range       |  Num Elems | Visualization
                    (0.00049, 15.2) |        792 | ########################################
                    (15.2   , 30.4) |          4 | 
                    (30.4   , 45.5) |          0 | 
                    (45.5   , 60.7) |          1 | 
                    (60.7   , 75.9) |          2 | 
                    (75.9   , 91.1) |          0 | 
                    (91.1   , 106 ) |          0 | 
                    (106    , 121 ) |          0 | 
                    (121    , 137 ) |          0 | 
                    (137    , 152 ) |          1 | 
[E]         FAILED | Output: 'pred_boxes' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'pred_logits' (dtype=float32, shape=(1, 200)) with 'pred_logits' (dtype=float32, shape=(1, 200))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:07:18: pred_logits | Stats: mean=-2.2528, std-dev=0.3234, var=0.10459, median=-2.2741, min=-3.3643 at (0, 167), max=-0.99596 at (0, 144), avg-magnitude=2.2528, p90=-1.8565, p95=-1.7238, p99=-1.3785
[I]             ---- Histogram ----
                Bin Range        |  Num Elems | Visualization
                (-3.36 , -3.13 ) |          1 | 
                (-3.13 , -2.89 ) |          2 | #
                (-2.89 , -2.65 ) |         14 | ########
                (-2.65 , -2.42 ) |         49 | ##############################
                (-2.42 , -2.18 ) |         64 | ########################################
                (-2.18 , -1.94 ) |         40 | #########################
                (-1.94 , -1.71 ) |         20 | ############
                (-1.71 , -1.47 ) |          5 | ###
                (-1.47 , -1.23 ) |          4 | ##
                (-1.23 , -0.996) |          1 | 
[I]         onnxrt-runner-N0-01/05/26-15:07:18: pred_logits | Stats: mean=-2.3136, std-dev=0.36742, var=0.135, median=-2.3567, min=-3.1664 at (0, 126), max=-1.0731 at (0, 175), avg-magnitude=2.3136, p90=-1.8362, p95=-1.7265, p99=-1.3385
[I]             ---- Histogram ----
                Bin Range        |  Num Elems | Visualization
                (-3.36 , -3.13 ) |          1 | 
                (-3.13 , -2.89 ) |          8 | ######
                (-2.89 , -2.65 ) |         29 | #######################
                (-2.65 , -2.42 ) |         43 | ##################################
                (-2.42 , -2.18 ) |         50 | ########################################
                (-2.18 , -1.94 ) |         39 | ###############################
                (-1.94 , -1.71 ) |         20 | ################
                (-1.71 , -1.47 ) |          6 | ####
                (-1.47 , -1.23 ) |          3 | ##
                (-1.23 , -0.996) |          1 | 
[I]         Error Metrics: pred_logits
[I]             Minimum Required Tolerance: elemwise error | [abs=1.2043] OR [rel=0.96577] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=0.29036, std-dev=0.22991, var=0.052859, median=0.2241, min=0.00070286 at (0, 67), max=1.2043 at (0, 89), avg-magnitude=0.29036, p90=0.62114, p95=0.76967, p99=0.93489
[I]                 ---- Histogram ----
                    Bin Range         |  Num Elems | Visualization
                    (0.000703, 0.121) |         49 | ####################################
                    (0.121   , 0.241) |         53 | ########################################
                    (0.241   , 0.362) |         39 | #############################
                    (0.362   , 0.482) |         24 | ##################
                    (0.482   , 0.603) |         12 | #########
                    (0.603   , 0.723) |         10 | #######
                    (0.723   , 0.843) |          7 | #####
                    (0.843   , 0.964) |          4 | ###
                    (0.964   , 1.08 ) |          0 | 
                    (1.08    , 1.2  ) |          2 | #
[I]             Relative Difference | Stats: mean=0.13184, std-dev=0.12588, var=0.015846, median=0.09891, min=0.00028166 at (0, 67), max=0.96577 at (0, 89), avg-magnitude=0.13184, p90=0.26058, p95=0.35903, p99=0.56951
[I]                 ---- Histogram ----
                    Bin Range          |  Num Elems | Visualization
                    (0.000282, 0.0968) |         98 | ########################################
                    (0.0968  , 0.193 ) |         62 | #########################
                    (0.193   , 0.29  ) |         23 | #########
                    (0.29    , 0.386 ) |          7 | ##
                    (0.386   , 0.483 ) |          5 | ##
                    (0.483   , 0.58  ) |          3 | #
                    (0.58    , 0.676 ) |          1 | 
                    (0.676   , 0.773 ) |          0 | 
                    (0.773   , 0.869 ) |          0 | 
                    (0.869   , 0.966 ) |          1 | 
[E]         FAILED | Output: 'pred_logits' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'presence_logits' (dtype=float32, shape=(1, 1)) with 'presence_logits' (dtype=float32, shape=(1, 1))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:07:18: presence_logits | Stats: mean=-3.6407, std-dev=0, var=0, median=-3.6407, min=-3.6407 at (0, 0), max=-3.6407 at (0, 0), avg-magnitude=3.6407, p90=-3.6407, p95=-3.6407, p99=-3.6407
[I]             ---- Values ----
                    [[-3.640685]]
[I]             ---- Histogram ----
                Bin Range      |  Num Elems | Visualization
                (-3.64, -3.52) |          1 | ########################################
                (-3.52, -3.41) |          0 | 
                (-3.41, -3.29) |          0 | 
                (-3.29, -3.18) |          0 | 
                (-3.18, -3.06) |          0 | 
                (-3.06, -2.95) |          0 | 
                (-2.95, -2.83) |          0 | 
                (-2.83, -2.71) |          0 | 
                (-2.71, -2.6 ) |          0 | 
                (-2.6 , -2.48) |          0 | 
[I]         onnxrt-runner-N0-01/05/26-15:07:18: presence_logits | Stats: mean=-2.4821, std-dev=0, var=0, median=-2.4821, min=-2.4821 at (0, 0), max=-2.4821 at (0, 0), avg-magnitude=2.4821, p90=-2.4821, p95=-2.4821, p99=-2.4821
[I]             ---- Values ----
                    [[-2.482077]]
[I]             ---- Histogram ----
                Bin Range      |  Num Elems | Visualization
                (-3.64, -3.52) |          0 | 
                (-3.52, -3.41) |          0 | 
                (-3.41, -3.29) |          0 | 
                (-3.29, -3.18) |          0 | 
                (-3.18, -3.06) |          0 | 
                (-3.06, -2.95) |          0 | 
                (-2.95, -2.83) |          0 | 
                (-2.83, -2.71) |          0 | 
                (-2.71, -2.6 ) |          0 | 
                (-2.6 , -2.48) |          1 | ########################################
[I]         Error Metrics: presence_logits
[I]             Minimum Required Tolerance: elemwise error | [abs=1.1586] OR [rel=0.46679] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=1.1586, std-dev=0, var=0, median=1.1586, min=1.1586 at (0, 0), max=1.1586 at (0, 0), avg-magnitude=1.1586, p90=1.1586, p95=1.1586, p99=1.1586
[I]                 ---- Values ----
                        [[1.1586082]]
[I]                 ---- Histogram ----
                    Bin Range      |  Num Elems | Visualization
                    (0.659, 0.759) |          0 | 
                    (0.759, 0.859) |          0 | 
                    (0.859, 0.959) |          0 | 
                    (0.959, 1.06 ) |          0 | 
                    (1.06 , 1.16 ) |          0 | 
                    (1.16 , 1.26 ) |          1 | ########################################
                    (1.26 , 1.36 ) |          0 | 
                    (1.36 , 1.46 ) |          0 | 
                    (1.46 , 1.56 ) |          0 | 
                    (1.56 , 1.66 ) |          0 | 
[I]             Relative Difference | Stats: mean=0.46679, std-dev=0, var=0, median=0.46679, min=0.46679 at (0, 0), max=0.46679 at (0, 0), avg-magnitude=0.46679, p90=0.46679, p95=0.46679, p99=0.46679
[I]                 ---- Values ----
                        [[0.4667898]]
[I]                 ---- Histogram ----
                    Bin Range         |  Num Elems | Visualization
                    (-0.0332, 0.0668) |          0 | 
                    (0.0668 , 0.167 ) |          0 | 
                    (0.167  , 0.267 ) |          0 | 
                    (0.267  , 0.367 ) |          0 | 
                    (0.367  , 0.467 ) |          0 | 
                    (0.467  , 0.567 ) |          1 | ########################################
                    (0.567  , 0.667 ) |          0 | 
                    (0.667  , 0.767 ) |          0 | 
                    (0.767  , 0.867 ) |          0 | 
                    (0.867  , 0.967 ) |          0 | 
[E]         FAILED | Output: 'presence_logits' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'pred_masks' (dtype=float32, shape=(1, 200, 288, 288)) with 'pred_masks' (dtype=float32, shape=(1, 200, 288, 288))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:07:18: pred_masks | Stats: mean=-17.966, std-dev=12.069, var=145.66, median=-17.336, min=-78.59 at (0, 24, 230, 154), max=9.0686 at (0, 33, 192, 166), avg-magnitude=17.987, p90=-1.9971, p95=-0.5272, p99=0.28164
[I]             ---- Histogram ----
                Bin Range        |  Num Elems | Visualization
                (-100  , -89.1 ) |          0 | 
                (-89.1 , -78.1 ) |          1 | 
                (-78.1 , -67   ) |       1281 | 
                (-67   , -56   ) |      31777 | 
                (-56   , -45   ) |     299428 | ##
                (-45   , -34   ) |    1390408 | ##########
                (-34   , -23   ) |    3754791 | ###########################
                (-23   , -11.9 ) |    5382122 | ########################################
                (-11.9 , -0.927) |    4604179 | ##################################
                (-0.927, 10.1  ) |    1124813 | ########
[I]         onnxrt-runner-N0-01/05/26-15:07:18: pred_masks | Stats: mean=-19.907, std-dev=13.444, var=180.73, median=-18.39, min=-100.1 at (0, 100, 184, 157), max=10.092 at (0, 158, 282, 215), avg-magnitude=19.925, p90=-3.4109, p95=-1.288, p99=0.17772
[I]             ---- Histogram ----
                Bin Range        |  Num Elems | Visualization
                (-100  , -89.1 ) |        436 | 
                (-89.1 , -78.1 ) |       3340 | 
                (-78.1 , -67   ) |      37138 | 
                (-67   , -56   ) |     173544 | #
                (-56   , -45   ) |     582371 | ####
                (-45   , -34   ) |    1682030 | ############
                (-34   , -23   ) |    3667927 | ############################
                (-23   , -11.9 ) |    5196738 | ########################################
                (-11.9 , -0.927) |    4558279 | ###################################
                (-0.927, 10.1  ) |     686997 | #####
[I]         Error Metrics: pred_masks
[I]             Minimum Required Tolerance: elemwise error | [abs=71.439] OR [rel=8.2647e+06] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=8.4159, std-dev=7.7006, var=59.299, median=6.2722, min=0 at (0, 4, 227, 131), max=71.439 at (0, 114, 172, 144), avg-magnitude=8.4159, p90=19.056, p95=23.809, p99=33.912
[I]                 ---- Histogram ----
                    Bin Range    |  Num Elems | Visualization
                    (0   , 7.14) |    9077192 | ########################################
                    (7.14, 14.3) |    4377686 | ###################
                    (14.3, 21.4) |    1950703 | ########
                    (21.4, 28.6) |     787138 | ###
                    (28.6, 35.7) |     272937 | #
                    (35.7, 42.9) |      85912 | 
                    (42.9, 50  ) |      26309 | 
                    (50  , 57.2) |       9677 | 
                    (57.2, 64.3) |       1195 | 
                    (64.3, 71.4) |         51 | 
[I]             Relative Difference | Stats: mean=2.5923, std-dev=2148.1, var=4.6144e+06, median=0.39781, min=0 at (0, 4, 227, 131), max=8.2647e+06 at (0, 134, 159, 97), avg-magnitude=2.5923, p90=1.1935, p95=2.3632, p99=11.902
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (0       , 8.26e+05) |   16588798 | ########################################
                    (8.26e+05, 1.65e+06) |          0 | 
                    (1.65e+06, 2.48e+06) |          0 | 
                    (2.48e+06, 3.31e+06) |          1 | 
                    (3.31e+06, 4.13e+06) |          0 | 
                    (4.13e+06, 4.96e+06) |          0 | 
                    (4.96e+06, 5.79e+06) |          0 | 
                    (5.79e+06, 6.61e+06) |          0 | 
                    (6.61e+06, 7.44e+06) |          0 | 
                    (7.44e+06, 8.26e+06) |          1 | 
[E]         FAILED | Output: 'pred_masks' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'semantic_seg' (dtype=float32, shape=(1, 1, 288, 288)) with 'semantic_seg' (dtype=float32, shape=(1, 1, 288, 288))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:07:18: semantic_seg | Stats: mean=-0.31233, std-dev=0.57507, var=0.33071, median=-0.31994, min=-8.318 at (0, 0, 167, 1), max=3.231 at (0, 0, 162, 187), avg-magnitude=0.47049, p90=0.25276, p95=0.54216, p99=1.2357
[I]             ---- Histogram ----
                Bin Range          |  Num Elems | Visualization
                (-15.3  , -13.4  ) |          0 | 
                (-13.4  , -11.5  ) |          0 | 
                (-11.5  , -9.6   ) |          0 | 
                (-9.6   , -7.7   ) |          1 | 
                (-7.7   , -5.8   ) |          4 | 
                (-5.8   , -3.9   ) |         95 | 
                (-3.9   , -2     ) |       1095 | 
                (-2     , -0.0949) |      59160 | ########################################
                (-0.0949, 1.81   ) |      22351 | ###############
                (1.81   , 3.71   ) |        238 | 
[I]         onnxrt-runner-N0-01/05/26-15:07:18: semantic_seg | Stats: mean=-2.5686, std-dev=1.2996, var=1.6889, median=-2.9381, min=-15.308 at (0, 0, 287, 97), max=3.7083 at (0, 0, 117, 179), avg-magnitude=2.6896, p90=-0.51426, p95=0.13506, p99=2.0755
[I]             ---- Histogram ----
                Bin Range          |  Num Elems | Visualization
                (-15.3  , -13.4  ) |          4 | 
                (-13.4  , -11.5  ) |          9 | 
                (-11.5  , -9.6   ) |         16 | 
                (-9.6   , -7.7   ) |         36 | 
                (-7.7   , -5.8   ) |        441 | 
                (-5.8   , -3.9   ) |       1951 | #
                (-3.9   , -2     ) |      64646 | ########################################
                (-2     , -0.0949) |      10397 | ######
                (-0.0949, 1.81   ) |       4274 | ##
                (1.81   , 3.71   ) |       1170 | 
[I]         Error Metrics: semantic_seg
[I]             Minimum Required Tolerance: elemwise error | [abs=13.127] OR [rel=6008.1] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=2.3233, std-dev=0.89382, var=0.79891, median=2.5438, min=1.0788e-05 at (0, 0, 130, 156), max=13.127 at (0, 0, 287, 97), avg-magnitude=2.3233, p90=3.1925, p95=3.3673, p99=3.7743
[I]                 ---- Histogram ----
                    Bin Range        |  Num Elems | Visualization
                    (1.08e-05, 1.31) |      12790 | ##############
                    (1.31    , 2.63) |      33420 | ####################################
                    (2.63    , 3.94) |      36182 | ########################################
                    (3.94    , 5.25) |        472 | 
                    (5.25    , 6.56) |         43 | 
                    (6.56    , 7.88) |         16 | 
                    (7.88    , 9.19) |          8 | 
                    (9.19    , 10.5) |          9 | 
                    (10.5    , 11.8) |          3 | 
                    (11.8    , 13.1) |          1 | 
[I]             Relative Difference | Stats: mean=1.2131, std-dev=22.438, var=503.48, median=0.87599, min=3.3699e-05 at (0, 0, 130, 156), max=6008.1 at (0, 0, 163, 181), avg-magnitude=1.2131, p90=1.0899, p95=1.3205, p99=4.7536
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (3.37e-05, 601     ) |      82939 | ########################################
                    (601     , 1.2e+03 ) |          3 | 
                    (1.2e+03 , 1.8e+03 ) |          1 | 
                    (1.8e+03 , 2.4e+03 ) |          0 | 
                    (2.4e+03 , 3e+03   ) |          0 | 
                    (3e+03   , 3.6e+03 ) |          0 | 
                    (3.6e+03 , 4.21e+03) |          0 | 
                    (4.21e+03, 4.81e+03) |          0 | 
                    (4.81e+03, 5.41e+03) |          0 | 
                    (5.41e+03, 6.01e+03) |          1 | 
[E]         FAILED | Output: 'semantic_seg' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[E]     FAILED | Mismatched outputs: ['vision_feature_0', 'vision_feature_1', 'vision_feature_2', 'text_features', 'encoder_hidden_state', 'decoder_hidden_states', 'pred_boxes', 'pred_logits', 'presence_logits', 'pred_masks', 'semantic_seg']
[E] Accuracy Summary | trt-runner-N0-01/05/26-15:07:18 vs. onnxrt-runner-N0-01/05/26-15:07:18 | Passed: 0/1 iterations | Pass Rate: 0.0%
Traceback (most recent call last):
  File "/home/zy/open_project/SAM3-TensorRT/python/polygraphy_test.py", line 45, in <module>
    main()
  File "/home/zy/open_project/SAM3-TensorRT/python/polygraphy_test.py", line 37, in main
    assert bool(Comparator.compare_accuracy(run_results, compare_func=CompareFunc.simple(atol=1e-8)))
AssertionError

There are a lot of differences in the encoder_hidden_states. It indicates that the detr_encoder brings some gaps.
Did you notice the gap in your device?

[Huntersdeng]

Then I use a real image to rerun polygraphy test. Here's the result:

[I] trt-runner-N0-01/05/26-15:40:43     | Activating and starting inference
[I] trt-runner-N0-01/05/26-15:40:43    
    ---- Inference Input(s) ----
    {pixel_values [dtype=float32, shape=(1, 3, 1008, 1008)]}
[I] trt-runner-N0-01/05/26-15:40:43    
    ---- Inference Output(s) ----
    {vision_feature_0 [dtype=float32, shape=(1, 256, 288, 288)],
     vision_feature_1 [dtype=float32, shape=(1, 256, 144, 144)],
     vision_feature_2 [dtype=float32, shape=(1, 256, 72, 72)],
     text_features [dtype=float32, shape=(1, 32, 256)],
     encoder_hidden_state [dtype=float32, shape=(1, 5184, 256)],
     decoder_hidden_states [dtype=float32, shape=(1, 200, 256)],
     pred_boxes [dtype=float32, shape=(1, 200, 4)],
     pred_logits [dtype=float32, shape=(1, 200)],
     presence_logits [dtype=float32, shape=(1, 1)],
     pred_masks [dtype=float32, shape=(1, 200, 288, 288)],
     semantic_seg [dtype=float32, shape=(1, 1, 288, 288)]}
[I] trt-runner-N0-01/05/26-15:40:43     | Completed 1 iteration(s) in 684.6 ms | Average inference time: 684.6 ms.
[I] onnxrt-runner-N0-01/05/26-15:40:43  | Activating and starting inference
[I] Creating ONNX-Runtime Inference Session with providers: ['CPUExecutionProvider']
2026-01-05 15:40:47.677032741 [W:onnxruntime:, graph.cc:120 MergeShapeInfo] Error merging shape info for output. '/vision_encoder/backbone/embeddings/Concat_output_0' source:{4} target:{5}. Falling back to lenient merge.
[I] onnxrt-runner-N0-01/05/26-15:40:43 
    ---- Inference Input(s) ----
    {pixel_values [dtype=float32, shape=(1, 3, 1008, 1008)]}
[I] onnxrt-runner-N0-01/05/26-15:40:43 
    ---- Inference Output(s) ----
    {vision_feature_0 [dtype=float32, shape=(1, 256, 288, 288)],
     vision_feature_1 [dtype=float32, shape=(1, 256, 144, 144)],
     vision_feature_2 [dtype=float32, shape=(1, 256, 72, 72)],
     text_features [dtype=float32, shape=(1, 32, 256)],
     encoder_hidden_state [dtype=float32, shape=(1, 5184, 256)],
     decoder_hidden_states [dtype=float32, shape=(1, 200, 256)],
     pred_boxes [dtype=float32, shape=(1, 200, 4)],
     pred_logits [dtype=float32, shape=(1, 200)],
     presence_logits [dtype=float32, shape=(1, 1)],
     pred_masks [dtype=float32, shape=(1, 200, 288, 288)],
     semantic_seg [dtype=float32, shape=(1, 1, 288, 288)]}
[I] onnxrt-runner-N0-01/05/26-15:40:43  | Completed 1 iteration(s) in 8021 ms | Average inference time: 8021 ms.
[I] Accuracy Comparison | trt-runner-N0-01/05/26-15:40:43 vs. onnxrt-runner-N0-01/05/26-15:40:43
[I]     Comparing Output: 'vision_feature_0' (dtype=float32, shape=(1, 256, 288, 288)) with 'vision_feature_0' (dtype=float32, shape=(1, 256, 288, 288))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[W]         onnxrt-runner-N0-01/05/26-15:40:43  | Output: vision_feature_0: Some values are 0. Will add a small epsilon quantity to these when computing relative difference. Note that this may cause some relative differences to be extremely high. 
[I]         trt-runner-N0-01/05/26-15:40:43: vision_feature_0 | Stats: mean=-0.016483, std-dev=0.018827, var=0.00035445, median=-0.013207, min=-0.23177 at (0, 45, 115, 157), max=0.21282 at (0, 23, 115, 157), avg-magnitude=0.018727, p90=0.0033401, p95=0.0079961, p99=0.019125
[I]             ---- Histogram ----
                Bin Range            |  Num Elems | Visualization
                (-0.232  , -0.187  ) |          9 | 
                (-0.187  , -0.143  ) |        329 | 
                (-0.143  , -0.0984 ) |      28894 | 
                (-0.0984 , -0.0539 ) |     845608 | ##
                (-0.0539 , -0.00947) |   11915097 | ########################################
                (-0.00947, 0.035   ) |    8408501 | ############################
                (0.035   , 0.0794  ) |      34162 | 
                (0.0794  , 0.124   ) |       1003 | 
                (0.124   , 0.168   ) |         57 | 
                (0.168   , 0.213   ) |          4 | 
[I]         onnxrt-runner-N0-01/05/26-15:40:43: vision_feature_0 | Stats: mean=-0.016483, std-dev=0.018827, var=0.00035447, median=-0.013207, min=-0.23171 at (0, 45, 115, 157), max=0.21283 at (0, 23, 115, 157), avg-magnitude=0.018727, p90=0.0033409, p95=0.007997, p99=0.019126
[I]             ---- Histogram ----
                Bin Range            |  Num Elems | Visualization
                (-0.232  , -0.187  ) |          9 | 
                (-0.187  , -0.143  ) |        329 | 
                (-0.143  , -0.0984 ) |      28903 | 
                (-0.0984 , -0.0539 ) |     845665 | ##
                (-0.0539 , -0.00947) |   11914814 | ########################################
                (-0.00947, 0.035   ) |    8408716 | ############################
                (0.035   , 0.0794  ) |      34164 | 
                (0.0794  , 0.124   ) |       1003 | 
                (0.124   , 0.168   ) |         57 | 
                (0.168   , 0.213   ) |          4 | 
[I]         Error Metrics: vision_feature_0
[I]             Minimum Required Tolerance: elemwise error | [abs=0.0014812] OR [rel=8.6134e+10] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=1.0346e-05, std-dev=1.2994e-05, var=1.6885e-10, median=7.3202e-06, min=0 at (0, 0, 13, 179), max=0.0014812 at (0, 27, 257, 5), avg-magnitude=1.0346e-05, p90=2.1532e-05, p95=2.8703e-05, p99=5.4108e-05
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (0       , 0.000148) |   21217186 | ########################################
                    (0.000148, 0.000296) |      13411 | 
                    (0.000296, 0.000444) |       2116 | 
                    (0.000444, 0.000592) |        604 | 
                    (0.000592, 0.000741) |        220 | 
                    (0.000741, 0.000889) |         74 | 
                    (0.000889, 0.00104 ) |         31 | 
                    (0.00104 , 0.00118 ) |         13 | 
                    (0.00118 , 0.00133 ) |          8 | 
                    (0.00133 , 0.00148 ) |          1 | 
[I]             Relative Difference | Stats: mean=5750.5, std-dev=2.0257e+07, var=4.1034e+14, median=0.00051864, min=0 at (0, 0, 13, 179), max=8.6134e+10 at (0, 243, 216, 192), avg-magnitude=5750.5, p90=0.0038306, p95=0.0078626, p99=0.039922
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (0       , 8.61e+09) |   21233662 | ########################################
                    (8.61e+09, 1.72e+10) |          0 | 
                    (1.72e+10, 2.58e+10) |          0 | 
                    (2.58e+10, 3.45e+10) |          0 | 
                    (3.45e+10, 4.31e+10) |          1 | 
                    (4.31e+10, 5.17e+10) |          0 | 
                    (5.17e+10, 6.03e+10) |          0 | 
                    (6.03e+10, 6.89e+10) |          0 | 
                    (6.89e+10, 7.75e+10) |          0 | 
                    (7.75e+10, 8.61e+10) |          1 | 
[E]         FAILED | Output: 'vision_feature_0' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'vision_feature_1' (dtype=float32, shape=(1, 256, 144, 144)) with 'vision_feature_1' (dtype=float32, shape=(1, 256, 144, 144))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:40:43: vision_feature_1 | Stats: mean=-0.012733, std-dev=1.1785, var=1.389, median=-0.021082, min=-10.692 at (0, 15, 57, 68), max=9.4799 at (0, 191, 67, 69), avg-magnitude=0.91824, p90=1.4549, p95=1.9202, p99=2.8998
[I]             ---- Histogram ----
                Bin Range        |  Num Elems | Visualization
                (-10.7 , -8.67 ) |          4 | 
                (-8.67 , -6.66 ) |         76 | 
                (-6.66 , -4.64 ) |       2090 | 
                (-4.64 , -2.62 ) |      78971 | 
                (-2.62 , -0.605) |    1502292 | ###################
                (-0.605, 1.41  ) |    3160730 | ########################################
                (1.41  , 3.43  ) |     542598 | ######
                (3.43  , 5.45  ) |      20898 | 
                (5.45  , 7.46  ) |        744 | 
                (7.46  , 9.48  ) |         13 | 
[I]         onnxrt-runner-N0-01/05/26-15:40:43: vision_feature_1 | Stats: mean=-0.012727, std-dev=1.1786, var=1.389, median=-0.021055, min=-10.691 at (0, 15, 57, 68), max=9.4809 at (0, 191, 67, 69), avg-magnitude=0.91824, p90=1.4549, p95=1.9202, p99=2.8998
[I]             ---- Histogram ----
                Bin Range        |  Num Elems | Visualization
                (-10.7 , -8.67 ) |          4 | 
                (-8.67 , -6.66 ) |         76 | 
                (-6.66 , -4.64 ) |       2089 | 
                (-4.64 , -2.62 ) |      78954 | 
                (-2.62 , -0.605) |    1502288 | ###################
                (-0.605, 1.41  ) |    3160804 | ########################################
                (1.41  , 3.43  ) |     542544 | ######
                (3.43  , 5.45  ) |      20898 | 
                (5.45  , 7.46  ) |        746 | 
                (7.46  , 9.48  ) |         13 | 
[I]         Error Metrics: vision_feature_1
[I]             Minimum Required Tolerance: elemwise error | [abs=0.07064] OR [rel=3610.5] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=0.00095379, std-dev=0.0012396, var=1.5367e-06, median=0.00063908, min=0 at (0, 0, 125, 15), max=0.07064 at (0, 94, 129, 3), avg-magnitude=0.00095379, p90=0.0019933, p95=0.0027977, p99=0.0057229
[I]                 ---- Histogram ----
                    Bin Range          |  Num Elems | Visualization
                    (0      , 0.00706) |    5277026 | ########################################
                    (0.00706, 0.0141 ) |      26873 | 
                    (0.0141 , 0.0212 ) |       3374 | 
                    (0.0212 , 0.0283 ) |        757 | 
                    (0.0283 , 0.0353 ) |        234 | 
                    (0.0353 , 0.0424 ) |         99 | 
                    (0.0424 , 0.0494 ) |         30 | 
                    (0.0494 , 0.0565 ) |         12 | 
                    (0.0565 , 0.0636 ) |          7 | 
                    (0.0636 , 0.0706 ) |          4 | 
[I]             Relative Difference | Stats: mean=0.010902, std-dev=2.1376, var=4.5695, median=0.00088148, min=0 at (0, 0, 125, 15), max=3610.5 at (0, 32, 51, 5), avg-magnitude=0.010902, p90=0.0066141, p95=0.01359, p99=0.068754
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (0       , 361     ) |    5308406 | ########################################
                    (361     , 722     ) |          3 | 
                    (722     , 1.08e+03) |          4 | 
                    (1.08e+03, 1.44e+03) |          1 | 
                    (1.44e+03, 1.81e+03) |          0 | 
                    (1.81e+03, 2.17e+03) |          1 | 
                    (2.17e+03, 2.53e+03) |          0 | 
                    (2.53e+03, 2.89e+03) |          0 | 
                    (2.89e+03, 3.25e+03) |          0 | 
                    (3.25e+03, 3.61e+03) |          1 | 
[E]         FAILED | Output: 'vision_feature_1' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'vision_feature_2' (dtype=float32, shape=(1, 256, 72, 72)) with 'vision_feature_2' (dtype=float32, shape=(1, 256, 72, 72))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:40:43: vision_feature_2 | Stats: mean=-0.018557, std-dev=0.69148, var=0.47814, median=-0.014763, min=-5.699 at (0, 43, 69, 14), max=5.3579 at (0, 186, 21, 40), avg-magnitude=0.52877, p90=0.82026, p95=1.0863, p99=1.6482
[I]             ---- Histogram ----
                Bin Range      |  Num Elems | Visualization
                (-5.7 , -4.59) |        121 | 
                (-4.59, -3.49) |        811 | 
                (-3.49, -2.38) |       3251 | 
                (-2.38, -1.28) |      40893 | ##
                (-1.28, -0.17) |     487000 | ###########################
                (-0.17, 0.936) |     696233 | ########################################
                (0.936, 2.04 ) |      94460 | #####
                (2.04 , 3.15 ) |       4061 | 
                (3.15 , 4.25 ) |        254 | 
                (4.25 , 5.36 ) |         20 | 
[I]         onnxrt-runner-N0-01/05/26-15:40:43: vision_feature_2 | Stats: mean=-0.018564, std-dev=0.69148, var=0.47815, median=-0.014714, min=-5.699 at (0, 43, 69, 14), max=5.3585 at (0, 186, 21, 40), avg-magnitude=0.52877, p90=0.82023, p95=1.0862, p99=1.6482
[I]             ---- Histogram ----
                Bin Range      |  Num Elems | Visualization
                (-5.7 , -4.59) |        121 | 
                (-4.59, -3.49) |        811 | 
                (-3.49, -2.38) |       3248 | 
                (-2.38, -1.28) |      40913 | ##
                (-1.28, -0.17) |     486985 | ###########################
                (-0.17, 0.936) |     696234 | ########################################
                (0.936, 2.04 ) |      94457 | #####
                (2.04 , 3.15 ) |       4060 | 
                (3.15 , 4.25 ) |        255 | 
                (4.25 , 5.36 ) |         20 | 
[I]         Error Metrics: vision_feature_2
[I]             Minimum Required Tolerance: elemwise error | [abs=0.051149] OR [rel=521.8] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=0.00056133, std-dev=0.0008327, var=6.9339e-07, median=0.00035995, min=0 at (0, 2, 35, 36), max=0.051149 at (0, 175, 64, 1), avg-magnitude=0.00056133, p90=0.0011562, p95=0.0016613, p99=0.0037197
[I]                 ---- Histogram ----
                    Bin Range          |  Num Elems | Visualization
                    (0      , 0.00511) |    1320614 | ########################################
                    (0.00511, 0.0102 ) |       5374 | 
                    (0.0102 , 0.0153 ) |        810 | 
                    (0.0153 , 0.0205 ) |        178 | 
                    (0.0205 , 0.0256 ) |         74 | 
                    (0.0256 , 0.0307 ) |         30 | 
                    (0.0307 , 0.0358 ) |         17 | 
                    (0.0358 , 0.0409 ) |          2 | 
                    (0.0409 , 0.046  ) |          2 | 
                    (0.046  , 0.0511 ) |          3 | 
[I]             Relative Difference | Stats: mean=0.0096612, std-dev=0.72934, var=0.53193, median=0.00087782, min=0 at (0, 2, 35, 36), max=521.8 at (0, 117, 8, 15), avg-magnitude=0.0096612, p90=0.0069484, p95=0.01441, p99=0.073367
[I]                 ---- Histogram ----
                    Bin Range    |  Num Elems | Visualization
                    (0   , 52.2) |    1327083 | ########################################
                    (52.2, 104 ) |         14 | 
                    (104 , 157 ) |          2 | 
                    (157 , 209 ) |          0 | 
                    (209 , 261 ) |          2 | 
                    (261 , 313 ) |          2 | 
                    (313 , 365 ) |          0 | 
                    (365 , 417 ) |          0 | 
                    (417 , 470 ) |          0 | 
                    (470 , 522 ) |          1 | 
[E]         FAILED | Output: 'vision_feature_2' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'text_features' (dtype=float32, shape=(1, 32, 256)) with 'text_features' (dtype=float32, shape=(1, 32, 256))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:40:43: text_features | Stats: mean=-0.0045127, std-dev=0.69486, var=0.48283, median=0.0011636, min=-4.3918 at (0, 6, 54), max=4.152 at (0, 1, 112), avg-magnitude=0.39573, p90=0.57606, p95=1.194, p99=2.1539
[I]             ---- Histogram ----
                Bin Range        |  Num Elems | Visualization
                (-4.39 , -3.54 ) |         24 | 
                (-3.54 , -2.68 ) |         36 | 
                (-2.68 , -1.83 ) |         91 | 
                (-1.83 , -0.974) |        318 | ##
                (-0.974, -0.12 ) |       2231 | ##################
                (-0.12 , 0.734 ) |       4804 | ########################################
                (0.734 , 1.59  ) |        457 | ###
                (1.59  , 2.44  ) |        191 | #
                (2.44  , 3.3   ) |         33 | 
                (3.3   , 4.15  ) |          7 | 
[I]         onnxrt-runner-N0-01/05/26-15:40:43: text_features | Stats: mean=-0.0045255, std-dev=0.69494, var=0.48294, median=0.0011169, min=-4.3921 at (0, 6, 54), max=4.1521 at (0, 1, 112), avg-magnitude=0.39579, p90=0.57612, p95=1.1946, p99=2.154
[I]             ---- Histogram ----
                Bin Range        |  Num Elems | Visualization
                (-4.39 , -3.54 ) |         24 | 
                (-3.54 , -2.68 ) |         36 | 
                (-2.68 , -1.83 ) |         91 | 
                (-1.83 , -0.974) |        318 | ##
                (-0.974, -0.12 ) |       2229 | ##################
                (-0.12 , 0.734 ) |       4806 | ########################################
                (0.734 , 1.59  ) |        458 | ###
                (1.59  , 2.44  ) |        190 | #
                (2.44  , 3.3   ) |         33 | 
                (3.3   , 4.15  ) |          7 | 
[I]         Error Metrics: text_features
[I]             Minimum Required Tolerance: elemwise error | [abs=0.0032849] OR [rel=1.1013] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=0.0002091, std-dev=0.00028135, var=7.9156e-08, median=0.00012389, min=0 at (0, 9, 141), max=0.0032849 at (0, 7, 125), avg-magnitude=0.0002091, p90=0.000452, p95=0.00068024, p99=0.0014991
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (0       , 0.000328) |       6794 | ########################################
                    (0.000328, 0.000657) |        960 | #####
                    (0.000657, 0.000985) |        221 | #
                    (0.000985, 0.00131 ) |        100 | 
                    (0.00131 , 0.00164 ) |         54 | 
                    (0.00164 , 0.00197 ) |         34 | 
                    (0.00197 , 0.0023  ) |         12 | 
                    (0.0023  , 0.00263 ) |          9 | 
                    (0.00263 , 0.00296 ) |          7 | 
                    (0.00296 , 0.00328 ) |          1 | 
[I]             Relative Difference | Stats: mean=0.0034213, std-dev=0.024785, var=0.00061429, median=0.00066412, min=0 at (0, 9, 141), max=1.1013 at (0, 15, 147), avg-magnitude=0.0034213, p90=0.0043839, p95=0.0094323, p99=0.051729
[I]                 ---- Histogram ----
                    Bin Range      |  Num Elems | Visualization
                    (0    , 0.11 ) |       8163 | ########################################
                    (0.11 , 0.22 ) |         15 | 
                    (0.22 , 0.33 ) |          7 | 
                    (0.33 , 0.441) |          3 | 
                    (0.441, 0.551) |          1 | 
                    (0.551, 0.661) |          0 | 
                    (0.661, 0.771) |          0 | 
                    (0.771, 0.881) |          0 | 
                    (0.881, 0.991) |          1 | 
                    (0.991, 1.1  ) |          2 | 
[E]         FAILED | Output: 'text_features' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'encoder_hidden_state' (dtype=float32, shape=(1, 5184, 256)) with 'encoder_hidden_state' (dtype=float32, shape=(1, 5184, 256))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:40:43: encoder_hidden_state | Stats: mean=-0.039795, std-dev=2.0686, var=4.2791, median=0.013235, min=-21.214 at (0, 720, 233), max=10.172 at (0, 2937, 165), avg-magnitude=1.4859, p90=2.2648, p95=3.0348, p99=4.7899
[I]             ---- Histogram ----
                Bin Range      |  Num Elems | Visualization
                (-23.6, -20.1) |         33 | 
                (-20.1, -16.6) |        830 | 
                (-16.6, -13.1) |       1606 | 
                (-13.1, -9.6 ) |       1972 | 
                (-9.6 , -6.11) |       5204 | 
                (-6.11, -2.62) |      91003 | ####
                (-2.62, 0.873) |     823970 | ########################################
                (0.873, 4.36 ) |     383034 | ##################
                (4.36 , 7.86 ) |      17913 | 
                (7.86 , 11.3 ) |       1539 | 
[I]         onnxrt-runner-N0-01/05/26-15:40:43: encoder_hidden_state | Stats: mean=-0.062906, std-dev=2.2612, var=5.1129, median=-0.010349, min=-23.564 at (0, 5122, 233), max=11.346 at (0, 4751, 92), avg-magnitude=1.6552, p90=2.5119, p95=3.3437, p99=5.1807
[I]             ---- Histogram ----
                Bin Range      |  Num Elems | Visualization
                (-23.6, -20.1) |        113 | 
                (-20.1, -16.6) |        849 | 
                (-16.6, -13.1) |       1572 | 
                (-13.1, -9.6 ) |       2328 | 
                (-9.6 , -6.11) |       7412 | 
                (-6.11, -2.62) |     118316 | ######
                (-2.62, 0.873) |     770218 | ########################################
                (0.873, 4.36 ) |     398926 | ####################
                (4.36 , 7.86 ) |      25246 | #
                (7.86 , 11.3 ) |       2124 | 
[I]         Error Metrics: encoder_hidden_state
[I]             Minimum Required Tolerance: elemwise error | [abs=9.8704] OR [rel=9.2973e+05] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=0.87471, std-dev=0.72999, var=0.53288, median=0.70191, min=4.7684e-07 at (0, 724, 106), max=9.8704 at (0, 5008, 233), avg-magnitude=0.87471, p90=1.8477, p95=2.306, p99=3.3157
[I]                 ---- Histogram ----
                    Bin Range         |  Num Elems | Visualization
                    (4.77e-07, 0.987) |     866846 | ########################################
                    (0.987   , 1.97 ) |     350626 | ################
                    (1.97    , 2.96 ) |      86358 | ###
                    (2.96    , 3.95 ) |      18925 | 
                    (3.95    , 4.94 ) |       3408 | 
                    (4.94    , 5.92 ) |        828 | 
                    (5.92    , 6.91 ) |         67 | 
                    (6.91    , 7.9  ) |         25 | 
                    (7.9     , 8.88 ) |         13 | 
                    (8.88    , 9.87 ) |          8 | 
[I]             Relative Difference | Stats: mean=6.2122, std-dev=1173.5, var=1.3771e+06, median=0.55019, min=2.5662e-07 at (0, 724, 106), max=9.2973e+05 at (0, 2454, 221), avg-magnitude=6.2122, p90=3.1019, p95=6.2294, p99=30.847
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (2.57e-07, 9.3e+04 ) |    1327093 | ########################################
                    (9.3e+04 , 1.86e+05) |          7 | 
                    (1.86e+05, 2.79e+05) |          1 | 
                    (2.79e+05, 3.72e+05) |          0 | 
                    (3.72e+05, 4.65e+05) |          1 | 
                    (4.65e+05, 5.58e+05) |          0 | 
                    (5.58e+05, 6.51e+05) |          0 | 
                    (6.51e+05, 7.44e+05) |          1 | 
                    (7.44e+05, 8.37e+05) |          0 | 
                    (8.37e+05, 9.3e+05 ) |          1 | 
[E]         FAILED | Output: 'encoder_hidden_state' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'decoder_hidden_states' (dtype=float32, shape=(1, 200, 256)) with 'decoder_hidden_states' (dtype=float32, shape=(1, 200, 256))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:40:43: decoder_hidden_states | Stats: mean=-0.0090985, std-dev=1.0438, var=1.0894, median=0.055445, min=-5.6894 at (0, 157, 57), max=4.7858 at (0, 21, 239), avg-magnitude=0.79701, p90=1.1937, p95=1.5418, p99=2.3537
[I]             ---- Histogram ----
                Bin Range        |  Num Elems | Visualization
                (-5.69 , -4.62 ) |         37 | 
                (-4.62 , -3.54 ) |        276 | 
                (-3.54 , -2.47 ) |        677 | #
                (-2.47 , -1.4  ) |       3379 | ######
                (-1.4  , -0.323) |      13534 | ########################
                (-0.323, 0.75  ) |      22129 | ########################################
                (0.75  , 1.82  ) |       9711 | #################
                (1.82  , 2.9   ) |       1256 | ##
                (2.9   , 3.97  ) |        183 | 
                (3.97  , 5.04  ) |         18 | 
[I]         onnxrt-runner-N0-01/05/26-15:40:43: decoder_hidden_states | Stats: mean=-0.0077455, std-dev=1.0439, var=1.0897, median=0.051506, min=-5.4087 at (0, 144, 57), max=5.0435 at (0, 144, 80), avg-magnitude=0.80262, p90=1.2112, p95=1.5616, p99=2.3123
[I]             ---- Histogram ----
                Bin Range        |  Num Elems | Visualization
                (-5.69 , -4.62 ) |         28 | 
                (-4.62 , -3.54 ) |        287 | 
                (-3.54 , -2.47 ) |        692 | #
                (-2.47 , -1.4  ) |       3232 | #####
                (-1.4  , -0.323) |      13884 | #########################
                (-0.323, 0.75  ) |      21660 | ########################################
                (0.75  , 1.82  ) |       9950 | ##################
                (1.82  , 2.9   ) |       1320 | ##
                (2.9   , 3.97  ) |        131 | 
                (3.97  , 5.04  ) |         16 | 
[I]         Error Metrics: decoder_hidden_states
[I]             Minimum Required Tolerance: elemwise error | [abs=9.6898] OR [rel=56182] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=0.49879, std-dev=0.43741, var=0.19132, median=0.38742, min=2.9802e-08 at (0, 152, 58), max=9.6898 at (0, 144, 80), avg-magnitude=0.49879, p90=1.0698, p95=1.3421, p99=1.956
[I]                 ---- Histogram ----
                    Bin Range         |  Num Elems | Visualization
                    (2.98e-08, 0.969) |      44599 | ########################################
                    (0.969   , 1.94 ) |       6067 | #####
                    (1.94    , 2.91 ) |        486 | 
                    (2.91    , 3.88 ) |         35 | 
                    (3.88    , 4.84 ) |          4 | 
                    (4.84    , 5.81 ) |          3 | 
                    (5.81    , 6.78 ) |          2 | 
                    (6.78    , 7.75 ) |          2 | 
                    (7.75    , 8.72 ) |          1 | 
                    (8.72    , 9.69 ) |          1 | 
[I]             Relative Difference | Stats: mean=5.0149, std-dev=274.6, var=75407, median=0.60529, min=7.7549e-08 at (0, 152, 58), max=56182 at (0, 111, 255), avg-magnitude=5.0149, p90=3.6214, p95=7.1121, p99=36.551
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (7.75e-08, 5.62e+03) |      51197 | ########################################
                    (5.62e+03, 1.12e+04) |          1 | 
                    (1.12e+04, 1.69e+04) |          0 | 
                    (1.69e+04, 2.25e+04) |          0 | 
                    (2.25e+04, 2.81e+04) |          1 | 
                    (2.81e+04, 3.37e+04) |          0 | 
                    (3.37e+04, 3.93e+04) |          0 | 
                    (3.93e+04, 4.49e+04) |          0 | 
                    (4.49e+04, 5.06e+04) |          0 | 
                    (5.06e+04, 5.62e+04) |          1 | 
[E]         FAILED | Output: 'decoder_hidden_states' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'pred_boxes' (dtype=float32, shape=(1, 200, 4)) with 'pred_boxes' (dtype=float32, shape=(1, 200, 4))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:40:43: pred_boxes | Stats: mean=0.48641, std-dev=0.21049, var=0.044304, median=0.48819, min=-0.077362 at (0, 1, 0), max=1.0301 at (0, 12, 3), avg-magnitude=0.48716, p90=0.75594, p95=0.8089, p99=0.96124
[I]             ---- Histogram ----
                Bin Range        |  Num Elems | Visualization
                (-0.078, 0.0384) |         26 | ####
                (0.0384, 0.155 ) |         33 | ######
                (0.155 , 0.271 ) |         71 | #############
                (0.271 , 0.388 ) |         84 | ###############
                (0.388 , 0.504 ) |        213 | ########################################
                (0.504 , 0.621 ) |        164 | ##############################
                (0.621 , 0.737 ) |        110 | ####################
                (0.737 , 0.854 ) |         70 | #############
                (0.854 , 0.97  ) |         23 | ####
                (0.97  , 1.09  ) |          6 | #
[I]         onnxrt-runner-N0-01/05/26-15:40:43: pred_boxes | Stats: mean=0.44481, std-dev=0.25568, var=0.065375, median=0.44414, min=-0.078021 at (0, 12, 0), max=1.0865 at (0, 12, 3), avg-magnitude=0.44561, p90=0.80818, p95=0.88573, p99=1.0085
[I]             ---- Histogram ----
                Bin Range        |  Num Elems | Visualization
                (-0.078, 0.0384) |         34 | ########
                (0.0384, 0.155 ) |         91 | #######################
                (0.155 , 0.271 ) |        110 | ############################
                (0.271 , 0.388 ) |        101 | ##########################
                (0.388 , 0.504 ) |        152 | ########################################
                (0.504 , 0.621 ) |        112 | #############################
                (0.621 , 0.737 ) |         68 | #################
                (0.737 , 0.854 ) |         86 | ######################
                (0.854 , 0.97  ) |         28 | #######
                (0.97  , 1.09  ) |         18 | ####
[I]         Error Metrics: pred_boxes
[I]             Minimum Required Tolerance: elemwise error | [abs=0.72484] OR [rel=23.422] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=0.11833, std-dev=0.11916, var=0.014198, median=0.082056, min=0.00022948 at (0, 19, 2), max=0.72484 at (0, 164, 0), avg-magnitude=0.11833, p90=0.2844, p95=0.33319, p99=0.52392
[I]                 ---- Histogram ----
                    Bin Range          |  Num Elems | Visualization
                    (0.000229, 0.0727) |        368 | ########################################
                    (0.0727  , 0.145 ) |        178 | ###################
                    (0.145   , 0.218 ) |        109 | ###########
                    (0.218   , 0.29  ) |         70 | #######
                    (0.29    , 0.363 ) |         44 | ####
                    (0.363   , 0.435 ) |         12 | #
                    (0.435   , 0.507 ) |         10 | #
                    (0.507   , 0.58  ) |          2 | 
                    (0.58    , 0.652 ) |          4 | 
                    (0.652   , 0.725 ) |          3 | 
[I]             Relative Difference | Stats: mean=0.6427, std-dev=1.5721, var=2.4714, median=0.19608, min=0.00042314 at (0, 19, 2), max=23.422 at (0, 189, 0), avg-magnitude=0.6427, p90=1.457, p95=2.6437, p99=6.4235
[I]                 ---- Histogram ----
                    Bin Range        |  Num Elems | Visualization
                    (0.000423, 2.34) |        752 | ########################################
                    (2.34    , 4.68) |         31 | #
                    (4.68    , 7.03) |          9 | 
                    (7.03    , 9.37) |          4 | 
                    (9.37    , 11.7) |          1 | 
                    (11.7    , 14.1) |          1 | 
                    (14.1    , 16.4) |          0 | 
                    (16.4    , 18.7) |          1 | 
                    (18.7    , 21.1) |          0 | 
                    (21.1    , 23.4) |          1 | 
[E]         FAILED | Output: 'pred_boxes' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'pred_logits' (dtype=float32, shape=(1, 200)) with 'pred_logits' (dtype=float32, shape=(1, 200))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:40:43: pred_logits | Stats: mean=-4.8056, std-dev=0.81995, var=0.67232, median=-4.7991, min=-6.2231 at (0, 28), max=2.2614 at (0, 157), avg-magnitude=4.8282, p90=-4.0129, p95=-3.9243, p99=-2.7893
[I]             ---- Histogram ----
                Bin Range          |  Num Elems | Visualization
                (-6.71  , -5.61  ) |         28 | ##########
                (-5.61  , -4.5   ) |        103 | ########################################
                (-4.5   , -3.39  ) |         66 | #########################
                (-3.39  , -2.29  ) |          2 | 
                (-2.29  , -1.18  ) |          0 | 
                (-1.18  , -0.0748) |          0 | 
                (-0.0748, 1.03   ) |          0 | 
                (1.03   , 2.14   ) |          0 | 
                (2.14   , 3.24   ) |          1 | 
                (3.24   , 4.35   ) |          0 | 
[I]         onnxrt-runner-N0-01/05/26-15:40:43: pred_logits | Stats: mean=-5.1101, std-dev=0.95784, var=0.91747, median=-5.1066, min=-6.7141 at (0, 153), max=4.3513 at (0, 144), avg-magnitude=5.1536, p90=-4.3142, p95=-4.0877, p99=-3.6351
[I]             ---- Histogram ----
                Bin Range          |  Num Elems | Visualization
                (-6.71  , -5.61  ) |         56 | ####################
                (-5.61  , -4.5   ) |        110 | ########################################
                (-4.5   , -3.39  ) |         33 | ############
                (-3.39  , -2.29  ) |          0 | 
                (-2.29  , -1.18  ) |          0 | 
                (-1.18  , -0.0748) |          0 | 
                (-0.0748, 1.03   ) |          0 | 
                (1.03   , 2.14   ) |          0 | 
                (2.14   , 3.24   ) |          0 | 
                (3.24   , 4.35   ) |          1 | 
[I]         Error Metrics: pred_logits
[I]             Minimum Required Tolerance: elemwise error | [abs=10.24] OR [rel=2.3533] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=0.7188, std-dev=0.97763, var=0.95575, median=0.55438, min=0.0048375 at (0, 6), max=10.24 at (0, 144), avg-magnitude=0.7188, p90=1.2548, p95=1.524, p99=2.8933
[I]                 ---- Histogram ----
                    Bin Range       |  Num Elems | Visualization
                    (0.00484, 1.03) |        159 | ########################################
                    (1.03   , 2.05) |         35 | ########
                    (2.05   , 3.08) |          4 | #
                    (3.08   , 4.1 ) |          0 | 
                    (4.1    , 5.12) |          0 | 
                    (5.12   , 6.15) |          0 | 
                    (6.15   , 7.17) |          0 | 
                    (7.17   , 8.19) |          1 | 
                    (8.19   , 9.22) |          0 | 
                    (9.22   , 10.2) |          1 | 
[I]             Relative Difference | Stats: mean=0.13932, std-dev=0.2041, var=0.041656, median=0.10749, min=0.0011292 at (0, 6), max=2.3533 at (0, 144), avg-magnitude=0.13932, p90=0.22524, p95=0.27422, p99=0.57234
[I]                 ---- Histogram ----
                    Bin Range        |  Num Elems | Visualization
                    (0.00113, 0.236) |        182 | ########################################
                    (0.236  , 0.472) |         12 | ##
                    (0.472  , 0.707) |          4 | 
                    (0.707  , 0.942) |          0 | 
                    (0.942  , 1.18 ) |          0 | 
                    (1.18   , 1.41 ) |          1 | 
                    (1.41   , 1.65 ) |          0 | 
                    (1.65   , 1.88 ) |          0 | 
                    (1.88   , 2.12 ) |          0 | 
                    (2.12   , 2.35 ) |          1 | 
[E]         FAILED | Output: 'pred_logits' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'presence_logits' (dtype=float32, shape=(1, 1)) with 'presence_logits' (dtype=float32, shape=(1, 1))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:40:43: presence_logits | Stats: mean=3.5595, std-dev=0, var=0, median=3.5595, min=3.5595 at (0, 0), max=3.5595 at (0, 0), avg-magnitude=3.5595, p90=3.5595, p95=3.5595, p99=3.5595
[I]             ---- Values ----
                    [[3.5594957]]
[I]             ---- Histogram ----
                Bin Range    |  Num Elems | Visualization
                (3.56, 3.57) |          1 | ########################################
                (3.57, 3.57) |          0 | 
                (3.57, 3.58) |          0 | 
                (3.58, 3.58) |          0 | 
                (3.58, 3.59) |          0 | 
                (3.59, 3.6 ) |          0 | 
                (3.6 , 3.6 ) |          0 | 
                (3.6 , 3.61) |          0 | 
                (3.61, 3.62) |          0 | 
                (3.62, 3.62) |          0 | 
[I]         onnxrt-runner-N0-01/05/26-15:40:43: presence_logits | Stats: mean=3.6227, std-dev=0, var=0, median=3.6227, min=3.6227 at (0, 0), max=3.6227 at (0, 0), avg-magnitude=3.6227, p90=3.6227, p95=3.6227, p99=3.6227
[I]             ---- Values ----
                    [[3.6227472]]
[I]             ---- Histogram ----
                Bin Range    |  Num Elems | Visualization
                (3.56, 3.57) |          0 | 
                (3.57, 3.57) |          0 | 
                (3.57, 3.58) |          0 | 
                (3.58, 3.58) |          0 | 
                (3.58, 3.59) |          0 | 
                (3.59, 3.6 ) |          0 | 
                (3.6 , 3.6 ) |          0 | 
                (3.6 , 3.61) |          0 | 
                (3.61, 3.62) |          0 | 
                (3.62, 3.62) |          1 | ########################################
[I]         Error Metrics: presence_logits
[I]             Minimum Required Tolerance: elemwise error | [abs=0.063251] OR [rel=0.01746] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=0.063251, std-dev=0, var=0, median=0.063251, min=0.063251 at (0, 0), max=0.063251 at (0, 0), avg-magnitude=0.063251, p90=0.063251, p95=0.063251, p99=0.063251
[I]                 ---- Values ----
                        [[0.0632515]]
[I]                 ---- Histogram ----
                    Bin Range          |  Num Elems | Visualization
                    (-0.437 , -0.337 ) |          0 | 
                    (-0.337 , -0.237 ) |          0 | 
                    (-0.237 , -0.137 ) |          0 | 
                    (-0.137 , -0.0367) |          0 | 
                    (-0.0367, 0.0633 ) |          0 | 
                    (0.0633 , 0.163  ) |          1 | ########################################
                    (0.163  , 0.263  ) |          0 | 
                    (0.263  , 0.363  ) |          0 | 
                    (0.363  , 0.463  ) |          0 | 
                    (0.463  , 0.563  ) |          0 | 
[I]             Relative Difference | Stats: mean=0.01746, std-dev=0, var=0, median=0.01746, min=0.01746 at (0, 0), max=0.01746 at (0, 0), avg-magnitude=0.01746, p90=0.01746, p95=0.01746, p99=0.01746
[I]                 ---- Values ----
                        [[0.01745954]]
[I]                 ---- Histogram ----
                    Bin Range          |  Num Elems | Visualization
                    (-0.483 , -0.383 ) |          0 | 
                    (-0.383 , -0.283 ) |          0 | 
                    (-0.283 , -0.183 ) |          0 | 
                    (-0.183 , -0.0825) |          0 | 
                    (-0.0825, 0.0175 ) |          1 | ########################################
                    (0.0175 , 0.117  ) |          0 | 
                    (0.117  , 0.217  ) |          0 | 
                    (0.217  , 0.317  ) |          0 | 
                    (0.317  , 0.417  ) |          0 | 
                    (0.417  , 0.517  ) |          0 | 
[E]         FAILED | Output: 'presence_logits' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'pred_masks' (dtype=float32, shape=(1, 200, 288, 288)) with 'pred_masks' (dtype=float32, shape=(1, 200, 288, 288))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:40:43: pred_masks | Stats: mean=-8.1487, std-dev=6.935, var=48.094, median=-6.7954, min=-66.464 at (0, 181, 185, 143), max=10.437 at (0, 172, 88, 5), avg-magnitude=8.1915, p90=-1.3854, p95=-0.60388, p99=0.57092
[I]             ---- Histogram ----
                Bin Range      |  Num Elems | Visualization
                (-98.1, -86.9) |          0 | 
                (-86.9, -75.7) |          0 | 
                (-75.7, -64.6) |          8 | 
                (-64.6, -53.4) |       2408 | 
                (-53.4, -42.3) |      83631 | 
                (-42.3, -31.1) |     303875 | #
                (-31.1, -20  ) |     458436 | #
                (-20  , -8.82) |    4582663 | ################
                (-8.82, 2.34 ) |   11118718 | ########################################
                (2.34 , 13.5 ) |      39061 | 
[I]         onnxrt-runner-N0-01/05/26-15:40:43: pred_masks | Stats: mean=-9.7259, std-dev=7.9383, var=63.017, median=-8.146, min=-98.052 at (0, 115, 204, 146), max=13.489 at (0, 181, 216, 96), avg-magnitude=9.7593, p90=-2.0298, p95=-0.65509, p99=0.4114
[I]             ---- Histogram ----
                Bin Range      |  Num Elems | Visualization
                (-98.1, -86.9) |         13 | 
                (-86.9, -75.7) |        388 | 
                (-75.7, -64.6) |       9587 | 
                (-64.6, -53.4) |      44799 | 
                (-53.4, -42.3) |     110798 | 
                (-42.3, -31.1) |     294532 | #
                (-31.1, -20  ) |     788502 | ###
                (-20  , -8.82) |    6219156 | ###########################
                (-8.82, 2.34 ) |    9089919 | ########################################
                (2.34 , 13.5 ) |      31106 | 
[I]         Error Metrics: pred_masks
[I]             Minimum Required Tolerance: elemwise error | [abs=75.517] OR [rel=3.4339e+06] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=4.0639, std-dev=4.7233, var=22.309, median=2.703, min=4.7684e-07 at (0, 75, 94, 286), max=75.517 at (0, 32, 222, 171), avg-magnitude=4.0639, p90=8.9541, p95=12.691, p99=23.596
[I]                 ---- Histogram ----
                    Bin Range        |  Num Elems | Visualization
                    (4.77e-07, 7.55) |   14313912 | ########################################
                    (7.55    , 15.1) |    1708268 | ####
                    (15.1    , 22.7) |     376164 | #
                    (22.7    , 30.2) |     125679 | 
                    (30.2    , 37.8) |      39442 | 
                    (37.8    , 45.3) |      14851 | 
                    (45.3    , 52.9) |       7807 | 
                    (52.9    , 60.4) |       2332 | 
                    (60.4    , 68  ) |        332 | 
                    (68      , 75.5) |         13 | 
[I]             Relative Difference | Stats: mean=2.6174, std-dev=1307.3, var=1.7091e+06, median=0.3569, min=7.3301e-08 at (0, 131, 160, 46), max=3.4339e+06 at (0, 186, 205, 139), avg-magnitude=2.6174, p90=1.115, p95=1.9776, p99=10.807
[I]                 ---- Histogram ----
                    Bin Range            |  Num Elems | Visualization
                    (7.33e-08, 3.43e+05) |   16588791 | ########################################
                    (3.43e+05, 6.87e+05) |          4 | 
                    (6.87e+05, 1.03e+06) |          0 | 
                    (1.03e+06, 1.37e+06) |          1 | 
                    (1.37e+06, 1.72e+06) |          0 | 
                    (1.72e+06, 2.06e+06) |          2 | 
                    (2.06e+06, 2.4e+06 ) |          1 | 
                    (2.4e+06 , 2.75e+06) |          0 | 
                    (2.75e+06, 3.09e+06) |          0 | 
                    (3.09e+06, 3.43e+06) |          1 | 
[E]         FAILED | Output: 'pred_masks' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[I]     Comparing Output: 'semantic_seg' (dtype=float32, shape=(1, 1, 288, 288)) with 'semantic_seg' (dtype=float32, shape=(1, 1, 288, 288))
[I]         Tolerance: [abs=1e-08, rel=1e-05] | Checking elemwise error
[I]         trt-runner-N0-01/05/26-15:40:43: semantic_seg | Stats: mean=-6.3962, std-dev=6.0442, var=36.532, median=-8.6375, min=-33.608 at (0, 0, 172, 182), max=14.36 at (0, 0, 50, 152), avg-magnitude=8.7164, p90=7.8336, p95=8.4555, p99=9.4093
[I]             ---- Histogram ----
                Bin Range      |  Num Elems | Visualization
                (-33.6, -28.8) |          2 | 
                (-28.8, -23.9) |          9 | 
                (-23.9, -19  ) |         32 | 
                (-19  , -14.2) |        342 | 
                (-14.2, -9.34) |      13615 | #########
                (-9.34, -4.48) |      56761 | ########################################
                (-4.48, 0.372) |        357 | 
                (0.372, 5.23 ) |        273 | 
                (5.23 , 10.1 ) |      11188 | #######
                (10.1 , 14.9 ) |        365 | 
[I]         onnxrt-runner-N0-01/05/26-15:40:43: semantic_seg | Stats: mean=-6.6537, std-dev=6.5555, var=42.974, median=-9.0476, min=-23.973 at (0, 0, 66, 29), max=14.935 at (0, 0, 50, 152), avg-magnitude=9.2668, p90=8.8604, p95=9.5186, p99=10.567
[I]             ---- Histogram ----
                Bin Range      |  Num Elems | Visualization
                (-33.6, -28.8) |          0 | 
                (-28.8, -23.9) |          2 | 
                (-23.9, -19  ) |         21 | 
                (-19  , -14.2) |        314 | 
                (-14.2, -9.34) |      29924 | #############################
                (-9.34, -4.48) |      40523 | ########################################
                (-4.48, 0.372) |        321 | 
                (0.372, 5.23 ) |        242 | 
                (5.23 , 10.1 ) |       9899 | #########
                (10.1 , 14.9 ) |       1698 | #
[I]         Error Metrics: semantic_seg
[I]             Minimum Required Tolerance: elemwise error | [abs=13.961] OR [rel=424.91] (requirements may be lower if both abs/rel tolerances are set)
[I]             Absolute Difference | Stats: mean=0.69968, std-dev=0.64282, var=0.41322, median=0.54099, min=9.5367e-07 at (0, 0, 36, 286), max=13.961 at (0, 0, 203, 178), avg-magnitude=0.69968, p90=1.4914, p95=1.8559, p99=2.8235
[I]                 ---- Histogram ----
                    Bin Range        |  Num Elems | Visualization
                    (9.54e-07, 1.4 ) |      72984 | ########################################
                    (1.4     , 2.79) |       9092 | ####
                    (2.79    , 4.19) |        677 | 
                    (4.19    , 5.58) |        129 | 
                    (5.58    , 6.98) |         36 | 
                    (6.98    , 8.38) |          8 | 
                    (8.38    , 9.77) |          9 | 
                    (9.77    , 11.2) |          4 | 
                    (11.2    , 12.6) |          2 | 
                    (12.6    , 14  ) |          3 | 
[I]             Relative Difference | Stats: mean=0.084154, std-dev=1.4902, var=2.2207, median=0.058771, min=9.8925e-08 at (0, 0, 36, 286), max=424.91 at (0, 0, 220, 148), avg-magnitude=0.084154, p90=0.15674, p95=0.19453, p99=0.30675
[I]                 ---- Histogram ----
                    Bin Range        |  Num Elems | Visualization
                    (9.89e-08, 42.5) |      82943 | ########################################
                    (42.5    , 85  ) |          0 | 
                    (85      , 127 ) |          0 | 
                    (127     , 170 ) |          0 | 
                    (170     , 212 ) |          0 | 
                    (212     , 255 ) |          0 | 
                    (255     , 297 ) |          0 | 
                    (297     , 340 ) |          0 | 
                    (340     , 382 ) |          0 | 
                    (382     , 425 ) |          1 | 
[E]         FAILED | Output: 'semantic_seg' | Difference exceeds tolerance (rel=1e-05, abs=1e-08)
[E]     FAILED | Mismatched outputs: ['vision_feature_0', 'vision_feature_1', 'vision_feature_2', 'text_features', 'encoder_hidden_state', 'decoder_hidden_states', 'pred_boxes', 'pred_logits', 'presence_logits', 'pred_masks', 'semantic_seg']
[E] Accuracy Summary | trt-runner-N0-01/05/26-15:40:43 vs. onnxrt-runner-N0-01/05/26-15:40:43 | Passed: 0/1 iterations | Pass Rate: 0.0%
Traceback (most recent call last):
  File "/home/zy/open_project/SAM3-TensorRT/python/polygraphy_test.py", line 52, in <module>
    main()
  File "/home/zy/open_project/SAM3-TensorRT/python/polygraphy_test.py", line 44, in main
    assert bool(Comparator.compare_accuracy(run_results, compare_func=CompareFunc.simple(atol=1e-8)))
AssertionError

At the very begining, the vision_feature and text feature brings big difference.
I don't know which ops brings such a big difference.

[dataplayer12]

@Huntersdeng thank you for raising the issue. While exporting the TRT graph, currently we fold the tokenizer output into the graph so that the prompt is fixed for a given engine. So, please make sure that the value of prompt is same on this line and this line. The default values are different, one is dog, another is snow which is why you may have seen this difference. So, can you please make them the same, run the script and post 3 images: original, semantic segmentation output from python, semantic segmentation output from TensorRT.

[dataplayer12]

When I test with the same prompt, both pathways produce at least visually same result. I havent tested with polygraphy yet.

TensorRT

PyTorch

[dataplayer12]

Also, while playing with your picture, I discovered #8
Please use the workaround if you encounter it. I will fix it when I get time

[Huntersdeng]

Thank you for your reply.
I haven't try your inference code because I also want to get box and score. Therefore, I use my own inference code in python, and I got these results:
pytorch:

onnx: TensorRT: Though the mask looks similar, the box and score is not the same.

[dataplayer12]

@Huntersdeng how are you pre processing and post processing the image in python? The model outputs only masks and boxes scores are calculated by hugging face post processing libraries. If the mask from trt graph is the same, any discrepancy is most likely due to the way your application is post processing the masks

[Huntersdeng]

well, I use the official api:
Preprocess:

image = Image.open("/home/zy/Downloads/0069656e59962cccacb28582d8d36030.jpeg").convert("RGB")
pixel_values = processor(images=image, return_tensors="pt").to("cpu")["pixel_values"]

Postprocess:

results = processor.post_process_instance_segmentation(sam3_output, 0.3, 0.5, [image.size[::-1]])[0]

[Huntersdeng]

Sorry to confuse you. Let me clarify:
First I modify the Sam3ONNXWrapper in onnxexport.py because I also need boxes and scores:

class Sam3ONNXWrapper(torch.nn.Module):
    def __init__(self, sam3, input_ids, attention_mask):
        super().__init__()
        self.sam3 = sam3
        self.register_buffer("const_input_ids", input_ids.to(torch.int64).cpu())
        self.register_buffer("const_attention_mask", attention_mask.to(torch.int64).cpu())

    def forward(self, pixel_values):
        outputs = self.sam3(
            pixel_values=pixel_values,
            input_ids=self.const_input_ids,
            attention_mask=self.const_attention_mask,
        )
        print(outputs.keys())
        # Typical useful outputs
        pred_boxes = outputs.pred_boxes
        pred_logits = outputs.pred_logits
        pred_masks = outputs.pred_masks
        presence_score = outputs.presence_logits
        return (pred_boxes, pred_logits, pred_masks, presence_score)

By running it, I get a onnx model like this:

Then I use trtexec to export it to TensorRT:

/usr/src/tensorrt/bin/trtexec --onnx=onnx_weights/sam3_static.onnx --saveEngine=onnx_weights/sam3.plan --verbose --fp16

And I write python inference codes, like this:

import tensorrt as trt
import pycuda.driver as cuda
import pycuda.autoinit
import numpy as np
import torch
from transformers.models.sam3 import Sam3Processor
from transformers.models.sam3.modeling_sam3 import Sam3ImageSegmentationOutput
from PIL import Image
import matplotlib.pyplot as plt
import matplotlib.patches as patches
from matplotlib.colors import to_rgba

TRT_LOGGER = trt.Logger(trt.Logger.INFO)
TRT_LOGGER.min_severity = trt.Logger.Severity.INFO


class HostDeviceMem(object):
    def __init__(self, name, host_mem, device_mem, shape):
        self.name = name
        self.host = host_mem
        self.device = device_mem
        self.shape = shape

    def __str__(self):
        return "Host:\n" + str(self.host) + "\nDevice:\n" + str(self.device)

    def __repr__(self):
        return self.__str__()


class AiEngineTrt:
    def __init__(self, model_path) -> None:
        with open(model_path, "rb") as f, trt.Runtime(TRT_LOGGER) as runtime:
            self.engine = runtime.deserialize_cuda_engine(f.read())
        self.context = self.engine.create_execution_context()
        self.inputs, self.outputs, self.bindings, self.stream = self.allocate_buffers(self.engine)

    def run(self, input_datas):
        if (len(input_datas) != len(self.inputs)):
            print(len(self.inputs))
            raise ValueError("input data size not match")
        for i in range(len(input_datas)):
            np.copyto(self.inputs[i].host, input_datas[i].reshape(-1).ravel())
        t_outputs = self.do_inference(
            self.context,
            bindings=self.bindings,
            inputs=self.inputs,
            outputs=self.outputs,
            stream=self.stream,
        )
        return t_outputs

    def allocate_buffers(self, engine):
        inputs = []
        outputs = []
        bindings = []
        stream = cuda.Stream()
        for binding in engine:
            print(binding)
            shape = engine.get_tensor_shape(binding)
            size = trt.volume(shape)
            dtype = trt.nptype(engine.get_tensor_dtype(binding))
            # Allocate host and device buffers
            host_mem = cuda.pagelocked_empty(size, dtype)
            device_mem = cuda.mem_alloc(host_mem.nbytes)
            # Append the device buffer to device bindings.
            bindings.append(int(device_mem))
            # Append to the appropriate list.
            if engine.get_tensor_mode(binding) == trt.TensorIOMode.INPUT:
                inputs.append(HostDeviceMem(binding, host_mem, device_mem, shape))
            else:
                outputs.append(HostDeviceMem(binding, host_mem, device_mem, shape))
            self.context.set_tensor_address(binding,  device_mem)
        return inputs, outputs, bindings, stream

    def do_inference(self, context, bindings, inputs, outputs, stream):
        # Transfer input data to the GPU.
        [cuda.memcpy_htod_async(inp.device, inp.host, stream) for inp in inputs]
        # Run inference.
        context.execute_async_v3(stream_handle=stream.handle)
        # Transfer predictions back from the GPU.
        [cuda.memcpy_dtoh_async(out.host, out.device, stream) for out in outputs]
        # Synchronize the stream
        stream.synchronize()
        # Return only the host outputs.
        return {out.name: out.host.reshape(out.shape) for out in outputs}
    
def vis_instance_seg(image,
                     boxes,
                     scores,
                     masks,
                     class_names=None,
                     score_thresh=0.5,
                     mask_alpha=0.45,
                     figsize=(10, 10)):
    """
    image      : (H, W, 3)  uint8
    boxes      : (N, 4)     xyxy
    scores     : (N,)
    masks      : (N, H, W) 
    class_names: List[str]
    """

    image = image.astype(np.uint8)
    keep = scores >= score_thresh
    boxes, scores, masks = boxes[keep], scores[keep], masks[keep]
    N = len(boxes)

    cmap = plt.get_cmap('tab20')
    colors = [cmap(i % 20) for i in range(N)]

    fig, ax = plt.subplots(1, 1, figsize=figsize)
    ax.imshow(image)
    ax.axis('off')

    for i in range(N):
        x1, y1, x2, y2 = boxes[i]
        rect = patches.Rectangle((x1, y1), x2-x1, y2-y1,
                                 linewidth=2, edgecolor=colors[i], facecolor='none')
        ax.add_patch(rect)

        label = f'{class_names[i] if class_names else i}: {scores[i]:.2f}'
        ax.text(x1, y1-5, label, color='white', fontsize=9,
                bbox=dict(boxstyle="square,pad=0.1",
                          facecolor=colors[i], edgecolor='none'))

        mask = masks[i] > 0.5         
        rgba = np.zeros((*mask.shape, 4))
        rgba[mask] = to_rgba(colors[i], alpha=mask_alpha)
        ax.imshow(rgba, origin='upper')

    plt.tight_layout()
    plt.show()
    return fig, ax


if __name__ == '__main__':
    processor = Sam3Processor.from_pretrained("/home/zy/weights/sam3")
    image = Image.open("/home/zy/Downloads/0069656e59962cccacb28582d8d36030.jpeg").convert("RGB")
    pixel_values = processor(images=image, return_tensors="pt").to("cpu")["pixel_values"].numpy()
    print(f"pixel_value: {pixel_values.shape}")
    trt_model = AiEngineTrt("/home/zy/open_project/SAM3-TensorRT/onnx_weights/sam3.plan")
    outputs = trt_model.run([pixel_values])
    for key, output in outputs.items():
        print(f"{key}: {output.shape}")
        print(f"min: {np.min(output)}; max: {np.max(output)}")
    sam3_output = Sam3ImageSegmentationOutput(
        pred_boxes=torch.tensor(outputs["pred_boxes"]),
        pred_logits=torch.tensor(outputs["pred_logits"]),
        pred_masks=torch.tensor(outputs["pred_masks"]),
        presence_logits=torch.tensor(outputs["presence_score"])
    )

    results = processor.post_process_instance_segmentation(sam3_output, 0.3, 0.5, [image.size[::-1]])[0]
    print(results)

    vis_instance_seg(np.array(image), results["boxes"], results["scores"], results["masks"], ["dog"])

And I get inference result:

In conclusion, I use the Sam3Processor to preprocess and postprocess, but get different results between pytorch and TensorRT. So some gaps must happen in the TensorRT model.

[dataplayer12]

As it stands the binding boxes issue is outside the scope of the project. I might incorporate binding boxes into the cpp version some day at which point the issue will fall within the scope of the repo. Since you are writing most of your own code and my analysis has not shown the issue you are facing, I'm not sure how much I can help.

[dataplayer12]

For what its worth, I have fixed the issue I found while working with your image. Issue: #8 , PR #9 . This does not address the issue you are having about apparently inconsistent engine outputs which I have not been able to reproduce with my code.