support for inputs3

Author: xadupre

CHANGELOGS.rst

@@ -4,7 +4,10 @@ Change Logs
 0.7.0
 +++++
 
-* :pr:`143`: compares intermediate results
+* :pr:`143`: compares intermediate results,
+  support for second inputs with different dimension,
+  rename test_helper into validate,
+  support ``interpolate_pos_encoding`` for ``VitModel``
 
 0.6.3
 +++++

_doc/api/torch_models/index.rst

@@ -7,7 +7,7 @@ onnx_diagnostic.torch_models
 
     hghub/index
     llms
-    test_helper
+    validate
 
 .. automodule:: onnx_diagnostic.torch_models
     :members:

_doc/api/torch_models/test_helper.rst

@@ -0,0 +1,7 @@
+
+onnx_diagnostic.torch_models.validate
+=====================================
+
+.. automodule:: onnx_diagnostic.torch_models.validate
+    :members:
+    :no-undoc-members:

_doc/api/torch_models/validate.rst

@@ -4,7 +4,7 @@
 ===================================================
 
 The command line is a wrapper around function
-:func:`onnx_diagnostic.torch_models.test_helper.validate_model`.
+:func:`onnx_diagnostic.torch_models.validate.validate_model`.
 
 Description
 +++++++++++
@@ -110,7 +110,7 @@ Run onnxruntime fusions
 
 This option runs `transformers optimizations <https://onnxruntime.ai/docs/performance/transformers-optimization.html>`_ 
 implemented in :epkg:`onnxruntime`. The list of supported ``model_type`` can be found in the documentation
-of function :func:`onnx_diagnostic.torch_models.test_helper.run_ort_fusion`.
+of function :func:`onnx_diagnostic.torch_models.validate.run_ort_fusion`.
 
 .. code-block::
 

_doc/cmds/validate.rst

@@ -11,7 +11,7 @@
     requires_onnxscript,
     requires_transformers,
 )
-from onnx_diagnostic.torch_models.test_helper import (
+from onnx_diagnostic.torch_models.validate import (
     get_inputs_for_task,
     validate_model,
     filter_inputs,
@@ -21,7 +21,7 @@
 from onnx_diagnostic.tasks import supported_tasks
 
 
-class TestTestHelper(ExtTestCase):
+class TestValidate(ExtTestCase):
     def test_get_inputs_for_task(self):
         fcts = supported_tasks()
         for task in self.subloop(sorted(fcts)):
@@ -221,14 +221,39 @@ def test_validate_model_modelbuilder(self):
             do_run=True,
             verbose=10,
             exporter="modelbuilder",
-            dump_folder="dump_test_validate_model_onnx_dynamo",
+            dump_folder="dump_test_validate_model_modelbuilder",
         )
         self.assertIsInstance(summary, dict)
         self.assertIsInstance(data, dict)
         self.assertLess(summary["disc_onnx_ort_run_abs"], 1e-4)
         onnx_filename = data["onnx_filename"]
         self.assertExists(onnx_filename)
 
+    @requires_torch("2.7")
+    @hide_stdout()
+    @ignore_warnings(FutureWarning)
+    @requires_transformers("4.51")
+    def test_validate_model_vit_model(self):
+        mid = "ydshieh/tiny-random-ViTForImageClassification"
+        summary, data = validate_model(
+            mid,
+            do_run=True,
+            verbose=10,
+            exporter="onnx-dynamo",
+            dump_folder="dump_test_validate_model_onnx_dynamo",
+            inputs2=True,
+        )
+        self.assertIsInstance(summary, dict)
+        self.assertIsInstance(data, dict)
+        self.assertLess(summary["disc_onnx_ort_run_abs"], 1e-3)
+        self.assertLess(summary["disc_onnx_ort_run2_abs"], 1e-3)
+        self.assertEqual("dict(pixel_values:A1s2x3x30x30)", summary["run_feeds_inputs"])
+        self.assertEqual("dict(pixel_values:A1s3x3x31x31)", summary["run_feeds_inputs2"])
+        self.assertEqual("#1[A1s2x2]", summary["run_output_inputs"])
+        self.assertEqual("#1[A1s3x2]", summary["run_output_inputs2"])
+        onnx_filename = data["onnx_filename"]
+        self.assertExists(onnx_filename)
+
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)

_unittests/ut_torch_models/test_test_helpers.py renamed to _unittests/ut_torch_models/test_validate.py

@@ -8,7 +8,7 @@
     requires_experimental,
     requires_transformers,
 )
-from onnx_diagnostic.torch_models.test_helper import validate_model
+from onnx_diagnostic.torch_models.validate import validate_model
 
 
 class TestValidateModel(ExtTestCase):

_unittests/ut_torch_models/test_validate_models.py

@@ -373,6 +373,13 @@ def get_parser_validate() -> ArgumentParser:
         action=BooleanOptionalAction,
         help="validate the trained model (requires downloading)",
     )
+    parser.add_argument(
+        "--inputs2",
+        default=True,
+        action=BooleanOptionalAction,
+        help="if run is on, the command lines validates the model on a "
+        "second set of inputs to check the exported model supports dynamism",
+    )
     parser.add_argument(
         "--runtime",
         choices=["onnxruntime", "torch", "ref"],
@@ -440,7 +447,7 @@ def get_parser_validate() -> ArgumentParser:
 
 def _cmd_validate(argv: List[Any]):
     from .helpers import string_type
-    from .torch_models.test_helper import get_inputs_for_task, validate_model
+    from .torch_models.validate import get_inputs_for_task, validate_model
     from .tasks import supported_tasks
 
     parser = get_parser_validate()
@@ -492,6 +499,7 @@ def _cmd_validate(argv: List[Any]):
             runtime=args.runtime,
             repeat=args.repeat,
             warmup=args.warmup,
+            inputs2=args.inputs2,
         )
         print("")
         print("-- summary --")

onnx_diagnostic/_command_lines_parser.py

@@ -52,7 +52,7 @@ def get_inputs(
         input_width, int
     ), f"Unexpected type for input_width {type(input_width)}{config}"
     assert isinstance(
-        input_width, int
+        input_height, int
     ), f"Unexpected type for input_height {type(input_height)}{config}"
 
     shapes = {
@@ -67,6 +67,9 @@ def get_inputs(
             -1, 1
         ),
     )
+    if model.__class__.__name__ == "ViTForImageClassification":
+        inputs["interpolate_pos_encoding"] = True
+        shapes["interpolate_pos_encoding"] = None
     res = dict(inputs=inputs, dynamic_shapes=shapes)
     if add_second_input:
         res["inputs2"] = get_inputs(

onnx_diagnostic/tasks/image_classification.py

@@ -4093,3 +4093,30 @@ def _ccached_microsoft_phi4_reasoning():
             "vocab_size": 200064,
         }
     )
+
+
+def _ccached_ydshieh_tiny_random_vit_for_image_classification():
+    "ydshieh/tiny-random-ViTForImageClassification"
+    return transformers.Phi3Config(
+        **{
+            "_name_or_path": ".temp/dummy/vit/ViTForImageClassification",
+            "architectures": ["ViTForImageClassification"],
+            "attention_probs_dropout_prob": 0.1,
+            "encoder_stride": 2,
+            "hidden_act": "gelu",
+            "hidden_dropout_prob": 0.1,
+            "hidden_size": 32,
+            "image_size": 30,
+            "initializer_range": 0.02,
+            "intermediate_size": 37,
+            "layer_norm_eps": 1e-12,
+            "model_type": "vit",
+            "num_attention_heads": 4,
+            "num_channels": 3,
+            "num_hidden_layers": 5,
+            "patch_size": 2,
+            "qkv_bias": true,
+            "torch_dtype": "float32",
+            "transformers_version": "4.24.0.dev0",
+        }
+    )