Adds variable to track random operators

CHANGELOGS.rst

@@ -4,6 +4,8 @@ Change Logs
 0.8.1
 +++++
 
+* :pr:`286`: adds variable to track random nodes in models
+
 0.8.0
 +++++
 

_unittests/ut_export/test_shape_helper.py

@@ -106,10 +106,15 @@ def test_all_dynamic_shape_all_transformers_cache(self):
         ]
         with torch_export_patches(patch_transformers=True):
             for cache, exds in caches:
-                with self.subTest(cache_name=cache.__class__.__name__):
+                with self.subTest(cache_name=cache.__class__.__name__, patch=True):
                     ds = all_dynamic_shapes_from_inputs(cache)
                     self.assertEqual(exds, ds)
 
+        for cache, exds in caches:
+            with self.subTest(cache_name=cache.__class__.__name__, patch=False):
+                ds = all_dynamic_shapes_from_inputs(cache)
+                self.assertEqual(exds, ds)
+
     @requires_transformers("4.52")
     @requires_torch("2.7.99")
     def test_all_dynamic_shapes_from_inputs(self):

_unittests/ut_helpers/test_args_helper.py

@@ -1,9 +1,12 @@
 import unittest
 from onnx_diagnostic.ext_test_case import ExtTestCase
-from onnx_diagnostic.helpers.args_helper import get_parsed_args
+from onnx_diagnostic.helpers.args_helper import get_parsed_args, check_cuda_availability
 
 
 class TestHelpers(ExtTestCase):
+    def test_check_cuda_availability(self):
+        check_cuda_availability()
+
     def test_args(self):
         try:
             args = get_parsed_args(

_unittests/ut_helpers/test_bench_run.py

@@ -9,11 +9,15 @@
     get_machine,
     make_configs,
     run_benchmark,
+    _clean_string,
 )
 from onnx_diagnostic.helpers.cache_helper import make_dynamic_cache, CacheKeyValue
 
 
 class TestBenchRun(ExtTestCase):
+    def test__clean_string(self):
+        self.assertEqual("r", _clean_string("r"))
+
     def test_reg(self):
         text = ":m,6;"
         m = _extract_metrics(text)

_unittests/ut_helpers/test_graph_helper.py

@@ -33,6 +33,8 @@ def test_computation_order(self):
             proto.graph.node, [i.name for i in [*proto.graph.input, *proto.graph.initializer]]
         )
         self.assertEqual([1, 2, 3], order)
+        gr = GraphRendering(proto)
+        self.assertEqual(repr(gr), "GraphRendering(<ModelProto>)")
 
     def test_graph_positions1(self):
         proto = oh.make_model(

_unittests/ut_helpers/test_helper.py

@@ -62,19 +62,21 @@ def test_string_type(self):
         s = string_type(obj)
         self.assertEqual(s, "dict(a:A7r1,b:#1[float],c:(int,))")
 
+    @hide_stdout()
     def test_string_dict(self):
         a = np.array([1], dtype=np.float32)
         obj = {"a": a, "b": {"r": 5.6}, "c": {1}}
-        s = string_type(obj)
+        s = string_type(obj, verbose=10)
         self.assertEqual(s, "dict(a:A1r1,b:dict(r:float),c:{int})")
 
+    @hide_stdout()
     def test_string_type_array(self):
         a = np.array([1], dtype=np.float32)
         t = torch.tensor([1])
         obj = {"a": a, "b": t}
-        s = string_type(obj, with_shape=False)
+        s = string_type(obj, with_shape=False, verbose=10)
         self.assertEqual(s, "dict(a:A1r1,b:T7r1)")
-        s = string_type(obj, with_shape=True)
+        s = string_type(obj, with_shape=True, verbose=10)
         self.assertEqual(s, "dict(a:A1s1,b:T7s1)")
 
     def test_string_sig_f(self):
@@ -92,6 +94,17 @@ def __init__(self, a, b=3, c=4, e=5):
         ssig = string_sig(A(1, c=8))
         self.assertEqual(ssig, "A(a=1, c=8)")
 
+    def test_ort_value(self):
+        import onnxruntime as onnxrt
+
+        numpy_arr_input = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32)
+        ortvalue = onnxrt.OrtValue.ortvalue_from_numpy(numpy_arr_input)
+        self.assertEqual("OV1r2", string_type(ortvalue))
+        self.assertEqual("OV1s3x2", string_type(ortvalue, with_shape=True))
+        self.assertEqual(
+            "OV(NO-NUMPY:FIXIT)", string_type(ortvalue, with_shape=True, with_min_max=True)
+        )
+
     def test_pretty_onnx(self):
         proto = oh.make_model(
             oh.make_graph(
@@ -122,7 +135,8 @@ def test_print_pretty_onnx(self):
                 [
                     oh.make_node("Sigmoid", ["Y"], ["sy"]),
                     oh.make_node("Mul", ["Y", "sy"], ["ysy"]),
-                    oh.make_node("Mul", ["X", "ysy"], ["final"]),
+                    oh.make_node("Cast", ["ysy"], ["ysyy"], to=1),
+                    oh.make_node("Mul", ["X", "ysyy"], ["final"]),
                 ],
                 "-nd-",
                 [

_unittests/ut_helpers/test_log_helper.py

@@ -183,6 +183,7 @@ def test_cube_logs_excel(self):
                     ["version.*", "model.*"],
                     ["time_latency", "time_baseline"],
                     key_agg=["model_name"],
+                    plots=True,
                 ),
             },
             verbose=1,
@@ -268,7 +269,7 @@ def test_cube_logs_performance_cube_time(self):
         cube = CubeLogsPerformance(dfs, keep_last_date=True)
         cube.load()
         ct = cube.clone()
-        self.assertEqual((52, 116), ct.shape)
+        self.assertEqual((52, 117), ct.shape)
 
     def test_duplicate(self):
         df = pandas.DataFrame(
@@ -523,6 +524,14 @@ def test_cube_sbs_no_time(self):
         self.assertEqual(sbs_agg.shape, (2, 11))
         self.assertEqual(sbs_agg.index.names, ["date", "METRICS"])
         self.assertEqual(sorted(sbs_agg.columns.names), ["CONF", "exporter"])
+        output = self.get_dump_file("test_cube_sbs_no_time.xlsx")
+        cube.to_excel(
+            output,
+            views=["time_p"],
+            time_mask=True,
+            verbose=0,
+            sbs=dict(CFA=dict(exporter="E1", opt="O"), CFB=dict(exporter="E2", opt="O")),
+        )
 
     def test_cube_sbs_with_time(self):
         df = pandas.DataFrame(

_unittests/ut_helpers/test_memory_peak.py

@@ -9,10 +9,22 @@
     ignore_warnings,
     requires_cuda,
 )
-from onnx_diagnostic.helpers.memory_peak import get_memory_rss, start_spying_on
+from onnx_diagnostic.helpers.memory_peak import get_memory_rss, start_spying_on, Monitor
 
 
 class TestMemoryPeak(ExtTestCase):
+    def test_basic(self):
+        m = Monitor()
+        self.assertEqual(
+            repr(m),
+            "Monitor(begin=0, end=0, peak=0, average=0, n=0, d_end=0, d_peak=0, d_avg=0)",
+        )
+        m.update(1)
+        self.assertEqual(
+            repr(m),
+            "Monitor(begin=1, end=1, peak=1, average=1, n=1, d_end=0, d_peak=0, d_avg=0.0)",
+        )
+
     @skipif_ci_apple("stuck")
     def test_memory(self):
         mem = get_memory_rss(os.getpid())

_unittests/ut_helpers/test_mini_onnx_builder.py

@@ -1,18 +1,27 @@
 import unittest
 import numpy as np
+import onnx
 import torch
 from onnx_diagnostic.ext_test_case import ExtTestCase
 from onnx_diagnostic.reference import ExtendedReferenceEvaluator
 from onnx_diagnostic.helpers.mini_onnx_builder import (
     create_onnx_model_from_input_tensors,
     create_input_tensors_from_onnx_model,
+    proto_from_array,
     MiniOnnxBuilder,
 )
 from onnx_diagnostic.helpers.cache_helper import make_dynamic_cache, CacheKeyValue
 from onnx_diagnostic.helpers import string_type
 
 
 class TestMiniOnnxBuilder(ExtTestCase):
+    def test_proto_from_array(self):
+        self.assertRaise(lambda: proto_from_array(None), TypeError)
+        t = torch.tensor([[0, 2.0], [3, 0]]).to_sparse()
+        self.assertRaise(lambda: proto_from_array(t), NotImplementedError)
+        tp = proto_from_array(torch.tensor([[0, 2.0], [3, 0]]).to(torch.bfloat16))
+        self.assertEqual(tp.data_type, onnx.TensorProto.BFLOAT16)
+
     def test_mini_onnx_builder_sequence_onnx(self):
         builder = MiniOnnxBuilder()
         builder.append_output_sequence("name", [np.array([6, 7])])
@@ -31,6 +40,25 @@ def test_mini_onnx_builder_sequence_ort(self):
         got = ref.run(None, {})
         self.assertEqualAny([np.array([6, 7])], got[0])
 
+    def test_mini_onnx_builder_sequence_ort_randomize(self):
+        from onnxruntime import InferenceSession
+
+        builder = MiniOnnxBuilder()
+        builder.append_output_initializer(
+            "name1", np.array([6, 7], dtype=np.float32), randomize=True
+        )
+        builder.append_output_initializer(
+            "name2", np.array([-6, 7], dtype=np.float32), randomize=True
+        )
+        onx = builder.to_onnx()
+        ref = InferenceSession(onx.SerializeToString(), providers=["CPUExecutionProvider"])
+        got = ref.run(None, {})
+        self.assertEqual((2,), got[0].shape)
+        self.assertEqual(np.float32, got[0].dtype)
+        self.assertGreaterOrEqual(got[0].min(), 0)
+        self.assertEqual((2,), got[1].shape)
+        self.assertEqual(np.float32, got[1].dtype)
+
     def test_mini_onnx_builder(self):
         data = [
             (

_unittests/ut_helpers/test_onnx_helper.py

@@ -18,6 +18,7 @@
     tensor_statistics,
     enumerate_results,
     shadowing_names,
+    onnx_dtype_name,
 )
 
 
@@ -295,6 +296,7 @@ def test_enumerate_results(self):
         self.assertEqual(2, len(list(enumerate_results(model, "X", verbose=2))))
         self.assertEqual(2, len(list(enumerate_results(model, "Z", verbose=2))))
 
+    @hide_stdout()
     def test_enumerate_results_loop(self):
         x = np.array([1, 2, 3, 4, 5]).astype(np.float32)
 
@@ -467,6 +469,13 @@ def _mkv_(name):
             shadowing_names(model),
         )
 
+    def test_onnx_dtype_name(self):
+        for k in dir(TensorProto):
+            if k.upper() == k and k not in {"DESCRIPTOR", "EXTERNAL"}:
+                self.assertEqual(k, onnx_dtype_name(getattr(TensorProto, k)))
+        self.assertRaise(lambda: onnx_dtype_name(1000), ValueError)
+        self.assertEqual(onnx_dtype_name(1000, exc=False), "UNEXPECTED")
+
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)

_unittests/ut_helpers/test_ort_session.py

@@ -295,6 +295,22 @@ def test_init_torch_bfloat16(self):
         self.assertIsInstance(got[0], torch.Tensor)
         self.assertEqualArray(expected[0], got[0])
 
+    def test_profiling(self):
+        model, feeds, expected = self._get_model_init(onnx.TensorProto.BFLOAT16)
+        wrap = InferenceSessionForTorch(
+            model,
+            providers="cpu",
+            graph_optimization_level=False,
+            enable_profiling=True,
+            optimized_model_filepath=self.get_dump_file("test_init_torch_bfloat16.onnx"),
+            log_severity_level=2,
+            log_verbosity_level=2,
+            disable_aot_function_inlining=1,
+        )
+        got = wrap.run(None, feeds)
+        self.assertIsInstance(got[0], torch.Tensor)
+        self.assertEqualArray(expected[0], got[0])
+
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)

_unittests/ut_helpers/test_rt_helper.py

@@ -1,5 +1,6 @@
 import os
 import unittest
+import onnx
 import torch
 from onnx_diagnostic.ext_test_case import (
     ExtTestCase,
@@ -12,6 +13,7 @@
     onnx_generate,
     generate_and_validate,
     onnx_generate_with_genai,
+    name_type_to_onnx_dtype,
 )
 from onnx_diagnostic.torch_models.hghub import get_untrained_model_with_inputs
 from onnx_diagnostic.torch_export_patches import torch_export_patches
@@ -76,6 +78,12 @@ def test_onnx_generate(self):
         self.assertNotEmpty(session)
         self.assertEqualArray(expected, res)
 
+    def test_name_type_to_onnx_dtype(self):
+        for name in ["int64", "int32", "int64", "float16", "float", "double", "bfloat16"]:
+            look = f"tensor({name})"
+            expected = getattr(onnx.TensorProto, name.upper())
+            self.assertEqual(expected, name_type_to_onnx_dtype(look))
+
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)

_unittests/ut_torch_onnx/test_sbs.py

@@ -48,7 +48,7 @@ def forward(self, x):
                 verbose=1,
             ),
         )
-        self.assertEqual(len(results), 4)
+        self.assertEqual(len(results), 5)
 
     @hide_stdout()
     @ignore_warnings((DeprecationWarning, FutureWarning, UserWarning))

onnx_diagnostic/_command_lines_parser.py

@@ -940,7 +940,7 @@ def get_parser_agg() -> ArgumentParser:
         "n_model_faster2x,n_model_faster3x,n_model_faster4x,n_node_attention,"
         "n_node_attention23,n_node_rotary_embedding,n_node_rotary_embedding23,"
         "n_node_gqa,n_node_layer_normalization,n_node_layer_normalization23,"
-        "peak_gpu_torch,peak_gpu_nvidia,n_node_control_flow,"
+        "peak_gpu_torch,peak_gpu_nvidia,n_node_control_flow,n_node_random,"
         "n_node_constant,n_node_shape,n_node_expand,"
         "n_node_function,n_node_initializer,n_node_scatter,"
         "time_export_unbiased,onnx_n_nodes_no_cst,n_node_initializer_small",

onnx_diagnostic/helpers/log_helper.py

@@ -1169,7 +1169,8 @@ def to_excel(
             assuming they should remain stale
         :param sbs: configurations to compare side-by-side, this adds two tabs,
             one gathering raw data about the two configurations, the other one
-            is aggregated by metrics
+            is aggregated by metrics, example:
+            ``=dict(CFA=dict(exporter="E1", opt="O"), CFB=dict(exporter="E2", opt="O"))``
         """
         if verbose:
             print(f"[CubeLogs.to_excel] create Excel file {output}, shape={self.shape}")
@@ -1611,6 +1612,7 @@ def __init__(
             "n_node_initializer_small",
             "n_node_layer_normalization",
             "n_node_layer_normalization23",
+            "n_node_random",
             "n_node_reshape",
             "n_node_rotary_embedding",
             "n_node_rotary_embedding23",
@@ -1802,6 +1804,16 @@ def first_err(df: pandas.DataFrame) -> pandas.Series:
                     + gdf(df, "op_onnx__InstanceNormlization", 0)
                     + gdf(df, "op_onnx__GroupNormalization", 0),
                 ),
+                n_node_random=lambda df: gpreserve(
+                    df,
+                    "time_latency_eager",
+                    gdf(df, "op_onnx__RandomNormal", 0)
+                    + gdf(df, "op_onnx__RandomNormalLike", 0)
+                    + gdf(df, "op_onnx__RandomUniform", 0)
+                    + gdf(df, "op_onnx__RandomUniformLike", 0)
+                    + gdf(df, "op_onnx__Multinomial", 0)
+                    + gdf(df, "op_onnx__Bernoulli", 0),
+                ),
                 n_node_attention=lambda df: gpreserve(
                     df,
                     "time_latency_eager",

onnx_diagnostic/helpers/memory_peak.py

@@ -47,6 +47,8 @@ def delta_end(self):
 
     @property
     def delta_avg(self):
+        if self.n_measures == 0:
+            return 0
         return self.average / self.n_measures - self.begin
 
     def __repr__(self):

onnx_diagnostic/helpers/mini_onnx_builder.py

@@ -52,7 +52,7 @@ def proto_from_array(
 
     tensor = TensorProto()
     tensor.dims.extend(arr_cpu.shape)
-    tensor.name = name
+    tensor.name = name or ""
     itype = dtype_to_tensor_dtype(arr_cpu.dtype)
     assert not hasattr(TensorProto, "INT4") or itype not in {
         TensorProto.INT4,

onnx_diagnostic/helpers/onnx_helper.py

@@ -331,7 +331,7 @@ def onnx_dtype_name(itype: int, exc: bool = True) -> str:
         print(onnx_dtype_name(7))
     """
     for k in dir(TensorProto):
-        if "FLOAT" in k or "INT" in k or "TEXT" in k or "BOOL" in k:
+        if k.upper() == k and k != "EXTERNAL":
             v = getattr(TensorProto, k)
             if v == itype:
                 return k