Skip to content

Commit 3974d97

Browse files
xaduprexadupre
authored
Simplified logs processing (#147)
* df * fix * help * add view agg * sig * status * fix * fix * fix issues * excelé * fix test * mymp
1 parent 60ffd12 commit 3974d97

File tree

8 files changed

+821
-19
lines changed

8 files changed

+821
-19
lines changed

‎CHANGELOGS.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ Change Logs
44
0.7.0
55
+++++
66

7+
* :pr:`147`: simplified log processing
78
* :pr:`146`: patch for IdeficsAttention, IdeficsEmbedding
89
* :pr:`145`: patch for _compute_dynamic_ntk_parameters (Phi3RotaryEmbedding)
910
* :pr:`144`: support for second inputs with different dimension,

‎_doc/api/helpers/index.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ onnx_diagnostic.helpers
1313
doc_helper
1414
graph_helper
1515
helper
16+
log_helper
1617
memory_peak
1718
mini_onnx_builder
1819
model_builder_helper

‎_doc/api/helpers/log_helper.rst‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
2+
onnx_diagnostic.helpers.log_helper
3+
==================================
4+
5+
.. automodule:: onnx_diagnostic.helpers.log_helper
6+
:members:
7+
:no-undoc-members:

‎_unittests/ut_helpers/test_log_helper.py‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
import io
2+
import os
3+
import textwrap
4+
import unittest
5+
import zipfile
6+
import pandas
7+
from onnx_diagnostic.ext_test_case import ExtTestCase, hide_stdout
8+
from onnx_diagnostic.helpers.log_helper import (
9+
CubeLogs,
10+
CubeViewDef,
11+
enumerate_csv_files,
12+
open_dataframe,
13+
)
14+
15+
16+
class TestLogHelper(ExtTestCase):
17+
@classmethod
18+
def df1(cls):
19+
return pandas.read_csv(
20+
io.StringIO(
21+
textwrap.dedent(
22+
"""
23+
date,version_python,version_transformers,model_name,model_exporter,time_load,time_latency,time_baseline,disc_ort,disc_ort2
24+
2025/01/01,3.13.3,4.52.4,phi3,export,0.5,0.1,0.1,1e-5,1e-5
25+
2025/01/02,3.13.3,4.52.4,phi3,export,0.6,0.11,0.1,1e-5,1e-5
26+
2025/01/01,3.13.3,4.52.4,phi4,export,0.5,0.1,0.105,1e-5,1e-5
27+
2025/01/01,3.12.3,4.52.4,phi4,onnx-dynamo,0.5,0.1,0.999,1e-5,1e-5
28+
"""
29+
)
30+
)
31+
)
32+
33+
@classmethod
34+
def cube1(cls, verbose=0):
35+
cube = CubeLogs(
36+
cls.df1(),
37+
recent=True,
38+
formulas={"speedup": lambda df: df["time_baseline"] / df["time_baseline"]},
39+
)
40+
return cube.load(verbose=verbose)
41+
42+
@hide_stdout()
43+
def test_cube_logs_load_df(self):
44+
df = self.df1()
45+
cube = CubeLogs(df)
46+
text = str(cube)
47+
self.assertIsInstance(text, str)
48+
self.assertRaise(lambda: cube.load(verbose=1), AssertionError)
49+
cube = CubeLogs(
50+
self.df1(),
51+
recent=True,
52+
formulas={"speedup": lambda df: df["time_baseline"] / df["time_baseline"]},
53+
)
54+
cube.load(verbose=1)
55+
text = str(cube)
56+
self.assertIsInstance(text, str)
57+
self.assertEqual((3, df.shape[1] + 1), cube.shape)
58+
self.assertEqual(set(cube.columns), {*df.columns, "speedup"})
59+
60+
@hide_stdout()
61+
def test_cube_logs_load_dfdf(self):
62+
df = self.df1()
63+
cube = CubeLogs([df, df], recent=True)
64+
cube.load(verbose=1)
65+
self.assertEqual((3, 10), cube.shape)
66+
67+
@hide_stdout()
68+
def test_cube_logs_load_list(self):
69+
cube = CubeLogs(
70+
[
71+
dict(
72+
date="1/1/2001",
73+
version_python="3.13",
74+
model_exporter="A",
75+
time_latency=5.6,
76+
),
77+
dict(
78+
date="1/1/2001",
79+
version_python="3.13",
80+
model_exporter="B",
81+
time_latency=5.7,
82+
),
83+
]
84+
)
85+
cube.load(verbose=1)
86+
self.assertEqual((2, 4), cube.shape)
87+
88+
def test_cube_logs_view_repr(self):
89+
v = CubeViewDef(["version.*", "model_name"], ["time_latency", "time_baseline"])
90+
r = repr(v)
91+
self.assertEqual(
92+
"CubeViewDef(key_index=['version.*', 'model_name'], "
93+
"values=['time_latency', 'time_baseline'])",
94+
r,
95+
)
96+
97+
@hide_stdout()
98+
def test_cube_logs_view(self):
99+
cube = self.cube1(verbose=1)
100+
view = cube.view(
101+
CubeViewDef(["version.*", "model_name"], ["time_latency", "time_baseline"])
102+
)
103+
self.assertEqual((3, 4), view.shape)
104+
self.assertEqual(
105+
[
106+
("time_baseline", "export"),
107+
("time_baseline", "onnx-dynamo"),
108+
("time_latency", "export"),
109+
("time_latency", "onnx-dynamo"),
110+
],
111+
list(view.columns),
112+
)
113+
self.assertEqual(
114+
[("3.12.3", "phi4"), ("3.13.3", "phi3"), ("3.13.3", "phi4")], list(view.index)
115+
)
116+
117+
view = cube.view(
118+
CubeViewDef(
119+
["version.*"], ["time_latency", "time_baseline"], order=["model_exporter"]
120+
)
121+
)
122+
self.assertEqual((2, 6), view.shape)
123+
self.assertEqual(
124+
[
125+
("time_baseline", "export", "phi3"),
126+
("time_baseline", "export", "phi4"),
127+
("time_baseline", "onnx-dynamo", "phi4"),
128+
("time_latency", "export", "phi3"),
129+
("time_latency", "export", "phi4"),
130+
("time_latency", "onnx-dynamo", "phi4"),
131+
],
132+
list(view.columns),
133+
)
134+
self.assertEqual(["3.12.3", "3.13.3"], list(view.index))
135+
136+
def test_cube_logs_view_agg(self):
137+
cube = self.cube1(verbose=0)
138+
view = cube.view(
139+
CubeViewDef(
140+
["version.*", "model.*"],
141+
["time_latency", "time_baseline"],
142+
key_agg=["model_name"],
143+
)
144+
)
145+
self.assertEqual((2, 2), view.shape)
146+
self.assertEqual(["time_baseline", "time_latency"], list(view.columns))
147+
self.assertEqual([("3.13.3", "export"), ("3.12.3", "onnx-dynamo")], list(view.index))
148+
149+
@hide_stdout()
150+
def test_cube_logs_excel(self):
151+
output = self.get_dump_file("test_cube_logs_excel.xlsx")
152+
cube = self.cube1(verbose=0)
153+
cube.to_excel(
154+
output,
155+
{
156+
"example": CubeViewDef(
157+
["version.*", "model_name"], ["time_latency", "time_baseline"]
158+
),
159+
"agg": CubeViewDef(
160+
["version.*", "model.*"],
161+
["time_latency", "time_baseline"],
162+
key_agg=["model_name"],
163+
),
164+
},
165+
verbose=1,
166+
)
167+
self.assertExists(output)
168+
169+
def test_enumerate_csv_files(self):
170+
df = self.df1()
171+
filename = self.get_dump_file("test_enumerate_csv_files.csv")
172+
df.to_csv(filename, index=False)
173+
zip_file = self.get_dump_file("test_enumerate_csv_files.zip")
174+
with zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED) as zipf:
175+
zipf.write(filename)
176+
177+
dirname = os.path.dirname(filename)
178+
data = [os.path.join(dirname, "*.csv"), os.path.join(dirname, "*.zip")]
179+
dfs = list(enumerate_csv_files(data, verbose=1))
180+
self.assertNotEmpty(dfs)
181+
for df in dfs:
182+
open_dataframe(df)
183+
184+
cube = CubeLogs(data, recent=True)
185+
cube.load(verbose=1)
186+
self.assertEqual((3, 11), cube.shape)
187+
self.assertIn("RAWFILENAME", cube.data.columns)
188+
189+
190+
if __name__ == "__main__":
191+
unittest.main(verbosity=2)

‎_unittests/ut_helpers/test_ort_session_tinyllm.py‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,16 +73,23 @@ def test_ort_value_more(self):
7373
@ignore_warnings((UserWarning, DeprecationWarning, FutureWarning))
7474
@hide_stdout()
7575
def test_check_allruntimes_on_tiny_llm(self):
76+
try:
77+
from experimental_experiment.torch_interpreter import to_onnx
78+
except ImportError:
79+
to_onnx = None
80+
7681
data = get_tiny_llm()
7782
model, inputs, ds = data["model"], data["inputs"], data["dynamic_shapes"]
7883
expected = model(**copy.deepcopy(inputs))
7984

80-
with torch_export_patches(patch_transformers=True):
81-
ep = torch.onnx.export(
82-
model, (), kwargs=copy.deepcopy(inputs), dynamic_shapes=ds, dynamo=True
83-
)
85+
with torch_export_patches(patch_transformers=True, stop_if_static=1):
86+
if to_onnx:
87+
proto = to_onnx(model, (), kwargs=copy.deepcopy(inputs), dynamic_shapes=ds)
88+
else:
89+
proto = torch.onnx.export(
90+
model, (), kwargs=copy.deepcopy(inputs), dynamic_shapes=ds, dynamo=True
91+
).model_proto
8492

85-
proto = ep.model_proto
8693
self.dump_onnx("test_check_allruntimes_on_tiny_llm.onnx", proto)
8794
feeds = make_feeds(proto, inputs, use_numpy=True, copy=True)
8895
sess = onnxruntime.InferenceSession(

‎_unittests/ut_torch_models/test_tiny_llms_onnx.py‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -97,29 +97,39 @@ def test_bypass_onnx_export_tiny_llm_official_nopositionids(self):
9797
@ignore_warnings((UserWarning, DeprecationWarning, FutureWarning))
9898
@hide_stdout()
9999
def test_bypass_onnx_export_tiny_llm_official_full(self):
100+
try:
101+
from experimental_experiment.torch_interpreter import to_onnx
102+
except ImportError:
103+
to_onnx = None
104+
100105
data = get_tiny_llm()
101106
model, inputs, ds = data["model"], data["inputs"], data["dynamic_shapes"]
102107
self.assertEqual(
103108
{"attention_mask", "past_key_values", "input_ids", "position_ids"}, set(inputs)
104109
)
105-
with torch_export_patches(patch_transformers=True, verbose=1) as modificator:
110+
with torch_export_patches(
111+
patch_transformers=True, verbose=1, stop_if_static=1
112+
) as modificator:
106113
new_inputs = modificator(copy.deepcopy(inputs))
107-
ep = torch.onnx.export(
108-
model,
109-
(),
110-
kwargs=new_inputs,
111-
dynamic_shapes=ds,
112-
dynamo=True,
113-
optimize=True,
114-
report=True,
115-
verify=False,
116-
)
114+
if to_onnx:
115+
proto = to_onnx(model, (), kwargs=new_inputs, dynamic_shapes=ds)
116+
else:
117+
proto = torch.onnx.export(
118+
model,
119+
(),
120+
kwargs=new_inputs,
121+
dynamic_shapes=ds,
122+
dynamo=True,
123+
optimize=True,
124+
report=True,
125+
verify=False,
126+
).model_proto
117127
# There are some discrepancies with torch==2.6
118128
if not has_torch("2.7"):
119129
raise unittest.SkipTest("discrepancies observed with torch<2.7")
120130
self.assert_onnx_disc(
121131
inspect.currentframe().f_code.co_name,
122-
ep.model_proto,
132+
proto,
123133
model,
124134
inputs,
125135
verbose=1,

0 commit comments

Comments
 (0)