check historical

Author: xadupre

CHANGELOGS.rst

@@ -4,6 +4,7 @@ Change Logs
 0.7.1
 +++++
 
+* :pr:`155`: better aggregation of historical data
 * :pr:`151`, :pr:`153`: adds command line ``agg``, class CubeLogsPerformance to produce timeseries
 * :pr:`152`: add a function to compute fully dynamic shapes given any inputs
 

_unittests/ut_helpers/test_log_helper.py

@@ -3,6 +3,7 @@
 import textwrap
 import unittest
 import zipfile
+import numpy as np
 import pandas
 from onnx_diagnostic.ext_test_case import ExtTestCase, hide_stdout
 from onnx_diagnostic.helpers.log_helper import (
@@ -46,7 +47,6 @@ def test_cube_logs_load_df(self):
         cube = CubeLogs(df)
         text = str(cube)
         self.assertIsInstance(text, str)
-        self.assertRaise(lambda: cube.load(verbose=1), AssertionError)
         cube = CubeLogs(
             self.df1(),
             recent=True,
@@ -229,6 +229,55 @@ def test_cube_logs_performance(self):
         )
         self.assertExists(output)
 
+    def test_duplicate(self):
+        df = pandas.DataFrame(
+            [
+                dict(date="2025/01/01", time_engine=0.5, model_name="A", version_engine="0.5"),
+                dict(date="2025/01/01", time_engine=0.5, model_name="A", version_engine="0.5"),
+            ]
+        )
+        cube = CubeLogs(df)
+        self.assertRaise(lambda: cube.load(), AssertionError)
+        CubeLogs(df, recent=True).load()
+
+    def test_historical(self):
+        # case 1
+        df = pandas.DataFrame(
+            [
+                dict(date="2025/01/01", time_p=0.51, exporter="E1", m_name="A", m_cls="CA"),
+                dict(date="2025/01/02", time_p=0.62, exporter="E1", m_name="A", m_cls="CA"),
+                dict(date="2025/01/01", time_p=0.53, exporter="E2", m_name="A", m_cls="CA"),
+                dict(date="2025/01/02", time_p=0.64, exporter="E2", m_name="A", m_cls="CA"),
+                dict(date="2025/01/01", time_p=0.55, exporter="E2", m_name="B", m_cls="CA"),
+                dict(date="2025/01/02", time_p=0.66, exporter="E2", m_name="B", m_cls="CA"),
+            ]
+        )
+        cube = CubeLogs(df, keys=["^m_*", "exporter"]).load()
+        view, view_def = cube.view(CubeViewDef(["^m_.*"], ["^time_.*"]), return_view_def=True)
+        self.assertEqual(
+            "CubeViewDef(key_index=['^m_.*'], values=['^time_.*'])", repr(view_def)
+        )
+        self.assertEqual(["METRICS", "exporter", "date"], view.columns.names)
+        got = view.values.ravel()
+        self.assertEqual(
+            sorted([0.51, 0.62, 0.53, 0.64, -1, -1, 0.55, 0.66]),
+            sorted(np.where(np.isnan(got), -1, got).tolist()),
+        )
+
+        # case 2
+        df = pandas.DataFrame(
+            [
+                dict(date="2025/01/02", time_p=0.62, exporter="E1", m_name="A", m_cls="CA"),
+                dict(date="2025/01/02", time_p=0.64, exporter="E2", m_name="A", m_cls="CA"),
+                dict(date="2025/01/01", time_p=0.51, exporter="E1", m_name="B", m_cls="CA"),
+                dict(date="2025/01/02", time_p=0.66, exporter="E2", m_name="B", m_cls="CA"),
+            ]
+        )
+        cube = CubeLogs(df, keys=["^m_*", "exporter"]).load()
+        view, view_def = cube.view(CubeViewDef(["^m_.*"], ["^time_.*"]), return_view_def=True)
+        self.assertEqual((2, 3), view.shape)
+        self.assertEqual(["METRICS", "exporter", "date"], view.columns.names)
+
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)

onnx_diagnostic/helpers/log_helper.py

@@ -573,7 +573,7 @@ def view(
                 )
         else:
             if verbose:
-                print("[CubeLogs.view] keep all colums")
+                print("[CubeLogs.view] keep all columns")
             key_columns = sorted(set_key_columns)
             unique = set()