Tutorial Notebook + trunc behavior

Author: AllenBaranov

sigllm/primitives/formatting/json_format.py

@@ -76,20 +76,28 @@ def _extract_d0_values(self, sample):
 
     def _format_as_integer_legacy(self, X, trunc=None):
         """
-        Legacy format_as_integer behavior.
+        Extract d0 values from parsed output.
         
-        - If trunc is None: returns all values (full round-trip for validation)
-        - If trunc is set: extracts only d0 values and truncates (for pipeline)
+        - trunc=None: return all d0 values (num_windows, num_samples, num_d0_values)
+        - trunc=int: return 3D array (num_windows, num_samples, trunc) 
         """
-        batch_rows = []
-        for window in X:
-            samples = []
-            for sample in window:
-                if trunc is None:
-                    tokens = re.findall(r'd\d+:(\d+)', sample)
-                    values = [int(v) for v in tokens]
-                else:
-                    values = self._extract_d0_values(sample)[:trunc]
-                samples.append(values)
-            batch_rows.append(samples)
-        return np.array(batch_rows, dtype=object)
+        if trunc is None:
+            batch_rows = []
+            for window in X:
+                samples = []
+                for sample in window:
+                    samples.append(self._extract_d0_values(sample))
+                batch_rows.append(samples)
+            return np.array(batch_rows, dtype=object)
+        
+        num_windows = len(X)
+        num_samples = len(X[0]) if num_windows > 0 else 0
+        result = np.zeros((num_windows, num_samples, trunc), dtype=int)
+        
+        for i, window in enumerate(X):
+            for j, sample in enumerate(window):
+                d0_values = self._extract_d0_values(sample)
+                for k in range(min(trunc, len(d0_values))):
+                    result[i, j, k] = d0_values[k]
+        
+        return result

sigllm/primitives/formatting/multivariate_formatting.py

@@ -10,7 +10,7 @@ def __init__(self, method_name: str, verbose: bool = False, **kwargs):
         self.metadata = {}
         self.verbose = verbose
 
-        if self.method_name != "persistence_control":
+        if self.method_name != "persistence_control" and self.config.get('trunc', None) == None:
             test_multivariate_formatting_validity(self, verbose=verbose)
 
 

sigllm/primitives/formatting/utils.py

@@ -20,43 +20,30 @@ def test_multivariate_formatting_validity(method, verbose=False):
     if verbose:
         print("Testing multivariate formatting method validity")
 
-    #raw_data = create_test_data()[:, 1:]
     raw_data = create_test_data().to_numpy()[:, 1:]
     windowed_data = np.array([raw_data[i:i+15,:] for i in range(0, len(raw_data)-15, 1)])
     data = (1000 * windowed_data).astype(int)
     if verbose:
         print(data.shape)
 
-    # Temporarily disable trunc for validation (we need full round-trip)
-    original_trunc = method.config.get('trunc')
-    had_trunc = 'trunc' in method.config
-    method.config['trunc'] = None
-    
-    try:
-        string_data = method.format_as_string(data, **method.config)
-        LLM_mock_output = np.array(string_data).reshape(-1, 1)
-        if verbose:
-            print(LLM_mock_output)
-        integer_data = method.format_as_integer(LLM_mock_output, **method.config)
-        if verbose:
-            print(f"Format as string output: {string_data}")
-
-        assert isinstance(string_data, list)
-        assert isinstance(string_data[0], str)
-        assert isinstance(integer_data, np.ndarray)
-
-        if method.method_name == "univariate_control":
-            assert np.all(integer_data.flatten() == data[:, :, 0].flatten())
-        else:
-            assert np.all(integer_data.flatten() == data.flatten())
-
-        print("Validation suite passed")
-    finally:
-        # Restore original trunc value
-        if had_trunc:
-            method.config['trunc'] = original_trunc
-        elif 'trunc' in method.config:
-            del method.config['trunc']
+    string_data = method.format_as_string(data, **method.config)
+    LLM_mock_output = np.array(string_data).reshape(-1, 1)
+    if verbose:
+        print(f"LLM mock output: {LLM_mock_output}")
+    integer_data = method.format_as_integer(LLM_mock_output, **method.config)
+    if verbose:
+        print(f"Format as string output: {string_data}")
+
+    assert isinstance(string_data, list)
+    assert isinstance(string_data[0], str)
+    assert isinstance(integer_data, np.ndarray)
+
+    if len(integer_data.flatten()) == len(data.flatten()):
+        assert np.all(integer_data.flatten() == data.flatten())
+    elif len(integer_data.flatten()) == len(data[:, :, 0].flatten()):
+        assert np.all(integer_data.flatten() == data[:, :, 0].flatten())
+    else:
+        raise ValueError(f"Validation suite failed: Dimensions do not match")