add callback for measurement filtering

Author: Priyadarshini Piramanayagam

nisystemlink/clients/testmonitor/utilities/_dataframe_utilities.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List
+from typing import Any, Callable, Dict, List, Optional
 
 import pandas as pd
 from nisystemlink.clients.testmonitor.models import Result, Step, StepProjection
@@ -39,11 +39,19 @@ def convert_results_to_dataframe(
     return normalized_dataframe
 
 
-def convert_steps_to_dataframe(steps: List[Step]) -> pd.DataFrame:
+def convert_steps_to_dataframe(
+    steps: List[Step],
+    is_valid_measurement: Optional[Callable[[Dict[str, Any]], bool]] = None,
+) -> pd.DataFrame:
     """Converts a list of steps into a normalized dataframe.
 
     Args:
         steps: A list of steps.
+        is_valid_measurement: Optional function to check if a measurement is valid. The method takes
+            a dictionary as input and returns a boolean value. If the function is not provided, the
+            default behavior is to keep only those measurements that have both 'name' and 'measurement' fields.
+            If none of the measurement data have the desired fields, the data.parameters will not
+            appear in the dataframe.
 
     Returns:
         DataFrame:
@@ -57,7 +65,7 @@ def convert_steps_to_dataframe(steps: List[Step]) -> pd.DataFrame:
             all other step fields are duplicated.
     """
     DATA_PARAMETERS = "data.parameters"
-    step_dicts = __convert_steps_to_dict(steps)
+    step_dicts = __convert_steps_to_dict(steps, is_valid_measurement)
     steps_dataframe = pd.json_normalize(step_dicts, sep=".")
     steps_dataframe = __explode_and_normalize(
         steps_dataframe, DATA_PARAMETERS, f"{DATA_PARAMETERS}."
@@ -156,7 +164,10 @@ def __is_property_header(header: str) -> bool:
     return header.startswith(DataFrameHeaders.PROPERTY_COLUMN_HEADER_PREFIX)
 
 
-def __convert_steps_to_dict(steps: List[Step]) -> List[Dict[str, Any]]:
+def __convert_steps_to_dict(
+    steps: List[Step],
+    is_valid_measurement: Optional[Callable[[Dict[str, Any]], bool]] = None,
+) -> List[Dict[str, Any]]:
     """Converts a list of steps to dictionaries, excluding None values.
 
     Args:
@@ -168,12 +179,44 @@ def __convert_steps_to_dict(steps: List[Step]) -> List[Dict[str, Any]]:
     steps_dict = []
     for step in steps:
         single_step_dict = step.dict(exclude_none=True)
+
+        if step.data is not None and step.data.parameters is not None:
+            single_step_dict["data"]["parameters"] = __get_valid_data_parameters(
+                single_step_dict, is_valid_measurement
+            )
+
         __normalize_inputs_outputs(single_step_dict, step)
         __normalize_step_status(single_step_dict)
         steps_dict.append(single_step_dict)
     return steps_dict
 
 
+def __get_valid_data_parameters(
+    step_dict: Dict[str, Any],
+    is_valid_measurement: Optional[Callable[[Dict[str, Any]], bool]] = None,
+) -> List[Dict[str, Any]]:
+    """Gets valid measurement data parameters from the step dictionary.
+
+    Args:
+        step_dict: A dictionary with step information.
+        is_valid_measurement: Optional callback function to check if a measurement is valid. The method takes
+            a dictionary as input and returns a boolean value. If the function is not provided, the
+            default behavior is to keep only those measurements that have both 'name' and 'measurement' fields.
+
+    Returns:
+        List[Dict[str, Any]]: A list of dictionaries containing valid measurement data.
+    """
+    allowed_measurement_keys = ["name", "units"]
+    valid_measurement_parameters = []
+    for parameter in step_dict["data"]["parameters"]:
+        if (is_valid_measurement and is_valid_measurement(parameter)) or all(
+            key in parameter for key in allowed_measurement_keys
+        ):
+            valid_measurement_parameters.append(parameter)
+
+    return valid_measurement_parameters
+
+
 def __normalize_inputs_outputs(
     step_dict: Dict[str, Any],
     step: Step,

tests/testmonitor/test_testmonitor_dataframe_utilities.py

@@ -174,7 +174,6 @@ def mock_steps_data() -> List[Step]:
                     "status": "Passed",
                     "lowLimit": "7.0",
                     "highLimit": "22.0",
-                    "measurement": "12.0",
                     "comparisonType": "GTLT",
                 },
             ],
@@ -210,38 +209,42 @@ def expected_steps_dataframe(mock_steps_data: List[Step]) -> pd.DataFrame:
         if not (step.data and step.data.parameters):
             continue
         for parametric_data in step.data.parameters:
-            parameter_data = {
-                f"data.parameters.{key}": value
-                for key, value in parametric_data.dict().items()
-            }
+            if (
+                "name" in parametric_data.dict()
+                and "measurement" in parametric_data.dict()
+            ):
+                parameter_data = {
+                    f"data.parameters.{key}": value
+                    for key, value in parametric_data.dict().items()
+                }
 
-            restructured_step = {
-                "name": step.name,
-                "step_type": step.step_type,
-                "step_id": step.step_id,
-                "parent_id": step.parent_id,
-                "result_id": step.result_id,
-                "path": step.path,
-                "path_ids": step.path_ids,
-                "status": (
-                    step.status.status_type.value
-                    if step.status and step.status.status_type != "CUSTOM"
-                    else step.status.status_name if step.status else None
-                ),
-                "total_time_in_seconds": step.total_time_in_seconds,
-                "started_at": step.started_at,
-                "updated_at": step.updated_at,
-                "data_model": step.data_model,
-                "has_children": step.has_children,
-                "workspace": step.workspace,
-                "keywords": step.keywords,
-                **inputs,
-                **outputs,
-                "data.text": step.data.text,
-                **parameter_data,
-                **properties,
-            }
-            restructured_mock_steps.append(restructured_step)
+                restructured_step = {
+                    "name": step.name,
+                    "step_type": step.step_type,
+                    "step_id": step.step_id,
+                    "parent_id": step.parent_id,
+                    "result_id": step.result_id,
+                    "path": step.path,
+                    "path_ids": step.path_ids,
+                    "status": (
+                        step.status.status_type.value
+                        if step.status and step.status.status_type != "CUSTOM"
+                        else step.status.status_name if step.status else None
+                    ),
+                    "total_time_in_seconds": step.total_time_in_seconds,
+                    "started_at": step.started_at,
+                    "updated_at": step.updated_at,
+                    "data_model": step.data_model,
+                    "has_children": step.has_children,
+                    "workspace": step.workspace,
+                    "keywords": step.keywords,
+                    **inputs,
+                    **outputs,
+                    "data.text": step.data.text,
+                    **parameter_data,
+                    **properties,
+                }
+                restructured_mock_steps.append(restructured_step)
     expected_dataframe = pd.DataFrame(restructured_mock_steps)
     expected_column_order = [
         "name",
@@ -291,6 +294,7 @@ def empty_steps_data() -> List:
 
 
 @pytest.mark.enterprise
+@pytest.mark.unit
 class TestTestmonitorDataframeUtilities:
     def test__convert_results_with_all_fields_to_dataframe__returns_whole_results_dataframe(
         self, results