logic to handle optional_extra

Signed-off-by: Jerry Guo <Jerry.Jinfeng.Guo@alliander.com>

Author: Jerry-Jinfeng-Guo

docs/converters/vision_converter.md

@@ -52,6 +52,33 @@ Vision introduced UUID based identifier system since version 9.7. It is implemen
 
 An examplery usage can be found in the example notebook as well as in the test cases.
 
+## Optional extra columns
+
+When working with Vision Excel exports, some metadata columns (like `GUID` or `StationID`) may not always be present, especially in partial exports. The `optional_extra` feature allows you to specify columns that should be included in `extra_info` if present, but won't cause conversion failure if missing.
+
+**Syntax:**
+```yaml
+grid:
+  Transformers:
+    transformer:
+      id:
+        auto_id:
+          key: Number
+      # ... other fields ...
+      extra:
+        - ID            # Required - fails if missing
+        - Name          # Required - fails if missing
+        - optional_extra:
+            - GUID      # Optional - skipped if missing
+            - StationID # Optional - skipped if missing
+```
+
+**Behavior:**
+- Required columns (listed directly under `extra`) will cause a KeyError if missing
+- Optional columns (nested under `optional_extra`) are silently skipped if not found
+- If some optional columns are present and others missing, only the present ones are included in `extra_info`
+- This feature is particularly useful for handling different Vision export configurations or versions
+
 ## Common/Known issues related to Vision
 So far we have the following issue known to us related to Vision exported spread sheets. We provide a solution from user perspective to the best of our knowledge.
 

src/power_grid_model_io/converters/tabular_converter.py

@@ -396,6 +396,7 @@ def _parse_col_def(  # pylint: disable = too-many-arguments,too-many-positional-
         col_def: Any,
         table_mask: Optional[np.ndarray],
         extra_info: Optional[ExtraInfo],
+        allow_missing: bool = False,
     ) -> pd.DataFrame:
         """Interpret the column definition and extract/convert/create the data as a pandas DataFrame.
 
@@ -404,15 +405,27 @@ def _parse_col_def(  # pylint: disable = too-many-arguments,too-many-positional-
           table: str:
           col_def: Any:
           extra_info: Optional[ExtraInfo]:
+          allow_missing: bool: If True, missing columns will return empty DataFrame instead of raising KeyError
 
         Returns:
 
         """
         if isinstance(col_def, (int, float)):
             return self._parse_col_def_const(data=data, table=table, col_def=col_def, table_mask=table_mask)
         if isinstance(col_def, str):
-            return self._parse_col_def_column_name(data=data, table=table, col_def=col_def, table_mask=table_mask)
+            return self._parse_col_def_column_name(
+                data=data, table=table, col_def=col_def, table_mask=table_mask, allow_missing=allow_missing
+            )
         if isinstance(col_def, dict):
+            # Check if this is an optional_extra wrapper
+            if len(col_def) == 1 and "optional_extra" in col_def:
+                # Extract the list of optional columns and parse as composite with allow_missing=True
+                optional_cols = col_def["optional_extra"]
+                if not isinstance(optional_cols, list):
+                    raise TypeError(f"optional_extra value must be a list, got {type(optional_cols).__name__}")
+                return self._parse_col_def_composite(
+                    data=data, table=table, col_def=optional_cols, table_mask=table_mask, allow_missing=True
+                )
             return self._parse_col_def_filter(
                 data=data,
                 table=table,
@@ -421,7 +434,9 @@ def _parse_col_def(  # pylint: disable = too-many-arguments,too-many-positional-
                 extra_info=extra_info,
             )
         if isinstance(col_def, list):
-            return self._parse_col_def_composite(data=data, table=table, col_def=col_def, table_mask=table_mask)
+            return self._parse_col_def_composite(
+                data=data, table=table, col_def=col_def, table_mask=table_mask, allow_missing=allow_missing
+            )
         raise TypeError(f"Invalid column definition: {col_def}")
 
     @staticmethod
@@ -454,6 +469,7 @@ def _parse_col_def_column_name(
         table: str,
         col_def: str,
         table_mask: Optional[np.ndarray] = None,
+        allow_missing: bool = False,
     ) -> pd.DataFrame:
         """Extract a column from the data. If the column doesn't exist, check if the col_def is a special float value,
         like 'inf'. If that's the case, create a single column pandas DataFrame containing the const value.
@@ -462,6 +478,7 @@ def _parse_col_def_column_name(
           data: TabularData:
           table: str:
           col_def: str:
+          allow_missing: bool: If True, return empty DataFrame when column is missing instead of raising KeyError
 
         Returns:
 
@@ -486,6 +503,15 @@ def _parse_col_def_column_name(
             const_value = float(col_def)
         except ValueError:
             # pylint: disable=raise-missing-from
+            if allow_missing:
+                # Return empty DataFrame with correct number of rows when column is optional and missing
+                self._log.debug(
+                    "Optional column not found",
+                    table=table,
+                    columns=" or ".join(f"'{col_name}'" for col_name in columns),
+                )
+                n_rows = len(table_data)
+                return pd.DataFrame(index=range(n_rows))
             columns_str = " and ".join(f"'{col_name}'" for col_name in columns)
             raise KeyError(f"Could not find column {columns_str} on table '{table}'")
 
@@ -778,13 +804,15 @@ def _parse_col_def_composite(
         table: str,
         col_def: list,
         table_mask: Optional[np.ndarray],
+        allow_missing: bool = False,
     ) -> pd.DataFrame:
         """Select multiple columns (each is created from a column definition) and return them as a new DataFrame.
 
         Args:
           data: TabularData:
           table: str:
           col_def: list:
+          allow_missing: bool: If True, skip missing columns instead of raising errors
 
         Returns:
 
@@ -797,10 +825,20 @@ def _parse_col_def_composite(
                 col_def=sub_def,
                 table_mask=table_mask,
                 extra_info=None,
+                allow_missing=allow_missing,
             )
             for sub_def in col_def
         ]
-        return pd.concat(columns, axis=1)
+        # Filter out empty DataFrames (from missing optional columns)
+        non_empty_columns = [col for col in columns if not col.empty and len(col.columns) > 0]
+        if not non_empty_columns:
+            # If all columns are missing, return an empty DataFrame with the correct number of rows
+            table_data = data[table]
+            if table_mask is not None:
+                table_data = table_data[table_mask]
+            n_rows = len(table_data)
+            return pd.DataFrame(index=range(n_rows))
+        return pd.concat(non_empty_columns, axis=1)
 
     def _get_id(self, table: str, key: Mapping[str, int], name: Optional[str]) -> int:
         """

tests/unit/converters/test_tabular_converter.py

@@ -462,6 +462,7 @@ def test_parse_col_def(converter: TabularConverter, tabular_data_no_units_no_sub
             table="nodes",
             col_def="col_name",
             table_mask=None,
+            allow_missing=False,
         )
 
     # type(col_def) == dict
@@ -499,6 +500,7 @@ def test_parse_col_def(converter: TabularConverter, tabular_data_no_units_no_sub
             table="nodes",
             col_def=[],
             table_mask=None,
+            allow_missing=False,
         )
 
 
@@ -1356,3 +1358,177 @@ def bool_fn_filter(row: pd.Series, **kwargs):
 def test_parse_table_filters__ndarray_data(converter: TabularConverter):
     numpy_tabular_data = TabularData(numpy_table=np.ones((4, 3)))
     assert converter._parse_table_filters(data=numpy_tabular_data, table="numpy_table", filtering_functions=[]) is None
+
+
+def test_optional_extra__all_columns_present(converter: TabularConverter):
+    """Test optional_extra when all optional columns are present in the data"""
+    # Arrange
+    data = TabularData(
+        test_table=pd.DataFrame(
+            {"id": [1, 2], "name": ["node1", "node2"], "guid": ["guid1", "guid2"], "station": ["st1", "st2"]}
+        )
+    )
+    col_def = {"optional_extra": ["guid", "station"]}
+
+    # Act
+    result = converter._parse_col_def(
+        data=data, table="test_table", col_def=col_def, table_mask=None, extra_info=None, allow_missing=False
+    )
+
+    # Assert
+    assert list(result.columns) == ["guid", "station"]
+    assert list(result["guid"]) == ["guid1", "guid2"]
+    assert list(result["station"]) == ["st1", "st2"]
+
+
+def test_optional_extra__some_columns_missing(converter: TabularConverter):
+    """Test optional_extra when some optional columns are missing from the data"""
+    # Arrange
+    data = TabularData(test_table=pd.DataFrame({"id": [1, 2], "name": ["node1", "node2"], "guid": ["guid1", "guid2"]}))
+    col_def = {"optional_extra": ["guid", "station"]}  # 'station' is missing
+
+    # Act
+    result = converter._parse_col_def(
+        data=data, table="test_table", col_def=col_def, table_mask=None, extra_info=None, allow_missing=False
+    )
+
+    # Assert - only 'guid' should be present
+    assert list(result.columns) == ["guid"]
+    assert list(result["guid"]) == ["guid1", "guid2"]
+
+
+def test_optional_extra__all_columns_missing(converter: TabularConverter):
+    """Test optional_extra when all optional columns are missing from the data"""
+    # Arrange
+    data = TabularData(test_table=pd.DataFrame({"id": [1, 2], "name": ["node1", "node2"]}))
+    col_def = {"optional_extra": ["guid", "station"]}  # Both are missing
+
+    # Act
+    result = converter._parse_col_def(
+        data=data, table="test_table", col_def=col_def, table_mask=None, extra_info=None, allow_missing=False
+    )
+
+    # Assert - should return empty DataFrame with correct number of rows
+    assert len(result) == 2
+    assert len(result.columns) == 0
+
+
+def test_optional_extra__mixed_with_required(converter: TabularConverter):
+    """Test mixing required and optional extra columns"""
+    # Arrange
+    data = TabularData(test_table=pd.DataFrame({"id": [1, 2], "name": ["node1", "node2"], "guid": ["guid1", "guid2"]}))
+    # Mix required columns with optional_extra
+    col_def = ["name", {"optional_extra": ["guid", "station"]}]
+
+    # Act
+    result = converter._parse_col_def(
+        data=data, table="test_table", col_def=col_def, table_mask=None, extra_info=None, allow_missing=False
+    )
+
+    # Assert - should have 'name' and 'guid', but not 'station'
+    assert list(result.columns) == ["name", "guid"]
+    assert list(result["name"]) == ["node1", "node2"]
+    assert list(result["guid"]) == ["guid1", "guid2"]
+
+
+def test_optional_extra__in_extra_info(converter: TabularConverter):
+    """Test that optional_extra works correctly with _handle_extra_info"""
+    # Arrange
+    data = TabularData(
+        test_table=pd.DataFrame(
+            {"id": [1, 2], "name": ["node1", "node2"], "guid": ["guid1", "guid2"]}  # 'station' is missing
+        )
+    )
+    uuids = np.array([100, 200])
+    extra_info: ExtraInfo = {}
+    col_def = {"optional_extra": ["guid", "station"]}
+
+    # Act
+    converter._handle_extra_info(
+        data=data, table="test_table", col_def=col_def, uuids=uuids, table_mask=None, extra_info=extra_info
+    )
+
+    # Assert - only 'guid' should be in extra_info, not 'station'
+    assert 100 in extra_info
+    assert 200 in extra_info
+    assert "guid" in extra_info[100]
+    assert "guid" in extra_info[200]
+    assert extra_info[100]["guid"] == "guid1"
+    assert extra_info[200]["guid"] == "guid2"
+    assert "station" not in extra_info[100]
+    assert "station" not in extra_info[200]
+
+
+def test_optional_extra__all_missing_no_extra_info(converter: TabularConverter):
+    """Test that when all optional columns are missing, no extra_info entries are created"""
+    # Arrange
+    data = TabularData(test_table=pd.DataFrame({"id": [1, 2], "name": ["node1", "node2"]}))  # Both optional missing
+    uuids = np.array([100, 200])
+    extra_info: ExtraInfo = {}
+    col_def = {"optional_extra": ["guid", "station"]}
+
+    # Act
+    converter._handle_extra_info(
+        data=data, table="test_table", col_def=col_def, uuids=uuids, table_mask=None, extra_info=extra_info
+    )
+
+    # Assert - no entries should be added to extra_info
+    assert len(extra_info) == 0
+
+
+def test_optional_extra__invalid_type():
+    """Test that optional_extra raises TypeError if value is not a list"""
+    # Arrange
+    converter = TabularConverter(mapping_file=MAPPING_FILE)
+    data = TabularData(test_table=pd.DataFrame({"id": [1, 2]}))
+    col_def = {"optional_extra": "not_a_list"}  # Invalid: should be a list
+
+    # Act & Assert
+    with pytest.raises(TypeError, match="optional_extra value must be a list"):
+        converter._parse_col_def(
+            data=data, table="test_table", col_def=col_def, table_mask=None, extra_info=None, allow_missing=False
+        )
+
+
+def test_optional_extra__integration():
+    """Integration test for optional_extra feature using a complete mapping file"""
+    # Arrange
+    mapping_file = Path(__file__).parents[2] / "data" / "config" / "test_optional_extra_mapping.yaml"
+    converter = TabularConverter(mapping_file=mapping_file)
+
+    # Create test data with some optional columns present and some missing
+    data = TabularData(
+        nodes=pd.DataFrame(
+            {
+                "node_id": [1, 2, 3],
+                "voltage": [10.5, 10.5, 0.4],
+                "ID": ["N1", "N2", "N3"],
+                "Name": ["Node 1", "Node 2", "Node 3"],
+                "GUID": ["guid-1", "guid-2", "guid-3"],
+                # Note: StationID column is missing (optional)
+            }
+        )
+    )
+
+    extra_info: ExtraInfo = {}
+
+    # Act
+    result = converter._parse_data(data=data, data_type=DatasetType.input, extra_info=extra_info)
+
+    # Assert
+    assert ComponentType.node in result
+    assert len(result[ComponentType.node]) == 3
+
+    # Check that extra_info contains the required and present optional fields
+    for node_id in result[ComponentType.node]["id"]:
+        assert node_id in extra_info
+        assert "ID" in extra_info[node_id]
+        assert "Name" in extra_info[node_id]
+        assert "GUID" in extra_info[node_id]  # Optional but present
+        assert "StationID" not in extra_info[node_id]  # Optional and missing
+
+    # Verify values
+    node_0_id = result[ComponentType.node]["id"][0]
+    assert extra_info[node_0_id]["ID"] == "N1"
+    assert extra_info[node_0_id]["Name"] == "Node 1"
+    assert extra_info[node_0_id]["GUID"] == "guid-1"