Merge pull request #1099 from PowerGridModel/feature/improve-dataset-error-messages

Error messages: clearer message when dataset is inconsistent

Author: Jerry-Jinfeng-Guo

src/power_grid_model/_core/buffer_handling.py

@@ -200,9 +200,15 @@ def _get_dense_buffer_properties(
     n_total_elements = actual_batch_size * n_elements_per_scenario
 
     if is_batch is not None and is_batch != actual_is_batch:
-        raise ValueError(f"Provided 'is batch' is incorrect for the provided data. {VALIDATOR_MSG}")
+        raise ValueError(
+            f"Incorrect/inconsistent data provided: {'batch' if actual_is_batch else 'single'} "
+            f"data provided but {'batch' if is_batch else 'single'} data expected. {VALIDATOR_MSG}"
+        )
     if batch_size is not None and batch_size != actual_batch_size:
-        raise ValueError(f"Provided 'batch size' is incorrect for the provided data. {VALIDATOR_MSG}")
+        raise ValueError(
+            f"Incorrect/inconsistent batch size provided: {actual_batch_size} scenarios provided "
+            f"but {batch_size} scenarios expected. {VALIDATOR_MSG}"
+        )
 
     return BufferProperties(
         is_sparse=is_sparse_property,

src/power_grid_model/_core/power_grid_dataset.py

@@ -240,7 +240,7 @@ def _add_data(self, data: Dataset):
             data: the data.
 
         Raises:
-            ValueError: if the component is unknown and allow_unknown is False.
+            ValueError: if the component is unknown.
             ValueError: if the data is inconsistent with the rest of the dataset.
             PowerGridError: if there was an internal error.
         """
@@ -294,7 +294,7 @@ def _register_attribute_buffer(self, component, attr, attr_data):
         assert_no_error()
 
     def _validate_properties(self, data: ComponentData, schema: ComponentMetaData):
-        properties = get_buffer_properties(data, schema=schema, is_batch=self._is_batch, batch_size=self._batch_size)
+        properties = get_buffer_properties(data, schema=schema, is_batch=None, batch_size=None)
         if properties.is_batch != self._is_batch:
             raise ValueError(
                 f"Dataset type (single or batch) must be consistent across all components. {VALIDATOR_MSG}"

src/power_grid_model/_core/utils.py

@@ -728,8 +728,13 @@ def check_indptr_consistency(indptr: IndexPointer, batch_size: int | None, conte
         raise ValueError(f"indptr should start from zero and end at size of data array. {VALIDATOR_MSG}")
     if np.any(np.diff(indptr) < 0):
         raise ValueError(f"indptr should be increasing. {VALIDATOR_MSG}")
-    if batch_size is not None and batch_size != indptr.size - 1:
-        raise ValueError(f"Provided batch size must be equal to actual batch size. {VALIDATOR_MSG}")
+
+    actual_batch_size = indptr.size - 1
+    if batch_size is not None and batch_size != actual_batch_size:
+        raise ValueError(
+            f"Incorrect/inconsistent batch size provided: {actual_batch_size} scenarios provided "
+            f"but {batch_size} scenarios expected. {VALIDATOR_MSG}"
+        )
 
 
 def get_dataset_type(data: Dataset) -> DatasetType:

tests/unit/test_buffer_handling.py

@@ -10,9 +10,8 @@
 import pytest
 
 from power_grid_model._core.buffer_handling import (
-    _get_dense_buffer_properties,
     _get_raw_attribute_data_view,
-    _get_sparse_buffer_properties,
+    get_buffer_properties,
     get_buffer_view,
 )
 from power_grid_model._core.dataset_definitions import ComponentType, DatasetType
@@ -26,16 +25,19 @@
 
 def load_data(component_type, is_batch, is_sparse, is_columnar):
     """Creates load data of different formats for testing"""
-    shape = (2, 4) if is_batch else (4,)
+    shape = (BATCH_DATASET_NDIM, SCENARIO_TOTAL_ELEMENTS) if is_batch else (SCENARIO_TOTAL_ELEMENTS,)
     load = initialize_array(DatasetType.update, component_type, shape)
     columnar_names = ["p_specified", "q_specified"]
 
     if is_columnar:
         if is_sparse:
-            return {"indptr": np.array([0, 5, 8]), "data": {k: load.reshape(-1)[k] for k in columnar_names}}
+            return {
+                "indptr": np.array([0, 5, BATCH_TOTAL_ELEMENTS]),
+                "data": {k: load.reshape(-1)[k] for k in columnar_names},
+            }
         return {k: load[k] for k in columnar_names}
     if is_sparse:
-        return {"indptr": np.array([0, 5, 8]), "data": load.reshape(-1)}
+        return {"indptr": np.array([0, 5, BATCH_TOTAL_ELEMENTS]), "data": load.reshape(-1)}
     return load
 
 
@@ -56,7 +58,7 @@ def test__get_dense_buffer_properties(component_type, is_batch, is_columnar):
     data = load_data(component_type, is_batch=is_batch, is_columnar=is_columnar, is_sparse=False)
     schema = power_grid_meta_data[DatasetType.update][component_type]
     batch_size = BATCH_DATASET_NDIM if is_batch else None
-    properties = _get_dense_buffer_properties(data, schema=schema, is_batch=is_batch, batch_size=batch_size)
+    properties = get_buffer_properties(data, schema=schema, is_batch=is_batch, batch_size=batch_size)
 
     assert not properties.is_sparse
     assert properties.is_batch == is_batch
@@ -82,7 +84,7 @@ def test__get_sparse_buffer_properties(component_type, is_columnar):
     data = load_data(component_type, is_batch=True, is_columnar=is_columnar, is_sparse=True)
 
     schema = power_grid_meta_data[DatasetType.update][component_type]
-    properties = _get_sparse_buffer_properties(data, schema=schema, batch_size=2)
+    properties = get_buffer_properties(data, schema=schema, batch_size=2)
 
     assert properties.is_sparse
     assert properties.is_batch
@@ -95,6 +97,82 @@ def test__get_sparse_buffer_properties(component_type, is_columnar):
         assert properties.columns is None
 
 
+@pytest.mark.parametrize(
+    ("component_type", "is_columnar"),
+    [
+        pytest.param(ComponentType.sym_load, True, id="sym_load-single-columnar"),
+        pytest.param(ComponentType.sym_load, False, id="sym_load-single-row_based"),
+        pytest.param(ComponentType.asym_load, True, id="asym_load-single-columnar"),
+        pytest.param(ComponentType.asym_load, False, id="asym_load-single-row_based"),
+    ],
+)
+def test__get_buffer_properties__batch_requested_for_single_data(component_type, is_columnar):
+    data = load_data(component_type, is_batch=False, is_columnar=is_columnar, is_sparse=False)
+    schema = power_grid_meta_data[DatasetType.update][component_type]
+
+    with pytest.raises(
+        ValueError, match="Incorrect/inconsistent data provided: single data provided but batch data expected."
+    ):
+        get_buffer_properties(data, schema=schema, is_batch=True, batch_size=BATCH_DATASET_NDIM)
+
+
+@pytest.mark.parametrize(
+    ("component_type", "is_sparse", "is_columnar"),
+    [
+        pytest.param(ComponentType.sym_load, False, True, id="sym_load-batch-columnar"),
+        pytest.param(ComponentType.sym_load, False, False, id="sym_load-batch-row_based"),
+        pytest.param(ComponentType.asym_load, False, True, id="asym_load-batch-columnar"),
+        pytest.param(ComponentType.asym_load, False, False, id="asym_load-batch-row_based"),
+        pytest.param(ComponentType.sym_load, True, True, id="sym_load-columnar"),
+        pytest.param(ComponentType.sym_load, True, False, id="sym_load-row_based"),
+        pytest.param(ComponentType.asym_load, True, True, id="asym_load-columnar"),
+        pytest.param(ComponentType.asym_load, True, False, id="asym_load-row_based"),
+    ],
+)
+def test__get_buffer_properties__single_requested_for_batch(component_type, is_sparse, is_columnar):
+    data = load_data(component_type, is_batch=True, is_columnar=is_columnar, is_sparse=is_sparse)
+    schema = power_grid_meta_data[DatasetType.update][component_type]
+
+    if is_sparse:
+        with pytest.raises(ValueError, match="Sparse data must be batch data"):
+            get_buffer_properties(data, schema=schema, is_batch=False, batch_size=None)
+    else:
+        with pytest.raises(
+            ValueError, match="Incorrect/inconsistent data provided: batch data provided but single data expected."
+        ):
+            get_buffer_properties(data, schema=schema, is_batch=False, batch_size=None)
+
+
+@pytest.mark.parametrize(
+    ("component_type", "is_sparse", "is_columnar"),
+    [
+        pytest.param(ComponentType.sym_load, False, True, id="sym_load-batch-columnar"),
+        pytest.param(ComponentType.sym_load, False, False, id="sym_load-batch-row_based"),
+        pytest.param(ComponentType.asym_load, False, True, id="asym_load-batch-columnar"),
+        pytest.param(ComponentType.asym_load, False, False, id="asym_load-batch-row_based"),
+        pytest.param(ComponentType.sym_load, True, True, id="sym_load-columnar"),
+        pytest.param(ComponentType.sym_load, True, False, id="sym_load-row_based"),
+        pytest.param(ComponentType.asym_load, True, True, id="asym_load-columnar"),
+        pytest.param(ComponentType.asym_load, True, False, id="asym_load-row_based"),
+    ],
+)
+def test__get_buffer_properties__wrong_batch_size(component_type, is_sparse, is_columnar):
+    data = load_data(component_type, is_batch=True, is_columnar=is_columnar, is_sparse=is_sparse)
+    schema = power_grid_meta_data[DatasetType.update][component_type]
+
+    actual_batch_size = BATCH_DATASET_NDIM
+    wrong_batch_size = actual_batch_size + 1
+
+    with pytest.raises(
+        ValueError,
+        match=(
+            f"Incorrect/inconsistent batch size provided: {actual_batch_size} scenarios provided but "
+            f"{wrong_batch_size} scenarios expected."
+        ),
+    ):
+        get_buffer_properties(data, schema=schema, is_batch=True, batch_size=wrong_batch_size)
+
+
 @pytest.mark.parametrize(
     ("component", "is_batch", "is_columnar", "is_sparse"),
     [

tests/unit/test_dataset.py

@@ -152,12 +152,28 @@ def test_const_dataset__sparse_batch_data(dataset_type):
     }
 
 
+def test_const_dataset__mixed_batch_type(dataset_type):
+    data = {
+        ComponentType.node: np.zeros(shape=3, dtype=power_grid_meta_data[dataset_type][ComponentType.node]),
+        ComponentType.line: np.zeros(shape=(2, 3), dtype=power_grid_meta_data[dataset_type][ComponentType.line]),
+    }
+    with pytest.raises(ValueError, match=r"Dataset type \(single or batch\) must be consistent across all components."):
+        CConstDataset(data, dataset_type)
+
+    data = {
+        ComponentType.node: np.zeros(shape=(2, 3), dtype=power_grid_meta_data[dataset_type][ComponentType.node]),
+        ComponentType.line: np.zeros(shape=3, dtype=power_grid_meta_data[dataset_type][ComponentType.line]),
+    }
+    with pytest.raises(ValueError, match=r"Dataset type \(single or batch\) must be consistent across all components."):
+        CConstDataset(data, dataset_type)
+
+
 def test_const_dataset__mixed_batch_size(dataset_type):
     data = {
         ComponentType.node: np.zeros(shape=(2, 3), dtype=power_grid_meta_data[dataset_type][ComponentType.node]),
         ComponentType.line: np.zeros(shape=(3, 3), dtype=power_grid_meta_data[dataset_type][ComponentType.line]),
     }
-    with pytest.raises(ValueError, match="Provided 'batch size' is incorrect for the provided data."):
+    with pytest.raises(ValueError, match="Dataset must have a consistent batch size across all components."):
         CConstDataset(data, dataset_type)