Merge branch 'main' into fix/nightly-fail-remove-publish

Author: nitbharambe

.github/workflows/build-test-release.yml

@@ -93,11 +93,11 @@ jobs:
 
       - name: Test and Coverage
         run: |
-          pytest --cov-report=xml:coverage.xml --cov-fail-under=0
+          pytest
 
           # Fix relative paths in coverage file
           # Known bug: https://community.sonarsource.com/t/sonar-on-github-actions-with-python-coverage-source-issue/36057
-          sed -i 's@/home/runner/work/power-grid-model-io/power-grid-model-io@/github/workspace@g' coverage.xml
+          sed -i 's@/home/runner/work/power-grid-model-io/power-grid-model-io@/github/workspace@g' python_coverage.xml
 
       - name: SonarCloud Scan
         if: ${{ (github.event_name == 'push') || (github.event.pull_request.head.repo.owner.login == 'PowerGridModel') }}

.gitignore

@@ -16,6 +16,7 @@ __pycache__/
 .mypy_cache/
 .pytest_cache/
 cov_html/
+python_coverage.xml
 
 # Virtual environment
 venv*/

pyproject.toml

@@ -81,7 +81,14 @@ power_grid_model_io = ["config/**/*.yaml", "py.typed"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests/unit"]
-addopts = ["--cov=power_grid_model_io", "--cov-report=term", "--cov-report=html:cov_html", "--cov-fail-under=98.5"]
+addopts = [
+    "--cov=power_grid_model_io",
+    "--cov-report=term",
+    "--cov-report=html:cov_html",
+    "--cov-report=xml:python_coverage.xml",
+    "--cov-fail-under=98.5",
+]
+xfail_strict = true
 
 [tool.black]
 line-length = 120

sonar-project.properties

@@ -10,4 +10,4 @@ sonar.sources=src
 sonar.tests=tests
 sonar.sourceEncoding=UTF-8
 sonar.python.version=3
-sonar.python.coverage.reportPaths=/github/workspace/coverage.xml
+sonar.python.coverage.reportPaths=python_coverage.xml

src/power_grid_model_io/converters/pgm_json_converter.py

@@ -13,7 +13,6 @@
 
 import numpy as np
 from power_grid_model import initialize_array
-from power_grid_model._utils import is_nan
 from power_grid_model.data_types import ComponentList, Dataset, SingleDataset, SinglePythonDataset
 from power_grid_model.utils import json_deserialize, json_serialize
 
@@ -22,6 +21,12 @@
 from power_grid_model_io.data_types import ExtraInfo, StructuredData
 from power_grid_model_io.utils.dict import merge_dicts
 
+_NAN_FUNC = {
+    np.dtype("f8"): lambda x: np.all(np.isnan(x)),
+    np.dtype("i4"): lambda x: np.all(x == np.iinfo(np.dtype("i4")).min),
+    np.dtype("i1"): lambda x: np.all(x == np.iinfo(np.dtype("i1")).min),
+}
+
 
 class PgmJsonConverter(BaseConverter[StructuredData]):
     """
@@ -216,8 +221,7 @@ def _is_batch(data: Dataset) -> bool:
             is_batch = is_dense_batch or is_sparse_batch
         return bool(is_batch)
 
-    @staticmethod
-    def _serialize_dataset(data: SingleDataset, extra_info: Optional[ExtraInfo] = None) -> SinglePythonDataset:
+    def _serialize_dataset(self, data: SingleDataset, extra_info: Optional[ExtraInfo] = None) -> SinglePythonDataset:
         """This function converts a single power-grid-model dataset to a structured dataset
 
         Args:
@@ -248,7 +252,7 @@ def _serialize_dataset(data: SingleDataset, extra_info: Optional[ExtraInfo] = No
                     {
                         attribute: obj[attribute].tolist()
                         for attribute in objects.dtype.names
-                        if not is_nan(obj[attribute])
+                        if not self._is_nan(obj[attribute])
                     },
                     extra_info.get(obj["id"], {}),
                 )
@@ -296,3 +300,16 @@ def _get_first_by(data: List[Dict[str, Any]], field: str, value: Any) -> Optiona
                 return entry
 
         return None
+
+    @staticmethod
+    def _is_nan(data: np.ndarray) -> bool:
+        """
+        Determine whether the data point is valid
+        Args:
+            data: a single scalar or numpy array
+
+        Returns:
+            True when all the data points are invalid
+            False otherwise
+        """
+        return bool(_NAN_FUNC[data.dtype](data))

tests/unit/converters/test_pgm_json_converter.py

@@ -155,3 +155,16 @@ def test_serialize_dataset(converter: PgmJsonConverter, pgm_input_data: SingleDa
     extra_info: ExtraInfo = {1: {"dummy": "data"}}
     structured_data_with_extra_info = converter._serialize_dataset(data=pgm_input_data, extra_info=extra_info)
     assert structured_data_with_extra_info == {"node": [{"id": 1, "dummy": "data"}, {"id": 2}]}
+
+
+def test_is_nan(converter: PgmJsonConverter):
+    single_value = np.array([np.nan])
+    assert converter._is_nan(single_value)
+    array_f8 = np.array([0.1, 0.2, np.nan], dtype=np.dtype("f8"))
+    assert not converter._is_nan(array_f8)
+    array_i4 = np.array([10, 2, -(2**31), 40], dtype=np.dtype("i4"))
+    assert not converter._is_nan(array_i4)
+    array_i1 = np.array([1, 0, -(2**7), 1], dtype=np.dtype("i1"))
+    assert not converter._is_nan(array_i1)
+    nan_array = np.array([np.nan, np.nan, np.nan])
+    assert converter._is_nan(nan_array)