feat: remove msgpack

Signed-off-by: jaapschoutenalliander <[email protected]>

Author: jaapschoutenalliander

pyproject.toml

@@ -30,7 +30,6 @@ dependencies = [
     "power-grid-model>=1.7",
     "rustworkx>= 0.15.1",
     "numpy>=2.0",
-    "msgpack>=1.1.2",
 ]
 dynamic = ["version"]
 

src/power_grid_model_ds/__init__.py

@@ -7,17 +7,13 @@
 from power_grid_model_ds._core.model.grids.base import Grid
 from power_grid_model_ds._core.utils.serialization import (
     load_grid_from_json,
-    load_grid_from_msgpack,
     save_grid_to_json,
-    save_grid_to_msgpack,
 )
 
 __all__ = [
     "Grid",
     "GraphContainer",
     "PowerGridModelInterface",
     "save_grid_to_json",
-    "save_grid_to_msgpack",
     "load_grid_from_json",
-    "load_grid_from_msgpack",
 ]

src/power_grid_model_ds/_core/model/grids/base.py

@@ -362,7 +362,7 @@ def get_downstream_nodes(self, node_id: int, inclusive: bool = False):
     def cache(self, cache_dir: Path, cache_name: str, compress: bool = True):
         """Cache Grid to a folder using pickle format.
 
-        Note: Consider using save_to_json() or save_to_msgpack() for better
+        Note: Consider using save_to_json() for better
         interoperability and standardized format.
 
         Args:

src/power_grid_model_ds/_core/utils/serialization.py

@@ -9,11 +9,10 @@
 import logging
 from ast import literal_eval
 from pathlib import Path
-from typing import Dict, Literal, Optional
+from typing import Dict, Optional
 
-import msgpack
 import numpy as np
-from power_grid_model.utils import json_deserialize, json_serialize, msgpack_deserialize, msgpack_serialize
+from power_grid_model.utils import json_deserialize, json_serialize
 
 from power_grid_model_ds._core.load_flow import PGM_ARRAYS, PowerGridModelInterface
 from power_grid_model_ds._core.model.arrays.base.array import FancyArray
@@ -187,60 +186,6 @@ def _create_grid_from_input_data(input_data: Dict, target_grid_class=None):
     return interface.create_grid_from_input_data()
 
 
-def _extract_msgpack_data(data: bytes, **kwargs):
-    """Extract input data and extensions from MessagePack data."""
-    try:
-        data_dict = msgpack.unpackb(data, raw=False)
-        if isinstance(data_dict, dict) and EXTENSIONS_KEY in data_dict:
-            # Extract extensions and deserialize core data
-            extensions = data_dict.pop(EXTENSIONS_KEY, {})
-            core_data = msgpack.packb(data_dict)
-            input_data = msgpack_deserialize(core_data, **kwargs)
-        else:
-            # No extensions, use power-grid-model directly
-            input_data = msgpack_deserialize(data, **kwargs)
-            extensions = {EXTENDED_COLUMNS_KEY: {}, CUSTOM_ARRAYS_KEY: {}}
-    except (msgpack.exceptions.ExtraData, ValueError, TypeError) as e:
-        # Handle MessagePack parsing failures:
-        # - ExtraData: malformed MessagePack data
-        # - ValueError/TypeError: invalid data structure or type issues
-        logger.warning(f"Failed to extract extensions from MessagePack data: {e}")
-        input_data = msgpack_deserialize(data, **kwargs)
-        extensions = {EXTENDED_COLUMNS_KEY: {}, CUSTOM_ARRAYS_KEY: {}}
-
-    return input_data, extensions
-
-
-def _get_serialization_path(path: Path, format_type: Literal["json", "msgpack", "auto"] = "auto") -> Path:
-    """Get the correct path for serialization format.
-
-    Args:
-        path: Base path
-        format_type: "json", "msgpack", or "auto" to detect from extension
-
-    Returns:
-        Path: Path with correct extension
-    """
-    json_extensions = [".json"]
-    msgpack_extensions = [".msgpack", ".mp"]
-
-    if format_type == "auto":
-        if path.suffix.lower() in json_extensions:
-            format_type = "json"
-        elif path.suffix.lower() in msgpack_extensions:
-            format_type = "msgpack"
-        else:
-            # Default to JSON
-            format_type = "json"
-
-    if format_type == "json" and path.suffix.lower() != json_extensions[0]:
-        return path.with_suffix(json_extensions[0])
-    if format_type == "msgpack" and path.suffix.lower() not in msgpack_extensions:
-        return path.with_suffix(msgpack_extensions[0])
-
-    return path
-
-
 def save_grid_to_json(
     grid,
     path: Path,
@@ -303,65 +248,3 @@ def load_grid_from_json(path: Path, target_grid_class=None):
     _restore_extensions_data(grid, extensions)
 
     return grid
-
-
-def save_grid_to_msgpack(grid, path: Path, use_compact_list: bool = True, preserve_extensions: bool = True) -> Path:
-    """Save a Grid object to MessagePack format with extensions support.
-
-    Args:
-        grid: The Grid object to serialize
-        path: The file path to save to
-        use_compact_list: Whether to use compact list format
-        preserve_extensions: Whether to save extended columns and custom arrays
-
-    Returns:
-        Path: The path where the file was saved
-    """
-    path.parent.mkdir(parents=True, exist_ok=True)
-
-    # Convert Grid to power-grid-model input format and serialize
-    interface = PowerGridModelInterface(grid=grid)
-    input_data = interface.create_input_from_grid()
-
-    core_data = msgpack_serialize(input_data, use_compact_list=use_compact_list)
-
-    # Add extensions if requested (requires re-serialization for MessagePack)
-    if preserve_extensions:
-        extensions = _extract_extensions_data(grid)
-        if extensions[EXTENDED_COLUMNS_KEY] or extensions[CUSTOM_ARRAYS_KEY]:
-            core_dict = msgpack.unpackb(core_data, raw=False)
-            core_dict[EXTENSIONS_KEY] = extensions
-            serialized_data = msgpack.packb(core_dict)
-        else:
-            serialized_data = core_data
-    else:
-        serialized_data = core_data
-
-    # Write to file
-    with open(path, "wb") as f:
-        f.write(serialized_data)
-
-    return path
-
-
-def load_grid_from_msgpack(path: Path, target_grid_class=None):
-    """Load a Grid object from MessagePack format with cross-type loading support.
-
-    Args:
-        path: The file path to load from
-        target_grid_class: Optional Grid class to load into. If None, uses default Grid.
-
-    Returns:
-        Grid: The deserialized Grid object of the specified target class
-    """
-    with open(path, "rb") as f:
-        data = f.read()
-
-    # Extract extensions and deserialize core data
-    input_data, extensions = _extract_msgpack_data(data)
-
-    # Create grid and restore extensions
-    grid = _create_grid_from_input_data(input_data, target_grid_class)
-    _restore_extensions_data(grid, extensions)
-
-    return grid

tests/unit/utils/test_serialization.py

@@ -8,7 +8,6 @@
 from dataclasses import dataclass
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Literal
 
 import numpy as np
 import pytest
@@ -18,12 +17,9 @@
 from power_grid_model_ds._core.model.arrays.base.array import FancyArray
 from power_grid_model_ds._core.utils.serialization import (
     _extract_extensions_data,
-    _get_serialization_path,
     _restore_extensions_data,
     load_grid_from_json,
-    load_grid_from_msgpack,
     save_grid_to_json,
-    save_grid_to_msgpack,
 )
 from power_grid_model_ds.arrays import LineArray
 from power_grid_model_ds.arrays import NodeArray as BaseNodeArray
@@ -95,46 +91,25 @@ def extended_grid():
 class TestSerializationFormats:
     """Test serialization across different formats and configurations"""
 
-    @pytest.mark.parametrize(
-        "format_type,preserve_ext", [("json", True), ("json", False), ("msgpack", True), ("msgpack", False)]
-    )
-    def test_basic_serialization_roundtrip(
-        self, basic_grid: Grid, temp_dir: Path, format_type: str, preserve_ext: bool
-    ):
+    @pytest.mark.parametrize("preserve_ext", [(True), (False)])
+    def test_basic_serialization_roundtrip(self, basic_grid: Grid, temp_dir: Path, preserve_ext: bool):
         """Test basic serialization roundtrip for all formats"""
-        ext = "json" if format_type == "json" else "msgpack"
-        path = temp_dir / f"test.{ext}"
-
-        # Save
-        if format_type == "json":
-            result_path = save_grid_to_json(basic_grid, path, preserve_extensions=preserve_ext)
-        else:
-            result_path = save_grid_to_msgpack(basic_grid, path, preserve_extensions=preserve_ext)
-
+        path = temp_dir / "test.json"
+        result_path = save_grid_to_json(basic_grid, path, preserve_extensions=preserve_ext)
         assert result_path.exists()
 
         # Load and verify
-        if format_type == "json":
-            loaded_grid = load_grid_from_json(path, target_grid_class=Grid)
-        else:
-            loaded_grid = load_grid_from_msgpack(path, target_grid_class=Grid)
+        loaded_grid = load_grid_from_json(path, target_grid_class=Grid)
         assert loaded_grid.node.size == basic_grid.node.size
         assert loaded_grid.line.size == basic_grid.line.size
         assert list(loaded_grid.node.id) == list(basic_grid.node.id)
 
-    @pytest.mark.parametrize("format_type", ["json", "msgpack"])
-    def test_extended_serialization_roundtrip(self, extended_grid: ExtendedGrid, temp_dir: Path, format_type: str):
+    def test_extended_serialization_roundtrip(self, extended_grid: ExtendedGrid, temp_dir: Path):
         """Test extended serialization preserving custom data"""
-        ext = "json" if format_type == "json" else "msgpack"
-        path = temp_dir / f"extended.{ext}"
+        path = temp_dir / "extended.json"
 
-        # Save with extensions
-        if format_type == "json":
-            save_grid_to_json(extended_grid, path, preserve_extensions=True)
-            loaded_grid = load_grid_from_json(path, target_grid_class=ExtendedGrid)
-        else:
-            save_grid_to_msgpack(extended_grid, path, preserve_extensions=True)
-            loaded_grid = load_grid_from_msgpack(path, target_grid_class=ExtendedGrid)
+        save_grid_to_json(extended_grid, path, preserve_extensions=True)
+        loaded_grid = load_grid_from_json(path, target_grid_class=ExtendedGrid)
 
         # Verify core data
         assert loaded_grid.node.size == extended_grid.node.size
@@ -148,37 +123,25 @@ def test_extended_serialization_roundtrip(self, extended_grid: ExtendedGrid, tem
 class TestCrossTypeCompatibility:
     """Test cross-type loading and compatibility"""
 
-    @pytest.mark.parametrize("format_type", ["json", "msgpack"])
-    def test_basic_to_extended_loading(self, basic_grid: Grid, temp_dir: Path, format_type: str):
+    def test_basic_to_extended_loading(self, basic_grid: Grid, temp_dir: Path):
         """Test loading basic grid into extended type"""
-        ext = "json" if format_type == "json" else "msgpack"
-        path = temp_dir / f"basic.{ext}"
+        path = temp_dir / "basic.json"
 
         # Save basic grid
-        if format_type == "json":
-            save_grid_to_json(basic_grid, path)
-            loaded_grid = load_grid_from_json(path, target_grid_class=ExtendedGrid)
-        else:
-            save_grid_to_msgpack(basic_grid, path)
-            loaded_grid = load_grid_from_msgpack(path, target_grid_class=ExtendedGrid)
+        save_grid_to_json(basic_grid, path)
+        loaded_grid = load_grid_from_json(path, target_grid_class=ExtendedGrid)
 
         # Core data should transfer
         assert loaded_grid.node.size == basic_grid.node.size
         assert loaded_grid.line.size == basic_grid.line.size
 
-    @pytest.mark.parametrize("format_type", ["json", "msgpack"])
-    def test_extended_to_basic_loading(self, extended_grid: ExtendedGrid, temp_dir: Path, format_type: str):
+    def test_extended_to_basic_loading(self, extended_grid: ExtendedGrid, temp_dir: Path):
         """Test loading extended grid into basic type"""
-        ext = "json" if format_type == "json" else "msgpack"
-        path = temp_dir / f"extended.{ext}"
+        path = temp_dir / "extended.json"
 
         # Save extended grid
-        if format_type == "json":
-            save_grid_to_json(extended_grid, path, preserve_extensions=True)
-            loaded_grid = load_grid_from_json(path, target_grid_class=Grid)
-        else:
-            save_grid_to_msgpack(extended_grid, path, preserve_extensions=True)
-            loaded_grid = load_grid_from_msgpack(path, target_grid_class=Grid)
+        save_grid_to_json(extended_grid, path, preserve_extensions=True)
+        loaded_grid = load_grid_from_json(path, target_grid_class=Grid)
 
         # Core data should transfer
         assert loaded_grid.node.size == extended_grid.node.size
@@ -251,46 +214,6 @@ class GridWithCustomArray(Grid):
         np.testing.assert_array_almost_equal(loaded_grid.custom_metadata.metadata_value, [1.5, 2.5, 3.5])
         np.testing.assert_array_equal(loaded_grid.custom_metadata.category, [1, 2, 1])
 
-        # Test MessagePack serialization
-        msgpack_path = temp_dir / "custom_array.msgpack"
-        save_grid_to_msgpack(grid, msgpack_path, preserve_extensions=True)
-
-        # Load back and verify
-        loaded_grid_mp = load_grid_from_msgpack(msgpack_path, target_grid_class=GridWithCustomArray)
-
-        # Verify core data
-        assert loaded_grid_mp.node.size == 2
-        np.testing.assert_array_equal(loaded_grid_mp.node.id, [1, 2])
-
-        # Verify custom array was preserved
-        assert hasattr(loaded_grid_mp, "custom_metadata")
-        assert loaded_grid_mp.custom_metadata.size == 3
-        np.testing.assert_array_equal(loaded_grid_mp.custom_metadata.id, [100, 200, 300])
-        np.testing.assert_array_almost_equal(loaded_grid_mp.custom_metadata.metadata_value, [1.5, 2.5, 3.5])
-        np.testing.assert_array_equal(loaded_grid_mp.custom_metadata.category, [1, 2, 1])
-
-
-class TestUtilityFunctions:
-    """Test utility functions and path handling"""
-
-    @pytest.mark.parametrize(
-        "input_path,format_type,expected",
-        [
-            ("test.json", "auto", "test.json"),
-            ("test.msgpack", "auto", "test.msgpack"),
-            ("test.mp", "auto", "test.mp"),
-            ("test.xyz", "auto", "test.json"),  # Unknown defaults to JSON
-            ("test.xyz", "json", "test.json"),
-            ("test.xyz", "msgpack", "test.msgpack"),
-        ],
-    )
-    def test_serialization_path_handling(
-        self, input_path: str, format_type: Literal["json", "msgpack", "auto"], expected: str
-    ):
-        """Test path handling and format detection"""
-        result = _get_serialization_path(Path(input_path), format_type)
-        assert result == Path(expected)
-
 
 class TestSpecialCases:
     """Test special cases and edge scenarios"""
@@ -300,18 +223,13 @@ def test_empty_grid_handling(self, temp_dir: Path):
         empty_grid = Grid.empty()
 
         json_path = temp_dir / "empty.json"
-        msgpack_path = temp_dir / "empty.msgpack"
 
         # Should handle empty grids
         save_grid_to_json(empty_grid, json_path)
-        save_grid_to_msgpack(empty_grid, msgpack_path)
 
         # Should load back as empty
         loaded_json = load_grid_from_json(json_path, target_grid_class=Grid)
-        loaded_msgpack = load_grid_from_msgpack(msgpack_path, target_grid_class=Grid)
-
         assert loaded_json.node.size == 0
-        assert loaded_msgpack.node.size == 0
 
     def test_custom_array_extraction_edge_cases(self, temp_dir: Path):
         """Test edge cases in custom array extraction"""