[ENH] Improve binary deserialization and grid reconstruction

Added support for handling orientation binary lengths and grid-specific binary deserialization, including custom grid and topography data. Enhanced grid metadata processing and validation to enable accurate model reconstruction.

Author: Leguark

gempy/core/data/encoders/binary_encoder.py

@@ -4,12 +4,73 @@
 from ..orientations import OrientationsTable
 
 
-def deserialize_input_data_tables(binary_array: bytes, name_id_map: dict, sp_binary_length_: int) -> tuple[OrientationsTable, SurfacePointsTable]:
+def deserialize_input_data_tables(binary_array: bytes, name_id_map: dict, 
+                                  sp_binary_length_: int, ori_binary_length_: int) -> tuple[OrientationsTable, SurfacePointsTable]:
+    """
+    Deserializes binary data into two tables: OrientationsTable and SurfacePointsTable.
+
+    This function takes a binary array, a mapping of names to IDs, and lengths for
+    specific parts of the binary data to extract and deserialize two distinct data
+    tables: OrientationsTable and SurfacePointsTable. It uses the provided lengths
+    to split the binary data accordingly and reconstructs the table contents from
+    their respective binary representations.
+
+    Args:
+        binary_array (bytes): A bytes array containing the serialized data for
+            both the OrientationsTable and SurfacePointsTable.
+        name_id_map (dict): A dictionary mapping names to IDs which is used to
+            help reconstruct the table objects.
+        sp_binary_length_ (int): The length of the binary segment corresponding
+            to the SurfacePointsTable data.
+        ori_binary_length_ (int): The length of the binary segment corresponding
+            to the OrientationsTable data.
+
+    Returns:
+        tuple[OrientationsTable, SurfacePointsTable]: A tuple containing two table
+        objects: first the OrientationsTable, and second the SurfacePointsTable.
+    """
     sp_binary = binary_array[:sp_binary_length_]
-    ori_binary = binary_array[sp_binary_length_:]
+    ori_binary = binary_array[sp_binary_length_:sp_binary_length_+ori_binary_length_]
     # Reconstruct arrays
     sp_data: np.ndarray = np.frombuffer(sp_binary, dtype=SurfacePointsTable.dt)
     ori_data: np.ndarray = np.frombuffer(ori_binary, dtype=OrientationsTable.dt)
     surface_points_table = SurfacePointsTable(data=sp_data, name_id_map=name_id_map)
     orientations_table = OrientationsTable(data=ori_data, name_id_map=name_id_map)
     return orientations_table, surface_points_table
+
+
+def deserialize_grid(binary_array:bytes, custom_grid_length: int, topography_length: int) -> tuple[np.ndarray, np.ndarray]:
+    """
+    Deserialize binary grid data into two numpy arrays.
+
+    This function takes a binary array representing a grid and splits it into two separate
+    numpy arrays: one for the custom grid and one for the topography. The binary array is
+    segmented based on the provided lengths for the custom grid and topography.
+
+    Args:
+        binary_array: The binary data representing the combined custom grid and topography data.
+        custom_grid_length: The length of the custom grid data segment in bytes.
+        topography_length: The length of the topography data segment in bytes.
+
+    Returns:
+        A tuple where the first element is a numpy array representing the custom grid, and
+        the second element is a numpy array representing the topography data.
+
+    Raises:
+        ValueError: If input lengths do not match the specified boundaries or binary data.
+    """
+
+    total_length = len(binary_array)
+    custom_grid_start = total_length - custom_grid_length - topography_length
+    custom_grid_end = total_length - topography_length
+    
+    topography_grid_start = total_length - topography_length
+    topography_grid_end = total_length
+
+    custom_grid_binary = binary_array[custom_grid_start:custom_grid_end]
+    topography_binary = binary_array[topography_grid_start:topography_grid_end]
+    custom_grid = np.frombuffer(custom_grid_binary)
+    topography = np.frombuffer(topography_binary)
+    
+    
+    return custom_grid, topography

gempy/core/data/grid.py

@@ -8,6 +8,8 @@
 from gempy_engine.core.data.centered_grid import CenteredGrid
 from gempy_engine.core.data.options import EvaluationOptions
 from gempy_engine.core.data.transforms import Transform
+from .encoders.binary_encoder import deserialize_grid
+from .encoders.converters import loading_model_context
 from .grid_modules import RegularGrid, CustomGrid, Sections
 from .grid_modules.topography import Topography
 
@@ -61,6 +63,24 @@ def deserialize_properties(cls, data: Union["Grid", dict], constructor: ModelWra
                     grid._active_grids = Grid.GridTypes(data["active_grids"])
                     # TODO: Digest binary data
 
+                    metadata = data.get('binary_meta_data', {})
+                    context = loading_model_context.get()
+
+                    if 'binary_body' not in context:
+                        return grid
+                    
+                    custom_grid_vals, topography_vals = deserialize_grid(
+                        binary_array=context['binary_body'],
+                        custom_grid_length=metadata["custom_grid_binary_length"],
+                        topography_length=metadata["topography_binary_length"]
+                    )
+                    
+                    if grid.custom_grid is not None:
+                        grid.custom_grid.values = custom_grid_vals.reshape(-1, 3)
+                    
+                    if grid.topography is not None:
+                        grid.topography.set_values2d(values=topography_vals)
+                    
                     grid._update_values()
                     return grid
                 case _:

gempy/core/data/grid_modules/topography.py

@@ -154,6 +154,9 @@ def set_values(self, values_2d: np.ndarray):
         # n,3 array
         self.values = values_2d.reshape((-1, 3), order='C')
         return self
+    
+    def set_values2d(self, values: np.ndarray):
+        self.values_2d = values.reshape(self.resolution)
 
     @property
     def topography_mask(self):

gempy/core/data/structural_frame.py

@@ -482,7 +482,8 @@ def deserialize_binary(cls, data: Union["StructuralFrame", dict], constructor: M
                 instance.orientations, instance.surface_points = deserialize_input_data_tables(
                     binary_array=context['binary_body'],
                     name_id_map=instance.surface_points_copy.name_id_map,
-                    sp_binary_length_=metadata["sp_binary_length"]
+                    sp_binary_length_=metadata["sp_binary_length"],
+                    ori_binary_length_=metadata["ori_binary_length"]
                 )
 
                 return instance

test/test_modules/test_grids/test_custom_grid.py

@@ -39,7 +39,7 @@ def test_custom_grid():
         file_name=f"verify/{geo_model.meta.name}"
     )
 
-    sol: gp.data.Solutions = gp.compute_model(geo_model, validate_serialization=False)
+    sol: gp.data.Solutions = gp.compute_model(geo_model, validate_serialization=True)
     np.testing.assert_array_equal(
         sol.raw_arrays.custom,
         np.array([3., 3., 3., 3., 1., 1., 1., 1.])

test/test_modules/test_grids/test_custom_grid.test_custom_grid.verify/fold.approved.txt

@@ -62,7 +62,8 @@
         "is_dirty": true,
         "basement_color": "#ffbe00",
         "binary_meta_data": {
-            "sp_binary_length": 1296
+            "sp_binary_length": 1296,
+            "ori_binary_length": 120
         }
     },
     "grid": {
@@ -89,6 +90,10 @@
         "_centered_grid": null,
         "_transform": null,
         "_octree_levels": -1,
+        "binary_meta_data": {
+            "custom_grid_binary_length": 192,
+            "topography_binary_length": 0
+        },
         "active_grids": 1029
     },
     "geophysics_input": null,