[ENH] Serialize grid

Author: Leguark

gempy/core/data/encoders/binary_encoder.py

@@ -60,15 +60,10 @@ def deserialize_grid(binary_array:bytes, custom_grid_length: int, topography_len
         ValueError: If input lengths do not match the specified boundaries or binary data.
     """
 
-    total_length = len(binary_array)
-    custom_grid_start = total_length - custom_grid_length - topography_length
-    custom_grid_end = total_length - topography_length
 
-    topography_grid_start = total_length - topography_length
-    topography_grid_end = total_length
 
-    custom_grid_binary = binary_array[custom_grid_start:custom_grid_end]
-    topography_binary = binary_array[topography_grid_start:topography_grid_end]
+    custom_grid_binary = binary_array[:custom_grid_length]
+    topography_binary = binary_array[custom_grid_length:custom_grid_length + topography_length]
     custom_grid = np.frombuffer(custom_grid_binary, dtype=np.float64)
     topography = np.frombuffer(topography_binary)
 

gempy/core/data/encoders/converters.py

@@ -52,9 +52,10 @@ def instantiate_if_necessary(data: dict, key: str, type: type) -> None:
 loading_model_context = ContextVar('loading_model_context', default={})
 
 @contextmanager
-def loading_model_from_binary(binary_body: bytes):
+def loading_model_from_binary(input_binary: bytes, grid_binary: bytes):
     token = loading_model_context.set({
-            'binary_body': binary_body,
+            'input_binary': input_binary,
+            'grid_binary': grid_binary
     })
     try:
         yield

gempy/core/data/grid.py

@@ -66,11 +66,11 @@ def deserialize_properties(cls, data: Union["Grid", dict], constructor: ModelWra
                     metadata = data.get('binary_meta_data', {})
                     context = loading_model_context.get()
 
-                    if 'binary_body' not in context:
+                    if 'grid_binary' not in context:
                         return grid
 
                     custom_grid_vals, topography_vals = deserialize_grid(
-                        binary_array=context['binary_body'],
+                        binary_array=context['grid_binary'],
                         custom_grid_length=metadata["custom_grid_binary_length"],
                         topography_length=metadata["topography_binary_length"]
                     )
@@ -90,16 +90,18 @@ def deserialize_properties(cls, data: Union["Grid", dict], constructor: ModelWra
 
     @property
     def grid_binary(self):
-        astype = self._custom_grid.values.astype("float64")
-        custom_grid_bytes = astype.tobytes() if self._custom_grid else b''
+        custom_grid_bytes = self._custom_grid.values.astype("float64").tobytes() if self._custom_grid else b''
         topography_bytes = self._topography.values.astype("float64").tobytes() if self._topography else b''
         return custom_grid_bytes + topography_bytes
 
+
+    _grid_binary_size: int = 0
     @computed_field
     def binary_meta_data(self) -> dict:
         return {
                 'custom_grid_binary_length': len(self._custom_grid.values.astype("float64").tobytes()) if self._custom_grid else 0,
-                'topography_binary_length': len(self._topography.values.astype("float64").tobytes()) if self._topography else 0
+                'topography_binary_length': len(self._topography.values.astype("float64").tobytes()) if self._topography else 0,
+                'grid_binary_size': self._grid_binary_size
         }
 
     @computed_field(alias="active_grids")

gempy/core/data/structural_frame.py

@@ -476,11 +476,11 @@ def deserialize_binary(cls, data: Union["StructuralFrame", dict], constructor: M
                 metadata = data.get('binary_meta_data', {})
                 context = loading_model_context.get()
 
-                if 'binary_body' not in context:
+                if 'input_binary' not in context:
                     return instance
 
                 instance.orientations, instance.surface_points = deserialize_input_data_tables(
-                    binary_array=context['binary_body'],
+                    binary_array=context['input_binary'],
                     name_id_map=instance.surface_points_copy.name_id_map,
                     sp_binary_length_=metadata["sp_binary_length"],
                     ori_binary_length_=metadata["ori_binary_length"]
@@ -492,12 +492,13 @@ def deserialize_binary(cls, data: Union["StructuralFrame", dict], constructor: M
 
         # Access the context variable to get injected data
 
-
+    _input_binary_size: int = 0
     @computed_field
     def binary_meta_data(self) -> dict:
         return {
                 'sp_binary_length': len(self.surface_points_copy.data.tobytes()),
-                'ori_binary_length': len(self.orientations_copy.data.tobytes()) 
+                'ori_binary_length': len(self.orientations_copy.data.tobytes()) ,
+                'input_binary_size': self._input_binary_size
         }
 
     # endregion

gempy/modules/serialization/save_load.py

@@ -62,11 +62,17 @@ def save_model(model: GeoModel, path: str | None = None, validate_serialization:
 
 
 def model_to_binary(model: GeoModel) -> bytes:
-    model_json = model.model_dump_json(by_alias=True, indent=4)
+
     # Compress the binary data
     zlib = require_zlib()
     compressed_binary_input = zlib.compress(model.structural_frame.input_tables_binary)
     compressed_binary_grid = zlib.compress(model.grid.grid_binary)
+
+    # * Add here the serialization meta parameters like: len_bytes
+    model.structural_frame._input_binary_size = len(compressed_binary_input)
+    model.grid._grid_binary_size = len(compressed_binary_grid)
+    
+    model_json = model.model_dump_json(by_alias=True, indent=4)
     binary_file = _to_binary(
         header_json=model_json,
         body_input=compressed_binary_input,
@@ -118,15 +124,29 @@ def load_model(path: str) -> GeoModel:
 
 
 def _deserialize_binary_file(binary_file):
+    import json
     # Get header length from first 4 bytes
     header_length = int.from_bytes(binary_file[:4], byteorder='little')
     # Split header and body
-    header_json = binary_file[4:4 + header_length].decode('utf-8')
-    binary_body = binary_file[4 + header_length:]
+    header_json= binary_file[4:4 + header_length].decode('utf-8')
+    header = json.loads(header_json)
+    input_metadata = header["structural_frame"]["binary_meta_data"]
+    input_size = input_metadata["input_binary_size"]
+    
+    grid_metadata = header["grid"]["binary_meta_data"]
+    grid_size = grid_metadata["grid_binary_size"]
+    
+    input_binary = binary_file[4 + header_length: 4 + header_length + input_size]
+    all_sections_length = 4 + header_length + input_size + grid_size
+    if all_sections_length != len(binary_file):
+        raise  ValueError("Binary file is corrupted")
+    
+    grid_binary = binary_file[4 + header_length + input_size: all_sections_length]
     zlib = require_zlib()
-    decompressed_binary = zlib.decompress(binary_body)
+    
     with loading_model_from_binary(
-            binary_body=decompressed_binary,
+            input_binary=(zlib.decompress(input_binary)),
+            grid_binary=(zlib.decompress(grid_binary))
     ):
         model = GeoModel.model_validate_json(header_json)
     return model

test/test_modules/test_grids/test_custom_grid.py

@@ -16,7 +16,8 @@
      [0, 0, 1000],
      [1000, 0, 1000],
      [0, 1000, 1000],
-     [1000, 1000, 1000]]
+     [1000, 1000, 1000]],
+    dtype=float
 )
 
 

test/test_modules/test_grids/test_custom_grid.test_custom_grid.verify/fold.approved.txt

@@ -61,9 +61,11 @@
         ],
         "is_dirty": true,
         "basement_color": "#ffbe00",
+        "_input_binary_size": 233,
         "binary_meta_data": {
             "sp_binary_length": 1296,
-            "ori_binary_length": 120
+            "ori_binary_length": 120,
+            "input_binary_size": 233
         }
     },
     "grid": {
@@ -90,9 +92,11 @@
         "_centered_grid": null,
         "_transform": null,
         "_octree_levels": -1,
+        "_grid_binary_size": 29,
         "binary_meta_data": {
             "custom_grid_binary_length": 192,
-            "topography_binary_length": 0
+            "topography_binary_length": 0,
+            "grid_binary_size": 29
         },
         "active_grids": 1029
     },