Fix updater for hdf5 files with custom data, and add hdf5 files to automatic updater testing

Author: momchil-flex

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 - Properly handle `.freqs` in `output_monitors` of adjoint plugin.
+- Simulation updater for `.hdf5` files with custom data.
 
 ## [2.4.2] - 2023-9-28
 

tests/sims/simulation_2_4_2.h5

@@ -50,9 +50,14 @@ def set_datasets_to_none(sim):
 def test_simulation_load_export():
     major, minor, patch = __version__.split(".")
     path = os.path.join(SIM_DIR, f"simulation_{major}_{minor}_{patch}.json")
+    # saving as .h5 since *.hdf5 is git ignored
+    path_hdf5 = os.path.join(SIM_DIR, f"simulation_{major}_{minor}_{patch}.h5")
     SIM.to_file(path)
+    SIM.to_hdf5(path_hdf5)
     SIM2 = td.Simulation.from_file(path)
+    SIM_HDF5 = td.Simulation.from_hdf5(path_hdf5)
     assert set_datasets_to_none(SIM) == SIM2, "original and loaded simulations are not the same"
+    assert SIM == SIM_HDF5, "original and loaded from hdf5 simulations are not the same"
 
 
 def test_simulation_load_export_yaml(tmp_path):

tests/test_components/test_IO.py

@@ -65,9 +65,10 @@ def ndarray_encoder(val):
 def _get_valid_extension(fname: str) -> str:
     """Return the file extension from fname, validated to accepted ones."""
     extension = "".join(pathlib.Path(fname).suffixes).lower()
-    if extension not in {".json", ".yaml", ".hdf5", ".hdf5.gz"}:
+    valid_extensions = [".json", ".yaml", ".hdf5", ".hdf5.gz", ".h5"]
+    if extension not in valid_extensions:
         raise FileError(
-            f"{fname}::File extension must be .json, .yaml, .hdf5, or .hdf5.gz, but '{extension}' given"
+            f"{fname}::File extension must be in {valid_extensions}, but '{extension}' given"
         )
     return extension
 
@@ -225,6 +226,7 @@ def dict_from_file(cls, fname: str, group_path: str = None) -> dict:
             ".yaml": cls.dict_from_yaml,
             ".hdf5": cls.dict_from_hdf5,
             ".hdf5.gz": cls.dict_from_hdf5_gz,
+            ".h5": cls.dict_from_hdf5,
         }[extension]
         return converter(**kwargs)
 

tidy3d/components/base.py

@@ -210,7 +210,7 @@ def update_1_8(sim_dict: dict) -> dict:
 
     def fix_missing_scalar_field(mnt_dict: dict) -> dict:
         for key, val in mnt_dict["field_dataset"].items():
-            if val == "XR.DATAARRAY":
+            if isinstance(val, str) and val == "XR.DATAARRAY":
                 mnt_dict["field_dataset"][key] = "ScalarFieldDataArray"
         return mnt_dict