feat(data storage): Data automatically stored internally when simulation or model relative path changed (#1126) (#1157)

* refact(paths): New simulation and model method "set_all_data_internal" added that sets all data internal.  This method is automatically called whenever the simulation or model path are changed.  This greatly simplifies flopy's behavior since flopy no longer needs to keep track of former and new external file paths that can possibly change multiple times before write is called.

* refact(path): Added combination of changing model relative path and setting all data external to test case.

* fix(cleanup): code cleanup

* fix(code cleanup): code cleanup

* fix(set data internal/external): code made more general to handle transient data without stress period keys

* fix(formatting)

* feat(data columns + paths): Flopy now automatically defaults to ncol for max_columns_of_data. Added support for specifying where to store external files when calling set_all_files_external.  Path is relative to model(s)/simulation.  Added several more path tests.

Author: spaulins-usgs

autotest/t504_test.py

@@ -76,9 +76,12 @@ def test001a_tharmonic():
     sim.simulation_data.mfpath.set_sim_path(run_folder)
 
     # write simulation to new location
-    sim.set_all_data_external()
+    sim.set_all_data_external(external_data_folder="data")
     sim.write_simulation(silent=True)
-
+    # verify external data written to correct location
+    data_path = os.path.join(run_folder, "data", "flow15.dis_botm.txt")
+    assert os.path.exists(data_path)
+    # model export test
     model = sim.get_model(model_name)
     model.export("{}/tharmonic.nc".format(model.model_ws))
     model.export("{}/tharmonic.shp".format(model.model_ws))

autotest/t505_test.py

@@ -34,6 +34,7 @@
 from flopy.mf6.modflow.mfutlts import ModflowUtlts
 from flopy.mf6.utils import testutils
 from flopy.mf6.mfbase import MFDataException
+from flopy.mf6.mfbase import ExtFileAction
 
 try:
     import shapefile
@@ -392,6 +393,9 @@ def np001():
     # write simulation to new location
     sim.set_all_data_external()
     sim.write_simulation()
+    assert (
+        sim.simulation_data.max_columns_of_data == dis_package.ncol.get_data()
+    )
 
     # run simulation
     if run:
@@ -420,6 +424,44 @@ def np001():
         # clean up
         sim.delete_output_files()
 
+    # test path changes, model file path relative to the simulation folder
+    md_folder = "model_folder"
+    model.set_model_relative_path(md_folder)
+    run_folder_new = os.path.join(run_folder, md_folder)
+    # set all data external
+    sim.set_all_data_external(external_data_folder="data")
+    sim.write_simulation()
+
+    assert (
+        sim.simulation_data.max_columns_of_data == dis_package.ncol.get_data()
+    )
+    # run simulation from new path with external files
+    if run:
+        sim.run_simulation()
+
+        # get expected results
+        budget_file = os.path.join(os.getcwd(), expected_cbc_file)
+        budget_obj = bf.CellBudgetFile(budget_file, precision="double")
+        budget_frf_valid = np.array(
+            budget_obj.get_data(text="FLOW-JA-FACE", full3D=True)
+        )
+
+        # compare output to expected results
+        head_file = os.path.join(os.getcwd(), expected_head_file)
+        head_new = os.path.join(run_folder_new, "np001_mod.hds")
+        outfile = os.path.join(run_folder_new, "head_compare.dat")
+        assert pymake.compare_heads(
+            None, None, files1=head_file, files2=head_new, outfile=outfile
+        )
+
+        budget_frf = sim.simulation_data.mfdata[
+            (model_name, "CBC", "FLOW-JA-FACE")
+        ]
+        assert array_util.array_comp(budget_frf_valid, budget_frf)
+
+        # clean up
+        sim.delete_output_files()
+
     try:
         error_occurred = False
         well_spd = {
@@ -594,6 +636,8 @@ def np002():
         sim_ws=run_folder,
         nocheck=True,
     )
+    sim.simulation_data.max_columns_of_data = 22
+
     name = sim.name_file
     assert name.continue_.get_data() == None
     assert name.nocheck.get_data() == True
@@ -659,6 +703,9 @@ def np002():
         idomain=2,
         filename="{}.dis".format(model_name),
     )
+    assert sim.simulation_data.max_columns_of_data == 22
+    sim.simulation_data.max_columns_of_data = dis_package.ncol.get_data()
+
     ic_vals = [
         100.0,
         100.0,
@@ -815,6 +862,20 @@ def np002():
     spd2 = ghb2.stress_period_data.get_data(1)
     assert spd2 is None
 
+    # test paths
+    sim_path_test = os.path.join(run_folder, "sim_path")
+    sim.set_sim_path(sim_path_test)
+    model.set_model_relative_path("model")
+    # make external data folder path relative to simulation folder
+    ext_folder = os.path.join("..", "data")
+    sim.set_all_data_external(external_data_folder=ext_folder)
+    sim.write_simulation()
+    # test
+    ext_full_path = os.path.join(sim_path_test, "data")
+    assert os.path.exists(
+        os.path.join(ext_full_path, "np002_mod.dis_botm.txt")
+    )
+
     return
 
 
@@ -2525,7 +2586,6 @@ def test006_2models_gnc():
     sim.simulation_data.mfpath.set_sim_path(run_folder)
 
     # write simulation to new location
-    sim.set_all_data_external()
     sim.write_simulation()
 
     # run simulation
@@ -2548,6 +2608,24 @@ def test006_2models_gnc():
             None, None, files1=head_file, files2=head_new, outfile=outfile
         )
 
+    # test external file paths
+    sim_path = os.path.join(run_folder, "path_test")
+    sim.simulation_data.mfpath.set_sim_path(sim_path)
+    model_1.set_model_relative_path("model1")
+    model_2.set_model_relative_path("model2")
+    sim.set_all_data_external(external_data_folder="data")
+    sim.write_simulation()
+    ext_file_path_1 = os.path.join(
+        sim_path, "model1", "data", "model1.dis_botm.txt"
+    )
+    assert os.path.exists(ext_file_path_1)
+    ext_file_path_2 = os.path.join(
+        sim_path, "model2", "data", "model2.dis_botm.txt"
+    )
+    assert os.path.exists(ext_file_path_2)
+    if run:
+        sim.run_simulation()
+
         # clean up
         sim.delete_output_files()
 
@@ -3165,8 +3243,8 @@ def test_transport():
 
 
 if __name__ == "__main__":
-    np002()
     np001()
+    np002()
     test004_bcfss()
     test005_advgw_tidal()
     test006_2models_gnc()

flopy/mf6/data/mfdataarray.py

@@ -3,12 +3,13 @@
 from collections import OrderedDict
 from ..data.mfstructure import DatumType
 from .mfdatastorage import DataStorage, DataStructureType, DataStorageType
-from ...utils.datautil import MultiList
+from ...utils.datautil import MultiList, DatumUtil
 from ..mfbase import ExtFileAction, MFDataException, VerbosityLevel
 from ..utils.mfenums import DiscretizationType
 from ...datbase import DataType
 from .mffileaccess import MFFileAccessArray
 from .mfdata import MFMultiDimVar, MFTransient
+from ...mbase import ModelInterface
 
 
 class MFArray(MFMultiDimVar):
@@ -483,7 +484,6 @@ def store_as_external_file(
             check_data : bool
                 Verify data prior to storing
         """
-
         storage = self._get_storage_obj()
         if storage is None:
             self._set_storage_obj(self._new_storage(False, True))
@@ -563,7 +563,6 @@ def store_as_external_file(
                 self._set_data(
                     external_data, layer=current_layer, check_data=False
                 )
-
             except Exception as ex:
                 type_, value_, traceback_ = sys.exc_info()
                 raise MFDataException(
@@ -582,6 +581,93 @@ def store_as_external_file(
                     ex,
                 )
 
+    def store_internal(
+        self,
+        layer=None,
+        check_data=True,
+    ):
+        """Stores data from layer `layer` internally.  For unlayered data do
+        not pass in `layer`.  If layer is not specified all layers will be
+        stored internally
+
+        Parameters
+        ----------
+            layer : int
+                Which layer to store in external file, `None` value stores all
+                layers.
+            check_data : bool
+                Verify data prior to storing
+        """
+        storage = self._get_storage_obj()
+        if storage is None:
+            self._set_storage_obj(self._new_storage(False, True))
+            storage = self._get_storage_obj()
+        # build list of layers
+        if layer is None:
+            layer_list = []
+            for index in range(0, storage.layer_storage.get_total_size()):
+                if (
+                    storage.layer_storage[index].data_storage_type
+                    == DataStorageType.external_file
+                ):
+                    layer_list.append(index)
+        else:
+            if (
+                storage.layer_storage[layer].data_storage_type
+                == DataStorageType.external_file
+            ):
+                layer_list = [layer]
+            else:
+                layer_list = []
+
+        # store data from each layer
+        for current_layer in layer_list:
+            if isinstance(current_layer, int):
+                current_layer = (current_layer,)
+            # get the layer's data
+            data = self._get_data(current_layer, True)
+
+            if data is None:
+                # do not write empty data to an internal file
+                continue
+            try:
+                # store layer's data internally
+                if (
+                    self._simulation_data.verbosity_level.value
+                    >= VerbosityLevel.verbose.value
+                ):
+                    print(
+                        "Storing {} layer {} internally.."
+                        ".".format(
+                            self.structure.name,
+                            current_layer[0] + 1,
+                        )
+                    )
+                factor = storage.layer_storage[current_layer].factor
+                internal_data = {
+                    "data": self._get_data(current_layer, True),
+                    "factor": factor,
+                }
+                self._set_data(
+                    internal_data, layer=current_layer, check_data=False
+                )
+            except Exception as ex:
+                type_, value_, traceback_ = sys.exc_info()
+                raise MFDataException(
+                    self.structure.get_model(),
+                    self.structure.get_package(),
+                    self._path,
+                    "storing data {} internally".format(self.structure.name),
+                    self.structure.name,
+                    inspect.stack()[0][3],
+                    type_,
+                    value_,
+                    traceback_,
+                    None,
+                    self._simulation_data.debug,
+                    ex,
+                )
+
     def has_data(self, layer=None):
         """Returns whether layer "layer_num" has any data associated with it.
 
@@ -1481,32 +1567,57 @@ def store_as_external_file(
             check_data : bool
                 Verify data prior to storing
         """
-
-        sim_time = self._data_dimensions.package_dim.model_dim[
-            0
-        ].simulation_time
-        num_sp = sim_time.get_num_stress_periods()
         # store each stress period in separate file(s)
-        for sp in range(0, num_sp):
-            if sp in self._data_storage:
-                self._current_key = sp
-                layer_storage = self._get_storage_obj().layer_storage
-                if (
-                    layer_storage.get_total_size() > 0
-                    and self._get_storage_obj()
-                    .layer_storage[0]
-                    .layer_storage_type
-                    != DataStorageType.external_file
-                ):
-                    fname, ext = os.path.splitext(external_file_path)
+        for sp in self._data_storage.keys():
+            self._current_key = sp
+            layer_storage = self._get_storage_obj().layer_storage
+            if (
+                layer_storage.get_total_size() > 0
+                and self._get_storage_obj().layer_storage[0].data_storage_type
+                != DataStorageType.external_file
+            ):
+                fname, ext = os.path.splitext(external_file_path)
+                if DatumUtil.is_int(sp):
                     full_name = "{}_{}{}".format(fname, sp + 1, ext)
-                    super().store_as_external_file(
-                        full_name,
-                        layer,
-                        binary,
-                        replace_existing_external,
-                        check_data,
-                    )
+                else:
+                    full_name = "{}_{}{}".format(fname, sp, ext)
+                super().store_as_external_file(
+                    full_name,
+                    layer,
+                    binary,
+                    replace_existing_external,
+                    check_data,
+                )
+
+    def store_internal(
+        self,
+        layer=None,
+        check_data=True,
+    ):
+        """Stores data from layer `layer` internally.  For unlayered data do
+        not pass in `layer`. If layer is not specified all layers will be
+        stored internally.
+
+        Parameters
+        ----------
+            layer : int
+                Which layer to store internally file, `None` value stores all
+                layers.
+            check_data : bool
+                Verify data prior to storing
+        """
+        for sp in self._data_storage.keys():
+            self._current_key = sp
+            layer_storage = self._get_storage_obj().layer_storage
+            if (
+                layer_storage.get_total_size() > 0
+                and self._get_storage_obj().layer_storage[0].data_storage_type
+                == DataStorageType.external_file
+            ):
+                super().store_internal(
+                    layer,
+                    check_data,
+                )
 
     def get_data(self, layer=None, apply_mult=True, **kwargs):
         """Returns the data associated with stress period key `layer`.