Some enhancements to the import/export rules functionality

gridpath/common_functions.py

@@ -305,6 +305,14 @@ def get_import_results_parser():
         "--results_import_rule",
         help="The name of the rule to use to decide whether to import results.",
     )
+    parser.add_argument(
+        "--ignore_incomplete",
+        default=False,
+        action="store_true",
+        help="Ignore problems with no results. Can be used to import results "
+        "for subproblems before all other subproblems have been solved. "
+        "Proceed with caution.",
+    )
 
     return parser
 

gridpath/import_scenario_results.py

@@ -21,6 +21,7 @@
 *gridpath_import_results* command when GridPath is installed.
 """
 
+import warnings
 from argparse import ArgumentParser
 import os.path
 import pandas as pd
@@ -63,6 +64,7 @@ def import_scenario_results_into_database(
     scenario_structure,
     db,
     scenario_directory,
+    ignore_incomplete,
     quiet,
 ):
     """
@@ -72,6 +74,7 @@ def import_scenario_results_into_database(
     :param scenario_structure:
     :param db:
     :param scenario_directory:
+    :param ignore_incomplete: boolean
     :param quiet: boolean
 
     :return:
@@ -148,13 +151,28 @@ def import_scenario_results_into_database(
 
                         # Import termination condition data
                         c = db.cursor()
-                        with open(
-                            os.path.join(
-                                results_directory, "termination_condition.txt"
-                            ),
-                            "r",
-                        ) as f:
-                            termination_condition = f.read()
+                        try:
+                            with open(
+                                os.path.join(
+                                    results_directory, "termination_condition.txt"
+                                ),
+                                "r",
+                            ) as f:
+                                termination_condition = f.read()
+                        except FileNotFoundError:
+                            if ignore_incomplete:
+                                warnings.warn(
+                                    "GridPath Warning: termination "
+                                    "condition file not found."
+                                )
+                                termination_condition = (
+                                    "termination condition file not found"
+                                )
+                            else:
+                                tc_fname = os.path.join(
+                                    results_directory, "termination_condition.txt"
+                                )
+                                raise FileNotFoundError(f"{tc_fname} not " f"found.")
 
                         termination_condition_sql = """
                             INSERT INTO results_scenario
@@ -180,10 +198,23 @@ def import_scenario_results_into_database(
                             many=False,
                         )
 
-                        with open(
-                            os.path.join(results_directory, "solver_status.txt"), "r"
-                        ) as status_f:
-                            solver_status = status_f.read()
+                        try:
+                            with open(
+                                os.path.join(results_directory, "solver_status.txt"),
+                                "r",
+                            ) as status_f:
+                                solver_status = status_f.read()
+                        except FileNotFoundError:
+                            if ignore_incomplete:
+                                warnings.warn(
+                                    "GridPath Warning: solver status " "file not found."
+                                )
+                                termination_condition = "solver status file not found"
+                            else:
+                                ss_fname = os.path.join(
+                                    results_directory, "solver_status.txt"
+                                )
+                                raise FileNotFoundError(f"{ss_fname} not found.")
 
                         # Only import other results if solver status was "ok"
                         # When the problem is infeasible, the solver status is "warning"
@@ -353,6 +384,7 @@ def main(args=None):
     scenario_location = parsed_arguments.scenario_location
     quiet = parsed_arguments.quiet
     import_rule = parsed_arguments.results_import_rule
+    ignore_incomplete = parsed_arguments.ignore_incomplete
 
     conn = connect_to_database(db_path=db_path)
     c = conn.cursor()
@@ -404,6 +436,7 @@ def main(args=None):
         scenario_structure=scenario_structure,
         db=conn,
         scenario_directory=scenario_directory,
+        ignore_incomplete=ignore_incomplete,
         quiet=quiet,
     )
 

gridpath/run_scenario.py

@@ -855,21 +855,21 @@ def save_results(
                 parsed_arguments.results_export_summary_rule
             ]["export_summary"](instance=instance, quiet=parsed_arguments.quiet)
 
-        if not parsed_arguments.quiet:
-            print("...exporting summary CSV results")
-        export_summary_results(
-            scenario_directory=scenario_directory,
-            weather_iteration=weather_iteration,
-            hydro_iteration=hydro_iteration,
-            availability_iteration=availability_iteration,
-            subproblem=subproblem,
-            stage=stage,
-            multi_stage=multi_stage,
-            instance=instance,
-            dynamic_components=dynamic_components,
-            export_summary_results_rule=export_summary_rule,
-            verbose=parsed_arguments.verbose,
-        )
+        if export_summary_rule:
+            if not parsed_arguments.quiet:
+                print("...exporting summary CSV results")
+            export_summary_results(
+                scenario_directory=scenario_directory,
+                weather_iteration=weather_iteration,
+                hydro_iteration=hydro_iteration,
+                availability_iteration=availability_iteration,
+                subproblem=subproblem,
+                stage=stage,
+                multi_stage=multi_stage,
+                instance=instance,
+                dynamic_components=dynamic_components,
+                verbose=parsed_arguments.verbose,
+            )
 
         export_pass_through_inputs(
             scenario_directory=scenario_directory,
@@ -1264,7 +1264,6 @@ def export_summary_results(
     multi_stage,
     instance,
     dynamic_components,
-    export_summary_results_rule,
     verbose,
 ):
     """
@@ -1280,29 +1279,28 @@ def export_summary_results(
 
     Export results for each loaded module (if applicable)
     """
-    if export_summary_results_rule:
-        # Determine/load modules and dynamic components
-        modules_to_use, loaded_modules = set_up_gridpath_modules(
-            scenario_directory=scenario_directory, multi_stage=multi_stage
-        )
+    # Determine/load modules and dynamic components
+    modules_to_use, loaded_modules = set_up_gridpath_modules(
+        scenario_directory=scenario_directory, multi_stage=multi_stage
+    )
 
-        n = 0
-        for m in loaded_modules:
-            if hasattr(m, "export_summary_results"):
-                if verbose:
-                    print(f"... {modules_to_use[n]}")
-                m.export_summary_results(
-                    scenario_directory,
-                    weather_iteration,
-                    hydro_iteration,
-                    availability_iteration,
-                    subproblem,
-                    stage,
-                    instance,
-                    dynamic_components,
-                )
+    n = 0
+    for m in loaded_modules:
+        if hasattr(m, "export_summary_results"):
+            if verbose:
+                print(f"... {modules_to_use[n]}")
+            m.export_summary_results(
+                scenario_directory,
+                weather_iteration,
+                hydro_iteration,
+                availability_iteration,
+                subproblem,
+                stage,
+                instance,
+                dynamic_components,
+            )
 
-            n += 1
+        n += 1
 
 
 def export_pass_through_inputs(