Fix parsl config deprecation issues (#78)

* Fix parsl config deprecation issues

* Add test coverage for all resource configurations

* Fix Python 3.9 compatibility (use Optional instead of |)

* Remove pinned python version from pre-commit config

* Fix linting in test_basic.py and duplicate key search config

* Remove unneeded commented lines in parsl_configurator.py

* Refactor tests: Rename to test_resource_config.py and use unittest

* Simplify test docstrings and comments

* Remove deprecated mom_lims from search config yaml

Author: wilsonbb

.pre-commit-config.yaml

@@ -54,7 +54,6 @@ repos:
         # supported by your project here, or alternatively use
         # pre-commit's default_language_version, see
         # https://pre-commit.com/#top_level-default_language_version
-        language_version: python3.12
     # Make sure Sphinx can build the documentation while explicitly omitting 
     # notebooks from the docs, so users don't have to wait through the execution 
     # of each notebook or each commit. By default, these will be checked in the 

scripts/generic_search_config.yaml

@@ -4,7 +4,6 @@ clip_negative: false
 cluster_eps: 120.0
 cluster_type: nn_start_end
 cluster_v_scale: 1.0
-coadds: []
 debug: true
 do_clustering: true
 do_mask: true
@@ -14,44 +13,38 @@ gpu_filter: true
 generator_config:
   angle_units: degree
   angles:
-  - 90
-  - -90
-  - 64
+    - 90
+    - -90
+    - 64
   given_ecliptic: null
   name: EclipticCenteredSearch
   velocities:
-  - 25.0
-  - 225.0
-  - 64
+    - 25.0
+    - 225.0
+    - 64
   velocity_units: pix / d
 im_filepath: null
 legacy_filename: null
 lh_level: 5.0
 max_lh: 10000.0
-mom_lims:
-- 35.5
-- 35.5
-- 2.0
-- 0.3
-- 0.3
 num_obs: 7
 peak_offset:
-- 2.0
-- 2.0
+  - 2.0
+  - 2.0
 psf_val: 1.4
 res_filepath: /sdf/home/c/colinc/rubin-user/20X20_patch250060
 result_filename: /sdf/home/c/colinc/rubin-user/20X20_patch250060/full_results.ecsv
 results_per_pixel: 4
 save_all_stamps: False
 sigmaG_lims:
-- 25
-- 75
+  - 25
+  - 75
 stamp_radius: 50
 coadds:
-- sum
-- mean
-- median
-- weighted
+  - sum
+  - mean
+  - median
+  - weighted
 stamp_type: sum
 track_filtered: false
 x_pixel_bounds: null

scripts/parsl_configurator.py

@@ -8,13 +8,15 @@
 
 script_directory = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
 
+
 def get_reflex_distance(basedir, mode="collection"):
     """Determine reflex distance, either by filename or by .collection content. 5/5/2025 COC"""
     reflex_distance = None
     first_ic_path = glob.glob(f"{basedir}/*.collection")[0]
     if mode == "collection":
         print(f"Determining reflex distances via first row of first collection in {basedir}...")
         import kbmod
+
         ic = kbmod.ImageCollection.read(first_ic_path)
         reflex_distance = ic["helio_guess_dist"][0]
     else:
@@ -25,10 +27,18 @@ def get_reflex_distance(basedir, mode="collection"):
     return reflex_distance
 
 
-def make_toml_file(basedir, generic_toml_path, site_name="Rubin", disable_cleanup=False, nworkers=32, repo_path="/repo/main", reflex_distances=None):
+def make_toml_file(
+    basedir,
+    generic_toml_path,
+    site_name="Rubin",
+    disable_cleanup=False,
+    nworkers=32,
+    repo_path="/repo/main",
+    reflex_distances=None,
+):
     #
     if reflex_distances == None:
-        print(f'WARNING: default reflex distances hardcoded to [39.0]')
+        print(f"WARNING: default reflex distances hardcoded to [39.0]")
         reflex_distances = [39.0]
     lookup_dict = {}
     lookup_dict["____basedir____"] = basedir
@@ -46,7 +56,7 @@ def make_toml_file(basedir, generic_toml_path, site_name="Rubin", disable_cleanu
         with open(generic_toml_path, "r") as g:
             for line in g:
                 line = line.strip()
-                if line.startswith("#") or len(line.strip()) == 0: # comment or blank line
+                if line.startswith("#") or len(line.strip()) == 0:  # comment or blank line
                     print(line, file=f)
                     continue
                 found_key = False
@@ -57,7 +67,7 @@ def make_toml_file(basedir, generic_toml_path, site_name="Rubin", disable_cleanu
                         break
                 if found_key == True:
                     continue
-                print(line, file=f) # catch-all
+                print(line, file=f)  # catch-all
     print(f"Finished writing {outfile} to disk.")
 
 
@@ -81,9 +91,6 @@ def make_sbatch_file(generic_sbatch_file, basedir, comment=None):
         with open(generic_sbatch_file, "r") as g:
             for line in g:
                 line = line.rstrip("\n")
-#               if line.startswith("#") or len(line) == 0:
-#                   print(line, file=f)
-#                   continue
                 found_key = False
                 for key in lookup_dict:
                     if key in line:
@@ -93,32 +100,91 @@ def make_sbatch_file(generic_sbatch_file, basedir, comment=None):
                 if found_key == True:
                     continue
                 print(line, file=f)
-    print(f'Wrote {outfile} to disk.')
+    print(f"Wrote {outfile} to disk.")
     return outfile
 
+
 def make_yaml_file(generic_yaml_file, basedir):
     yaml_outfile = f"{basedir}/search_config.yaml"
-    with open(generic_yaml_file, 'r') as g:
-        with open(yaml_outfile, 'w') as f:
+    with open(generic_yaml_file, "r") as g:
+        with open(yaml_outfile, "w") as f:
             for line in g:
                 line = line.rstrip("\n")
                 print(line, file=f)
-    print(f'Wrote {yaml_outfile} to disk.')
+    print(f"Wrote {yaml_outfile} to disk.")
     return yaml_outfile
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     import argparse
-    parser = argparse.ArgumentParser(description='Tool to generate .toml files for Parsl Workflow.') # 4/21/2025 COC
-    parser.add_argument('--basedir', dest='basedir', help='path to staging directory containing ImageCollections. Default: current working directory.', type=str, default=os.getcwd())
-    parser.add_argument('--reflex-distances', dest='reflex_distances', help='reflex-correction distances; default is None, which results in the directory being crawled and the first reflex-distance found in an ImageCollection is used. There should only be one distance, but multiple are supported.', type=float, default=None, nargs='+')
-    parser.add_argument('--site-name', dest='site_name', help='Observatory site name. Default: "Rubin"', type=str, default="Rubin")
-    parser.add_argument('--disable-cleanup', dest='disable_cleanup', help='disable deletion of reprojected WorkUnits after successful run. Default: False', type=bool, default=False)
-    parser.add_argument('--nworkers', dest='nworkers', help='number of workers (cores) to assume. Default: 32', type=int, default=32)
-    parser.add_argument('--repo-path', dest='repo_path', help='Butler repository path. Default: /repo/main', type=str, default="/repo/main")
-    parser.add_argument('--generic-toml-path', dest='generic_toml_path', help='Path to a special generic TOML file. Default: ~/generic_runtime_config.toml', type=str, default=f"{script_directory}/generic_runtime_config.toml")
-    parser.add_argument('--generic-sbatch-path', dest='generic_sbatch_path', help='Path to a special generic sbatch file. Default: ~/generic_parent_parsl_sbatch.sh', type=str, default=f"{script_directory}/generic_parent_parsl_sbatch.sh")
-    parser.add_argument('--generic-yaml-path', dest='generic_yaml_path', help='Path to a special generic kbmod search config yaml file. Default: ~/generic_search_config.yaml', type=str, default=f"{script_directory}/generic_search_config.yaml")
+
+    parser = argparse.ArgumentParser(
+        description="Tool to generate .toml files for Parsl Workflow."
+    )  # 4/21/2025 COC
+    parser.add_argument(
+        "--basedir",
+        dest="basedir",
+        help="path to staging directory containing ImageCollections. Default: current working directory.",
+        type=str,
+        default=os.getcwd(),
+    )
+    parser.add_argument(
+        "--reflex-distances",
+        dest="reflex_distances",
+        help="reflex-correction distances; default is None, which results in the directory being crawled and the first reflex-distance found in an ImageCollection is used. There should only be one distance, but multiple are supported.",
+        type=float,
+        default=None,
+        nargs="+",
+    )
+    parser.add_argument(
+        "--site-name",
+        dest="site_name",
+        help='Observatory site name. Default: "Rubin"',
+        type=str,
+        default="Rubin",
+    )
+    parser.add_argument(
+        "--disable-cleanup",
+        dest="disable_cleanup",
+        help="disable deletion of reprojected WorkUnits after successful run. Default: False",
+        type=bool,
+        default=False,
+    )
+    parser.add_argument(
+        "--nworkers",
+        dest="nworkers",
+        help="number of workers (cores) to assume. Default: 32",
+        type=int,
+        default=32,
+    )
+    parser.add_argument(
+        "--repo-path",
+        dest="repo_path",
+        help="Butler repository path. Default: /repo/main",
+        type=str,
+        default="/repo/main",
+    )
+    parser.add_argument(
+        "--generic-toml-path",
+        dest="generic_toml_path",
+        help="Path to a special generic TOML file. Default: ~/generic_runtime_config.toml",
+        type=str,
+        default=f"{script_directory}/generic_runtime_config.toml",
+    )
+    parser.add_argument(
+        "--generic-sbatch-path",
+        dest="generic_sbatch_path",
+        help="Path to a special generic sbatch file. Default: ~/generic_parent_parsl_sbatch.sh",
+        type=str,
+        default=f"{script_directory}/generic_parent_parsl_sbatch.sh",
+    )
+    parser.add_argument(
+        "--generic-yaml-path",
+        dest="generic_yaml_path",
+        help="Path to a special generic kbmod search config yaml file. Default: ~/generic_search_config.yaml",
+        type=str,
+        default=f"{script_directory}/generic_search_config.yaml",
+    )
     #
     args = parser.parse_args()
     #
@@ -127,14 +193,15 @@ def make_yaml_file(generic_yaml_file, basedir):
     reflex_distances = args.reflex_distances
     if reflex_distances == None or reflex_distances == []:
         reflex_distances = [get_reflex_distance(basedir=args.basedir)]
-    make_toml_file(basedir=args.basedir, 
-                generic_toml_path=args.generic_toml_path, 
-                site_name=args.site_name, 
-                disable_cleanup=args.disable_cleanup, 
-                nworkers=args.nworkers, 
-                repo_path=args.repo_path, 
-                reflex_distances=reflex_distances
-            )
+    make_toml_file(
+        basedir=args.basedir,
+        generic_toml_path=args.generic_toml_path,
+        site_name=args.site_name,
+        disable_cleanup=args.disable_cleanup,
+        nworkers=args.nworkers,
+        repo_path=args.repo_path,
+        reflex_distances=reflex_distances,
+    )
     make_sbatch_file(generic_sbatch_file=args.generic_sbatch_path, basedir=args.basedir)
     make_yaml_file(generic_yaml_file=args.generic_yaml_path, basedir=args.basedir)
     #

src/kbmod_wf/resource_configs/dev_configuration.py

@@ -13,11 +13,6 @@ def dev_resource_config():
     return Config(
         # put the log files in in the top level folder, "run_logs".
         run_dir=os.path.join(project_dir, "run_logs", datetime.date.today().isoformat()),
-        app_cache=True,
-        checkpoint_mode="task_exit",
-        checkpoint_files=get_all_checkpoints(
-            os.path.join(project_dir, "run_logs", datetime.date.today().isoformat())
-        ),
         executors=[
             ThreadPoolExecutor(
                 label="local_dev_testing",

src/kbmod_wf/resource_configs/klone_configuration.py

@@ -15,11 +15,6 @@
 
 def klone_resource_config():
     return Config(
-        app_cache=True,
-        checkpoint_mode="task_exit",
-        checkpoint_files=get_all_checkpoints(
-            os.path.join("/gscratch/dirac/kbmod/workflow/run_logs", datetime.date.today().isoformat())
-        ),
         run_dir=os.path.join("/gscratch/dirac/kbmod/workflow/run_logs", datetime.date.today().isoformat()),
         retries=1,
         executors=[