rebasing

Author: cghielmini

.pre-commit-config.yaml

@@ -134,3 +134,24 @@ repos:
           - "--max-line-length=88"
           - "--ignore=E203,W503,F811,I002"
           - "--max-complexity=12"
+  - repo: local
+    hooks:
+      - id: mypy
+        name: mypy
+        description: Type-check Python code, only for functions with type annotations
+        language: system
+        entry: mypy
+        pass_filenames: false
+        args:
+          - "-p"
+          - "engine"
+          - "-p"
+          - "tests"
+          - "-p"
+          - "util"
+          - "-p"
+          - "visualize"
+          - "--ignore-missing-imports"
+          - "--allow-untyped-defs"
+          - "--follow-imports=silent"
+          - "--strict-optional"

README.md

@@ -273,7 +273,7 @@ python ../externals/probtest/probtest.py check --reference-files stats_ref.csv,
 
 This check can be also visualized by:
 ```console
-python ../externals/probtest/probtest.py check-plot --input-file-ref stats_ref.csv --input-file-cur stats_exp_name.csv --tolerance-file-name exp_name_tolerance.csv --factor 5 --savedir ./plot_dir
+python ../externals/probtest/probtest.py check-plot --reference-files stats_ref.csv --current-files stats_exp_name.csv --tolerance-files exp_name_tolerance.csv --factor 5 --savedir ./plot_dir
 ```
 
 Note that the reference `--reference-files` and test stats files

engine/cdo_table.py

@@ -170,14 +170,18 @@ def cdo_table(
                 varnames = [
                     v
                     for v in list(ref_data.keys())
-                    if "time" in ref_data.variables.get(v).dims
+                    if (var := ref_data.variables.get(v)) is not None
+                    and "time" in var.dims
                 ]
 
                 for v in varnames:
-                    diff_data.variables.get(v).values = compute_rel_diff(
-                        ref_data.variables.get(v).values,
-                        perturb_data.variables.get(v).values,
-                    )
+                    if (ref_var := ref_data.variables.get(v)) is not None and (
+                        pert_var := perturb_data.variables.get(v)
+                    ) is not None:
+                        diff_data.variables[v].values = compute_rel_diff(
+                            ref_var.values,
+                            pert_var.values,
+                        )
 
                 diff_data.to_netcdf(f"{tmpdir}/{rf}")
                 ref_data.close()

engine/select_members.py

@@ -15,10 +15,10 @@
 import click
 
 from engine.tolerance import tolerance
-from util.click_util import cli_help
+from util.click_util import CommaSeparatedStrings, cli_help
 from util.dataframe_ops import check_file_with_tolerances
 from util.log_handler import logger
-from util.utils import FileInfo
+from util.utils import FileInfo, validate_single_stats_file
 
 
 def find_members_and_factor_validating_for_all_stats_files(
@@ -198,16 +198,22 @@ def check_selection_by_ids(
     help=cli_help["enable_check_only"],
 )
 @click.option(
-    "--stats-file-name",
-    help=cli_help["stats_file_name"],
+    "--ensemble-files",
+    type=CommaSeparatedStrings(),
+    default=[],
+    help=cli_help["ensemble_files"]
+    + "\nNote: this option accepts exactly one stats file.",
 )
 @click.option(
     "--selected-members-file-name",
     help=cli_help["selected_members_file_name"],
 )
 @click.option(
-    "--tolerance-file-name",
-    help=cli_help["tolerance_file_name"],
+    "--tolerance-files",
+    type=CommaSeparatedStrings(),
+    default=[],
+    help=cli_help["tolerance_files_output"]
+    + "\nNote: this option accepts exactly one stats file.",
 )
 @click.option(
     "--member-type",
@@ -248,9 +254,9 @@ def check_selection_by_ids(
 def select_members(
     experiment_name,
     enable_check_only,
-    stats_file_name,
+    ensemble_files,
     selected_members_file_name,
-    tolerance_file_name,
+    tolerance_files,
     member_type,
     max_member_count,
     total_member_count,
@@ -261,9 +267,24 @@ def select_members(
     """
     Selects members and writes them to a file together with the tolerance factor
     """
+
+    # check for valid input parameters
+    errors = []
+
+    stats_file_name = validate_single_stats_file(ensemble_files, "ensemble", errors)
+    tolerance_file_name = validate_single_stats_file(
+        tolerance_files, "tolerance", errors
+    )
+
     if max_member_count >= total_member_count:
-        logger.error("ERROR: max_member_count must be smaller than total_member_count")
+        errors.append("max_member_count must be smaller than total_member_count")
+
+    if errors:
+        for msg in errors:
+            logger.error("ERROR: %s", msg)
         sys.exit(1)
+
+    # start with selecting members
     if enable_check_only:
         check_selection_by_ids(
             stats_file_name=stats_file_name,

engine/tolerance.py

@@ -76,7 +76,6 @@ def tolerance(
         ensemble_files = expand_members(
             mem, member_ids=member_ids, member_type=member_type
         )
-
         dfs = [
             file_name_parser[info.file_type](info.path)
             for file in ensemble_files