fre.cmor: a CMIP7 example, toward general CMIP7 compatibility

.gitmodules

@@ -7,3 +7,7 @@
 [submodule "fre/tests/test_files/cmip6-cmor-tables"]
 	path = fre/tests/test_files/cmip6-cmor-tables
 	url = https://github.com/pcmdi/cmip6-cmor-tables
+[submodule "fre/tests/test_files/cmip7-cmor-tables"]
+	path = fre/tests/test_files/cmip7-cmor-tables
+	url = https://github.com/WCRP-CMIP/cmip7-cmor-tables.git
+	branch = main

fre/cmor/cmor_helpers.py

@@ -43,11 +43,11 @@
 import glob
 import json
 import logging
-import numpy as np
 import os
 from pathlib import Path
-from typing import Optional, Any, List, Union
+from typing import Optional, List, Union
 
+import numpy as np
 from netCDF4 import Dataset, Variable
 
 fre_logger = logging.getLogger(__name__)
@@ -410,7 +410,7 @@ def update_grid_and_label( json_file_path: str,
             fre_logger.info('Updated "nominal_resolution": %s', data["nominal_resolution"])
         except KeyError as e:
             fre_logger.error("Failed to update 'nominal_resolution': %s", e)
-            raise KeyError("Error while updating 'nominal_resolution'. Ensure the field exists and is modifiable.") from e
+            raise KeyError("Error updating 'nominal_resolution'. Ensure the field exists and is modifiable.") from e
 
         output_file_path = output_file_path or json_file_path
 
@@ -545,7 +545,7 @@ def conv_mip_to_bronx_freq(cmor_table_freq: str) -> Optional[str]:
     }
     bronx_freq = cmor_to_bronx_dict.get(cmor_table_freq)
     if bronx_freq is None:
-        fre_logger.warning(f'MIP table frequency = {cmor_table_freq} does not have a FRE-bronx equivalent')
+        fre_logger.warning('MIP table frequency = %s does not have a FRE-bronx equivalent', cmor_table_freq)
     if cmor_table_freq not in cmor_to_bronx_dict.keys():
         raise KeyError(f'MIP table frequency = "{cmor_table_freq}" is not a valid MIP frequency')
     return bronx_freq

fre/cmor/tests/test_cmor_helpers_update_grid_label.py

@@ -1,6 +1,12 @@
+'''
+unit tests for cmor_helpers.update_grid_and_label
+'''
+
 import json
-import pytest
+
 from pathlib import Path
+import pytest
+
 from fre.cmor.cmor_helpers import update_grid_and_label
 
 # Sample data for testing

fre/cmor/tests/test_cmor_mixer_calendar_integration.py

@@ -10,6 +10,7 @@ def mock_json_exp_config(tmp_path):
     Create a mock JSON experiment config file for testing.
     """
     config_data = {
+        "mip_era": "cmip6",
         "calendar": "360_day",
         "grid": "test_grid",
         "grid_label": "gn",

fre/cmor/tests/test_cmor_run_subtool.py

@@ -1,4 +1,6 @@
-''' tests for fre.cmor.cmor_run_subtool '''
+'''
+tests for fre.cmor.cmor_run_subtool
+'''
 import subprocess
 import shutil
 from pathlib import Path
@@ -20,7 +22,9 @@
     f'{CMIP6_TABLE_REPO_PATH}/Tables/CMIP6_Omon.json'
 
 def test_setup_cmor_cmip_table_repo():
-    ''' setup routine, make sure the recursively cloned tables exist '''
+    '''
+    setup routine, make sure the recursively cloned tables exist
+    '''
     assert all( [ Path(CMIP6_TABLE_REPO_PATH).exists(),
                   Path(TABLE_CONFIG).exists()
                   ] )
@@ -53,7 +57,8 @@ def test_setup_cmor_cmip_table_repo():
 
 
 def test_setup_fre_cmor_run_subtool(capfd):
-    ''' The routine generates a netCDF file from an ascii (cdl) file. It also checks for a ncgen
+    '''
+    The routine generates a netCDF file from an ascii (cdl) file. It also checks for a ncgen
     output file from prev pytest runs, removes it if it's present, and ensures the new file is
     created without error.
     '''
@@ -285,19 +290,18 @@ def test_git_cleanup():
     git's record of changed files. It's supposed to change as part of the test.
     '''
     is_ci = os.environ.get("GITHUB_WORKSPACE") is not None
-    if is_ci:
-      #doesn't run happily in CI and not needed
-      assert True
-    else:
-      git_cmd = f"git restore {EXP_CONFIG}"
-      restore = subprocess.run(git_cmd,
-                    shell=True,
-                    check=False)
-      check_cmd = f"git status | grep {EXP_CONFIG}"
-      check = subprocess.run(check_cmd,
-                             shell = True, check = False)
-      #first command completed, second found no file in git status
-      assert all([restore.returncode == 0, check.returncode == 1])
+    if not is_ci:
+        git_cmd = f"git restore {EXP_CONFIG}"
+        restore = subprocess.run(git_cmd,
+                                 shell=True,
+                                 check=False)
+        check_cmd = f"git status | grep {EXP_CONFIG}"
+        check = subprocess.run(check_cmd,
+                               shell = True,
+                               check = False)
+        #first command completed, second found no file in git status
+        assert all([restore.returncode == 0,
+                    check.returncode == 1])
 
 def test_cmor_run_subtool_raise_value_error():
     '''
@@ -343,8 +347,6 @@ def test_fre_cmor_run_subtool_empty_varlist(capfd):
         )
 
 
-@pytest.mark.xfail(reason='TODO req some quick rework of the opt_var_name logic- '
-                          'the current approach doesn\'t cut it')
 def test_fre_cmor_run_subtool_opt_var_name_not_in_table():
     ''' fre cmor run, exception,  '''
 

fre/cmor/tests/test_cmor_run_subtool_further_examples.py

@@ -1,24 +1,20 @@
 '''
-expanded set of tests for fre cmor run
-focus on
+expanded set of tests for fre cmor run focus on cases beyond test_cmor_run_subtool.py
 '''
 
 from datetime import date
 from pathlib import Path
 import shutil
-
 import glob
+#import time
+#import platform
+import subprocess
+import os
 
 import pytest
 
 from fre.cmor import cmor_run_subtool
 
-import time
-
-import platform
-
-import subprocess
-import os
 
 # global consts for these tests, with no/trivial impact on the results
 ROOTDIR='fre/tests/test_files'
@@ -187,15 +183,16 @@ def test_git_cleanup():
     git's record of changed files. It's supposed to change as part of the test.
     '''
     is_ci = os.environ.get("GITHUB_WORKSPACE") is not None
-    if is_ci: #git status/restore doesn't run happily in CI and is not needed
-      assert True
-    else:
-      git_cmd = f"git restore {EXP_CONFIG_DEFAULT}"
-      restore = subprocess.run(git_cmd,
-                    shell=True,
-                    check=False)
-      check_cmd = f"git status | grep {EXP_CONFIG_DEFAULT}"
-      check = subprocess.run(check_cmd,
-                             shell = True, check = False)
-      #first command completed, second found no file in git status
-      assert all([restore.returncode == 0, check.returncode == 1])
+    if not is_ci:
+        git_cmd = f"git restore {EXP_CONFIG_DEFAULT}"
+        restore = subprocess.run(git_cmd,
+                                 shell=True,
+                                 check=False)
+        check_cmd = f"git status | grep {EXP_CONFIG_DEFAULT}"
+        check = subprocess.run(check_cmd,
+                               shell = True,
+                               check = False)
+        #first command completed, second found no file in git status
+        assert all( [ restore.returncode == 0,
+                      check.returncode == 1 ] )
+

fre/tests/test_files/CMOR_CMIP7_input_example.json

@@ -0,0 +1,63 @@
+{
+    "#_TESTING_ONLY": " ***** This is for unit-test functionality of NOAA-GDFL's fre.cmor submodule for CMIP7, they do not reflect values used in actual production *****",
+    "contact ": "MIP participant mipmember@foobar.c.om",
+    "comment": "additional important information not fitting into other fields can be placed here",
+    "#_TRACKING": "***** anything to do with citing this data, accredation, licensing, and references go here *****",
+    "license": "CC-BY-4-0; CMIP7 data produced by CCCma is licensed under a Creative Commons Attribution 4.0 International License (https://creativecommons.org/licenses/by/4.0). Consult [TODO terms of use link] for terms of use governing CMIP7 output, including citation requirements and proper acknowledgment. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.",
+    "references": "Model described by Koder and Tolkien (J. Geophys. Res., 2001, 576-591).  Also see http://www.GICC.su/giccm/doc/index.html.  The ssp245 simulation is described in Dorkey et al. '(Clim. Dyn., 2003, 323-357.)'",
+    "drs_specs": "MIP-DRS7",
+    "archive_id": "WCRP",
+    "license_id": "CC-BY-4-0",
+    "tracking_prefix": "hdl:21.14107",
+    "#_MIP_DETAILS": "***** anything to do with identifying the specific MIP activity this configuration file is for *****",
+    "_cmip7_option": 1,
+    "mip_era": "CMIP7",
+    "parent_mip_era": "CMIP7",
+    "activity_id": "CMIP",
+    "parent_activity_id": "CMIP",
+    "#_SOURCE_SECTION": "***** anything to do with identifying this experiment, it's relationships to other experiments, the producers, and their institution *****",
+    "institution": "",
+    "institution_id": "CCCma",
+    "source": "CanESM6-MR:",
+    "source_id": "CanESM6-MR",
+    "source_type": "AOGCM ISM AER",
+    "experiment_id": "esm-piControl",
+    "sub_experiment": "none",
+    "sub_experiment_id": "none",
+    "parent_source_id": "CanESM6-MR",
+    "parent_experiment_id": "esm-piControl-spinup",
+    "#_INDICIES": "***** changed from ints to strings for CMIP7 *****",
+    "realization_index": "r3",
+    "initialization_index": "i1",
+    "physics_index": "p1",
+    "forcing_index": "f3",
+    "run_variant": "3rd realization",
+    "parent_variant_label": "r3i1p1f3",
+    "#_TEMPORAL_INFO": "***** anything to do with describing temporal aspects of the experiment *****",
+    "parent_time_units": "days since 1850-01-01",
+    "branch_method": "no parent",
+    "branch_time_in_child": 59400.0,
+    "branch_time_in_parent": 0.0,
+    "calendar": "julian",
+    "#_SPATIAL_INFO": "***** anything to do with describing physical aspects of the experiment *****",
+    "grid": "FOO_BAR_PLACEHOLD",
+    "grid_label": "g99",
+    "frequency": "mon",
+    "region": "glb",
+    "nominal_resolution": "10000 km",
+    "#_HISTORY_METADATA": "history attribute string and template, to create history field for output file",
+    "history": "Output from archivcl_A1.nce/giccm_03_std_2xCO2_2256.",
+    "_history_template": "%s ;rewrote data to be consistent with <activity_id> for variable <variable_id> found in table <table_id>.",
+    "#_OUTPUT_PATHS": "***** pathing/templates for output files *****",
+    "#_output_template_NOTE": "***** PCMDI/cmor 4e7f1f3d731077b7f65c188edefac924cc3e2779 Test/test_cmor_CMIP7.py L47 *****",
+    "#_output": "***** Root directory for output (can be either a relative or full path) *****",
+    "outpath": ".",
+    "#_output_path_template": "***** Template for output path directory using tables keys or global attributes, these should follow the relevant data reference syntax *****",
+    "output_path_template": "<activity_id><source_id><experiment_id><member_id><variable_id><branding_suffix><grid_label><version>",
+    "#_output_file_template": "***** Template for output filename using tables keys or global attributes, these should follow the relevant data reference syntax *****",
+    "output_file_template": "<variable_id><branding_suffix><frequency><region><grid_label><source_id><experiment_id><variant_id>[<time_range>].nc",
+    "#_INPUT_CONFIG_PATHS": "***** pathing/templates for input configuation files holding controlled vocabularies *****",
+    "_controlled_vocabulary_file": "../tables-cvs/cmor-cvs.json",
+    "_AXIS_ENTRY_FILE": "CMIP7_coordinate.json",
+    "_FORMULA_VAR_FILE": "CMIP7_formula_terms.json"
+}

fre/tests/test_fre_app_cli.py

@@ -104,7 +104,6 @@ def test_cli_fre_app_gen_time_averages_wrapper_opt_dne(capfd):
     assert result.exit_code == 2
     _out, _err = capfd.readouterr()
 
-@pytest.mark.xfail(reason='under-construction/developing') #TODO flesh out argument details, assertions and checks etc
 def test_cli_fre_app_gen_time_averages_wrapper_case(capfd):
     """
     fre app gen-time-averages-wrapper \