Merge pull request #116 from NOAA-GFDL/89-cesm

New 89 cesm

Author: ceblanton

.github/workflows/conda-pkg-extra-tests.yml

@@ -17,6 +17,7 @@ jobs:
         conda config --append channels conda-forge
         conda config --append channels noaa-gfdl
         conda install conda-build conda-verify
+        conda remove --name base conda-anaconda-telemetry
         conda build .
     - name: Run additional utilities as tests
       run: |

.github/workflows/test-pip-install-run-pytest.yml

@@ -13,7 +13,7 @@ jobs:
     - name: Set up Python 3.10
       uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
+        python-version: '3.11'
 
     - name: Install pip package
       run: |

catalogbuilder/intakebuilder/dat/gfdlcmipfreq.yaml

@@ -1,5 +1,9 @@
 monthly:
     frequency: mon
+h0:
+    frequency: mon
+h2:
+    frequency: day    
 daily:
     frequency: day 
 hourly:

catalogbuilder/intakebuilder/getinfo.py

@@ -130,6 +130,13 @@ def getInfoFromGFDLFilename(filename,dictInfo,configyaml):
           dictInfo["table_id"] = "fx"
     return dictInfo
 
+def getRealm(dictInfo):
+     realm = ""
+     if (dictInfo["source_id"] == "cam"):
+         realm = "atmos"
+         dictInfo["realm"] = realm
+     return(dictInfo)
+
 def getInfoFromGFDLDRS(dirpath,projectdir,dictInfo,configyaml,variable_id):
     '''
     Returns info from project directory and the DRS path to the file
@@ -184,6 +191,10 @@ def getInfoFromGFDLDRS(dirpath,projectdir,dictInfo,configyaml,variable_id):
          logger.debug("This is likely static")
          dictInfo["cell_methods"] = ""
          dictInfo["member_id"] = "" 
+    #CAM ESM: If realm is empty, ensure if there is a helper utility to populate this
+     
+      if("realm" not in dictInfo.keys()):
+         dictInfo = getRealm(dictInfo)
     return dictInfo
 
 def getInfoFromDRS(dirpath,projectdir,dictInfo):
@@ -217,6 +228,7 @@ def getInfoFromVarAtts(fname,variable_id,dictInfo,att="standard_name",filexra=No
     :return: dictInfo with all variable atts 
     '''
     #try:
+     
     filexr,filexra = return_xr(fname)
     if (dictInfo[att] == "na"):
       try:

catalogbuilder/intakebuilder/gfdlcrawler.py

@@ -54,7 +54,6 @@ def crawlLocal(projectdir, dictFilter,dictFilterIgnore,configyaml,slow):
       missingcols = [col for col in diffcols if col not in set_ftemplate]
       missingcols.remove("path") #because we get this anyway
       logger.debug("Missing cols from metadata sources:"+ (str)(missingcols))
-
     #Creating a dictionary to track the unique datasets we come across when using slow mode
     #The keys are the standard names and the values are lists tracking var_id,realm,etc..
     unique_datasets = {'':''}
@@ -138,7 +137,6 @@ def crawlLocal(projectdir, dictFilter,dictFilterIgnore,configyaml,slow):
                         if qualities in unique_datasets.keys():
                             standard_name=unique_datasets[qualities]
                             dictInfo["standard_name"]=standard_name
-
                         else:
                             logger.info("Retrieving standard_name from "+ (str)(filename))
                             getinfo.getInfoFromVarAtts(dictInfo["path"],dictInfo["variable_id"],dictInfo)

catalogbuilder/scripts/gen_intake_gfdl.py

@@ -123,6 +123,7 @@ def create_catalog(input_path, output_path, config, filter_realm, filter_freq, f
     if not slow and 'standard_name' in headers:
         #If we badly need standard name, we use gfdl cmip mapping tables especially when one does not prefer the slow option. Useful for MDTF runs
         df = pd.read_csv(os.path.abspath(csv_path), sep=",", header=0,index_col=False)
+        df['standard_name'] = df['standard_name'].astype(object)
         list_variable_id = []
         try:
             list_variable_id = df["variable_id"].unique().tolist()
@@ -144,7 +145,8 @@ def create_catalog(input_path, output_path, config, filter_realm, filter_freq, f
             except ValueError:
                 continue
             if var is not None and realm is not None:
-                df['standard_name'].loc[(df['variable_id'] == var) & (df['realm'] == realm) ] = v
+                mask = (df['variable_id'] == var) & (df['realm'] == realm)
+                df.loc[mask, 'standard_name'] = v
 
         if df is not None and len(df) != 0:
             with open(csv_path, 'w') as csvfile:

pyproject.toml

@@ -16,7 +16,8 @@ dependencies = [
     "xarray",
     "pandas",
     "jsondiff",
-    "intake-esm",
+    "intake-esm ==2025.12.12",
+    "IPython",
     "boto3",
     "polars-lts-cpu", #Hoping to remove this in the future
 ]