Updates to file chunking checks and consistency output (#45)

- Updated consistency output
  - Added dimensions
  - Added variable attributes data types
  - Added global attributes data types
  - Added non-required global attributes
- Updated file chunking check
  - Suppressing error for last year of a simulation
    if it is the officially expected end of the experiment

Author: sol1105

cc_plugin_cc6/base.py

@@ -319,22 +319,39 @@ def _read_CV(self, path, table_prefix, table_name):
 
     def _write_consistency_output(self):
         """Write output for consistency checks across files."""
-        # Dictionary of global attributes
+        # Dictionaries of global attributes and their data types
         required_attributes = self.CV.get("required_global_attributes", {})
-        file_attrs = {
+        file_attrs_req = {
             k: str(v) for k, v in self.xrds.attrs.items() if k in required_attributes
         }
+        file_attrs_nreq = {
+            k: str(v)
+            for k, v in self.xrds.attrs.items()
+            if k not in required_attributes
+            if k not in ["history"]
+        }
+        file_attrs_dtypes = {
+            k: type(v).__qualname__ for k, v in self.xrds.attrs.items()
+        }
         for k in required_attributes:
-            if k not in file_attrs:
-                file_attrs[k] = "unset"
-        # Dictionary of variable attributes
+            if k not in file_attrs_req:
+                file_attrs_req[k] = "unset"
+            if k not in file_attrs_dtypes:
+                file_attrs_dtypes[k] = "unset"
+        # Dictionaries of variable attributes and their data types
         var_attrs = {}
+        var_attrs_dtypes = {}
         for var in list(self.xrds.data_vars.keys()) + list(self.xrds.coords.keys()):
             var_attrs[var] = {
                 key: str(value)
                 for key, value in self.xrds[var].attrs.items()
                 if key not in ["history"]
             }
+            var_attrs_dtypes[var] = {
+                key: type(value).__qualname__
+                for key, value in self.xrds[var].attrs.items()
+                if key not in ["history"]
+            }
         # Dictionary of time information
         time_info = {}
         if self.time is not None:
@@ -363,12 +380,27 @@ def _write_consistency_output(self):
             coord_checksums[coord_var] = md5(
                 str(self.xrds[coord_var].values.tobytes()).encode("utf-8")
             ).hexdigest()
+        # Dictionary of dimension sizes
+        dims = dict(self.xrds.sizes)
+        # Do not compare time dimension size, only name
+        if self.time is not None:
+            dimt = self.time.dims[0]
+            dims[dimt] = "n"
+        # Dictionary of variable data types
+        var_dtypes = {}
+        for var in list(self.xrds.data_vars.keys()) + list(self.xrds.coords.keys()):
+            var_dtypes[var] = str(self.xrds[var].dtype)
         # Write combined dictionary
         with open(self.consistency_output, "w") as f:
             json.dump(
                 {
-                    "global_attributes": file_attrs,
+                    "global_attributes": file_attrs_req,
+                    "global_attributes_non_required": file_attrs_nreq,
+                    "global_attributes_dtypes": file_attrs_dtypes,
                     "variable_attributes": var_attrs,
+                    "variable_attributes_dtypes": var_attrs_dtypes,
+                    "variable_dtypes": var_dtypes,
+                    "dimensions": dims,
                     "coordinates": coord_checksums,
                     "time_info": time_info,
                 },

cc_plugin_cc6/cc6.py

@@ -216,6 +216,19 @@ def check_time_chunking(self, ds):
             messages.append(f"Frequency '{self.frequency}' not supported.")
             return self.make_result(level, score, out_of, desc, messages)
 
+        # Expected last simulation year
+        # -> suppress error for last year of a simulation
+        #    if it is the "official" last year
+        expected_last_sim_year = {
+            "evaluation": 2024,
+            "historical": 2014,
+            "ssp119": 2100,
+            "ssp126": 2100,
+            "ssp245": 2100,
+            "ssp370": 2100,
+            "ssp585": 2100,
+        }
+
         # Get the time dimension, calendar and units
         if self.time is None:
             messages.append("Coordinate variable 'time' not found in file.")
@@ -255,7 +268,6 @@ def check_time_chunking(self, ds):
                 year, 1, 1, 0, 0, 0, calendar=self.calendar
             )
         else:
-
             year = first_time.year + 1
             while str(year)[-1] != "1":
                 year += 1
@@ -298,6 +310,15 @@ def check_time_chunking(self, ds):
         else:
             messages.append(f"Cannot interpret cell_methods '{self.cell_methods}'.")
 
+        # Consider experiment end
+        exp_last_year = expected_last_sim_year.get(
+            self._get_attr("driving_experiment_id", None), None
+        )
+        if exp_last_year:
+            expected_end_date_exp = expected_end_date.replace(year=exp_last_year)
+        else:
+            expected_end_date_exp = expected_end_date
+
         if len(messages) == 0:
             errmsg = (
                 f"{'Apart from the first and last files of a timeseries ' if nyears>1 else ''}'{nyears}' "
@@ -311,7 +332,7 @@ def check_time_chunking(self, ds):
                     + errmsg
                 )
             # Check if the last time is equal to the expected end date
-            if last_time != expected_end_date:
+            if last_time != expected_end_date and last_time != expected_end_date_exp:
                 messages.append(
                     f"The last timestep differs from expectation ('{expected_end_date}'): '{last_time}'. "
                     + errmsg