#680: Enhance extract logging (add frequency values) (#765)

* #680: Add frequency value.

* #680: Remove a file committed by mistake.

* Revert "#680: Add frequency value."

This reverts commit 6f7ade9.

* #680: Attempt to correct.

* #680: reapply newlines.

* #680: Fix a broken test.

* #680: Remove empty dicts from output {}.

* #680: Add stash codes to output again.

* #680: Reimplement stash codes in extract logs.

* #680: Add logging for constraint without stash. Revert a change.

* #680: remove a comment.

---------

Co-authored-by: Lauren Boon <[email protected]>

Author: mo-gill

cdds/cdds/extract/common.py

@@ -834,7 +834,7 @@ def configure_mappings(mappings):
                 else:
                     mip_table = " - "
                 msg += " {:<15} {:<15}\n".format(
-                    var["var"], "[ {} ]".format(mip_table))
+                    var["var"], "[ {},@{} ]".format(mip_table, var["frequency"]))
 
     if len(msg) > 0:
         logger.info("\n ----- Embargoed Variable Mappings -----\n{} \n".format(msg))
@@ -1028,13 +1028,20 @@ def condense_constraints(variable_constraints) -> dict:
         A defaultdict where the keys are hashed representations of constraints and the
         values are sets of 'stash' values corresponding to those constraints.
     """
+    logger = logging.getLogger(__name__)
     condensed_constraints = defaultdict(set)
 
     for variable in variable_constraints:
         for constraint in variable["constraint"]:
-            if "stash" in constraint:
-                stash = constraint.pop("stash")
-                hashed_constraint = json.dumps(constraint, sort_keys=True)
-                condensed_constraints[hashed_constraint].add(stash)
+            if "stash" not in constraint:
+                logger.critical("Constraint without stash found: {}".format(constraint))
+                continue
+            stash = constraint["stash"]
+            # Create a copy and remove stash for grouping whilst avoiding mutating the
+            # original (needed to ensure stash codes appear in logs).
+            constraint_without_stash = constraint.copy()
+            constraint_without_stash.pop("stash")
+            hashed_constraint = json.dumps(constraint_without_stash, sort_keys=True)
+            condensed_constraints[hashed_constraint].add(stash)
 
     return condensed_constraints

cdds/cdds/extract/filters.py

@@ -130,10 +130,18 @@ def set_mappings(self, request):
                     else:
                         self.mappings[key].append(var)
                         if var["status"] == "embargoed":
+                            # Look up frequency from original var_list
+                            frequency = None
+                            for original_var in self.var_list["variables"]:
+                                if (original_var["name"] == var["name"] and
+                                    original_var["table"] == var["table"]):
+                                    frequency = original_var["frequency"]
+                                    break
                             self.mappings_embargoed[key].append(
                                 {
                                     "table": var["table"],
-                                    "var": var["name"]
+                                    "var": var["name"],
+                                    "frequency": frequency
                                 }
                             )
 

cdds/cdds/extract/variables.py

@@ -70,7 +70,8 @@ def create_variables_list(self, file_content):
                     {
                         "name": str(var['label']),
                         "table": str(var['miptable']),
-                        "stream": str(var["stream"])
+                        "stream": str(var["stream"]),
+                        "frequency": str(var["frequency"])
                     })
             else:
                 reason_str = "not active "

cdds/cdds/tests/test_extract/test_filters.py

@@ -21,13 +21,13 @@
 class TestFilters(unittest.TestCase):
 
     VAR_LIST = [
-        {"table": "Amon", "name": "tas"},
-        {"table": "Amon", "name": "pr"},
-        {"table": "Amon", "name": "uo"}
+        {"table": "Amon", "name": "tas", "frequency": "mon"},
+        {"table": "Amon", "name": "pr", "frequency": "mon"},
+        {"table": "Amon", "name": "uo", "frequency": "mon"}
     ]
 
     VAR_LIST_OCEAN = [
-        {"table": "Omon", "name": "tos"},
+        {"table": "Omon", "name": "tos", "frequency": "mon"},
     ]
 
     def setUp(self):
@@ -262,7 +262,7 @@ def test_embargo_response(self, mock_mass_filters):
         mock_mass_filters.return_value = model_to_mip_response
         filters.set_mappings(self.request)
 
-        self.assertDictEqual({"ap5": [{"var": "pr", "table": "Amon"}]},
+        self.assertDictEqual({"ap5": [{"var": "pr", "table": "Amon", "frequency": "mon"}]},
                              filters.get_embargoed_mappings())
         # The expected behaviour of filters.set_mappings is to add all
         # entries from the model_to_mip_response which have status "ok" or