Merge branch 'master' into c1f_and_common_mode

Author: susannaaz

sotodlib/preprocess/pcore.py

@@ -553,7 +553,7 @@ def gen_metric(
         proc_aman,
         flags_key=None,
         percentiles=[0, 50, 75, 90, 95, 100],
-        tags=[],
+        tags={},
     ):
         """ Generate a QA metric from the output of this process.
 
@@ -569,9 +569,11 @@ def gen_metric(
             process name.
         percentiles : list
             Percentiles to compute across detectors
-        tags : list
-            Keys into `metadata.det_info` to record as tags with the Influx line.
-            Added to the default list ["wafer_slot", "tel_tube", "wafer.bandpass"].
+        tags : dict
+            The values are keys into `metadata.det_info` to record as tags with
+            the Influx line. The keys are addded to the default list
+            ["wafer_slot", "tel_tube", "bandpass"] with "bandpass" being taken
+            from "wafer.bandpass" or "det_cal.bandpass" if the former isn't found.
 
         Returns
         -------
@@ -592,17 +594,28 @@ def gen_metric(
                 raise ValueError(f"Could not parse flags_key {flags_key}")
 
         # add specified tags
-        tag_keys = ["wafer_slot", "tel_tube", "wafer.bandpass"]
-        tag_keys += [t for t in tags if t not in tag_keys]
+        from ..qa.metrics import _get_tag, _has_tag
+        tag_keys = {
+            "wafer_slot": "wafer_slot",
+            "tel_tube": "tel_tube",
+        }
+
+        if _has_tag(meta.det_info, 'wafer.bandpass'):
+            bandpasses = meta.det_info.wafer.bandpass
+            tag_keys["bandpass"] = "wafer.bandpass"
+        else:
+            bandpasses = meta.det_info.det_cal.bandpass
+            tag_keys["bandpass"] = "det_cal.bandpass"
+
+        tag_keys.update(tags)
 
         tags = []
         vals = []
-        from ..qa.metrics import _get_tag, _has_tag
         # record one metric per wafer slot, per bandpass
-        for bp in np.unique(meta.det_info.wafer.bandpass):
+        for bp in np.unique(bandpasses):
             for ws in np.unique(meta.det_info.wafer_slot):
                 subset = np.where(
-                    (meta.det_info.wafer_slot == ws) & (meta.det_info.wafer.bandpass == bp)
+                    (meta.det_info.wafer_slot == ws) & (bandpasses == bp)
                 )[0]
 
                 # Compute the number of samples that were flagged
@@ -623,7 +636,7 @@ def gen_metric(
 
                     # get the tags for this wafer (all detectors in this subset share these)
                     tags_base = {
-                        k: _get_tag(meta.det_info, k, subset[0]) for k in tag_keys if _has_tag(meta.det_info, k)
+                        k: _get_tag(meta.det_info, i, subset[0]) for k, i in tag_keys.items() if _has_tag(meta.det_info, i)
                     }
                     tags_base["telescope"] = meta.obs_info.telescope
 

sotodlib/preprocess/processes.py

@@ -832,16 +832,27 @@ def gen_metric(cls, meta, proc_aman):
             `site_pipeline.monitor.Monitor.record`
         """
         # record one metric per wafer_slot per bandpass
-        # extract these tags for the metric
-        tag_keys = ["wafer_slot", "tel_tube", "wafer.bandpass"]
+        # add specified tags
+        from ..qa.metrics import _get_tag, _has_tag
+        tag_keys = {
+            "wafer_slot": "wafer_slot",
+            "tel_tube": "tel_tube",
+        }
+
+        if _has_tag(meta.det_info, 'wafer.bandpass'):
+            bandpasses = meta.det_info.wafer.bandpass
+            tag_keys["bandpass"] = "wafer.bandpass"
+        else:
+            bandpasses = meta.det_info.det_cal.bandpass
+            tag_keys["bandpass"] = "det_cal.bandpass"
+
         tags = []
         vals = []
-        from ..qa.metrics import _get_tag, _has_tag
         import re
-        for bp in np.unique(meta.det_info.wafer.bandpass):
+        for bp in np.unique(bandpasses):
             for ws in np.unique(meta.det_info.wafer_slot):
                 subset = np.where(
-                    (meta.det_info.wafer_slot == ws) & (meta.det_info.wafer.bandpass == bp)
+                    (meta.det_info.wafer_slot == ws) & (bandpasses == bp)
                 )[0]
 
                 if len(subset) > 0:
@@ -867,7 +878,7 @@ def gen_metric(cls, meta, proc_aman):
                     mean = coeff_amp[nonzero].mean(axis=0)
 
                     tags_base = {
-                        k: _get_tag(meta.det_info, k, subset[0]) for k in tag_keys if _has_tag(meta.det_info, k)
+                        k: _get_tag(meta.det_info, i, subset[0]) for k, i in tag_keys.items() if _has_tag(meta.det_info, i)
                     }
                     tags_base["telescope"] = meta.obs_info.telescope
 

sotodlib/qa/metrics.py

@@ -169,9 +169,11 @@ class PreprocessValidDets(PreprocessQA):
     The config entry supports a `process_args` block where the following options can be
     specified:
 
-    tags : list
-        Keys into `metadata.det_info` to record as tags with the Influx line.
-        Added to the default list ["wafer_slot", "tel_tube", "wafer.bandpass"].
+    tags : dict
+        The values are keys into `metadata.det_info` to record as tags with
+        the Influx line. The keys are addded to the default list
+        ["wafer_slot", "tel_tube", "bandpass"] with "bandpass" being taken
+        from "wafer.bandpass" or "det_cal.bandpass" if the former isn't found.
     thresh : float
         The threshold for the fraction of valid samples above which a detector is
         deemed good (default 0.75)
@@ -192,24 +194,35 @@ def __init__(
         # bypass the PreprocessQA __init__
         super(PreprocessQA, self).__init__(*args, **kwargs)
         # extract parameters
-        self._tags = process_args.get("tags", [])
+        self._tags = process_args.get("tags", {})
         self._thresh = process_args.get("thresh", 0.75)
         self._key = process_args.get("process_name", "glitches")
 
     def _process(self, meta):
 
         # add specified tags
-        tag_keys = ["wafer_slot", "tel_tube", "wafer.bandpass"]
-        tag_keys += [t for t in self._tags if t not in tag_keys]
+        tag_keys = {
+            "wafer_slot": "wafer_slot",
+            "tel_tube": "tel_tube",
+        }
+
+        if _has_tag(meta.det_info, 'wafer.bandpass'):
+            bandpasses = meta.det_info.wafer.bandpass
+            tag_keys["bandpass"] = "wafer.bandpass"
+        else:
+            bandpasses = meta.det_info.det_cal.bandpass
+            tag_keys["bandpass"] = "det_cal.bandpass"
+
+        tag_keys.update(self._tags)
 
         # record one metric per wafer slot, per bandpass
         # extract these tags for the metric
         tags = []
         vals = []
-        for bp in np.unique(meta.det_info.wafer.bandpass):
+        for bp in np.unique(bandpasses):
             for ws in np.unique(meta.det_info.wafer_slot):
                 subset = np.where(
-                    (meta.det_info.wafer_slot == ws) & (meta.det_info.wafer.bandpass == bp)
+                    (meta.det_info.wafer_slot == ws) & (bandpasses == bp)
                 )[0]
 
                 if len(subset) > 0:
@@ -224,7 +237,7 @@ def _process(self, meta):
 
                     # get the tags for this wafer (all detectors in this subset share these)
                     tags_i = {
-                        k: _get_tag(meta.det_info, k, subset[0]) for k in tag_keys if _has_tag(meta.det_info, k)
+                        k: _get_tag(meta.det_info, i, subset[0]) for k, i in tag_keys.items() if _has_tag(meta.det_info, i)
                     }
                     tags_i["telescope"] = meta.obs_info.telescope
 
@@ -248,9 +261,11 @@ class PreprocessArrayNET(PreprocessQA):
     The config entry supports a `process_args` block where the following
     options can be specified:
 
-    tags : list
-        Keys into `metadata.det_info` to record as tags with the Influx line.
-        Added to the default list ["wafer_slot", "tel_tube", "wafer.bandpass"].
+    tags : dict
+        The values are keys into `metadata.det_info` to record as tags with
+        the Influx line. The keys are addded to the default list
+        ["wafer_slot", "tel_tube", "bandpass"] with "bandpass" being taken
+        from "wafer.bandpass" or "det_cal.bandpass" if the former isn't found.
     noise_aman : str
         The name of the axis manager that holds the white noise array.
         (default 'noise')
@@ -269,22 +284,34 @@ def __init__(
         # bypass the PreprocessQA __init__
         super(PreprocessQA, self).__init__(*args, **kwargs)
         # extract parameters
-        self._tags = process_args.get("tags", [])
+        self._tags = process_args.get("tags", {})
         self._noise_aman = process_args.get("noise_aman", "noise")
-        self._unit_factor = process_args.get("unit_factor", 1e6)
+        self._field_name = process_args.get("field_name", "")
+        self._unit_factor = process_args.get("unit_factor", 1)
 
     def _process(self, meta):
 
         # record one metric per wafer_slot per bandpass
         # extract these tags for the metric
-        tag_keys = ["wafer_slot", "tel_tube", "wafer.bandpass"]
-        tag_keys += [t for t in self._tags if t not in tag_keys]
+        tag_keys = {
+            "wafer_slot": "wafer_slot",
+            "tel_tube": "tel_tube",
+        }
+
+        if _has_tag(meta.det_info, 'wafer.bandpass'):
+            bandpasses = meta.det_info.wafer.bandpass
+            tag_keys["bandpass"] = "wafer.bandpass"
+        else:
+            bandpasses = meta.det_info.det_cal.bandpass
+            tag_keys["bandpass"] = "det_cal.bandpass"
+
+        tag_keys.update(self._tags)
         tags = []
         vals = []
-        for bp in np.unique(meta.det_info.wafer.bandpass):
+        for bp in np.unique(bandpasses):
             for ws in np.unique(meta.det_info.wafer_slot):
                 subset = np.where(
-                    (meta.det_info.wafer_slot == ws) & (meta.det_info.wafer.bandpass == bp)
+                    (meta.det_info.wafer_slot == ws) & (bandpasses == bp)
                 )[0]
 
                 white_noise = meta.preprocess[self._noise_aman].white_noise[subset] * self._unit_factor
@@ -294,14 +321,14 @@ def _process(self, meta):
                     vals.append(np.sqrt(1.0 / np.nansum(1.0 / (white_noise[good_indices])**2)))
 
                     tags_base = {
-                        k: _get_tag(meta.det_info, k, subset[0]) for k in tag_keys if _has_tag(meta.det_info, k)
+                        k: _get_tag(meta.det_info, i, subset[0]) for k, i in tag_keys.items() if _has_tag(meta.det_info, i)
                     }
                     tags_base["telescope"] = meta.obs_info.telescope
                     tags.append(tags_base)
 
         obs_time = [meta.obs_info.timestamp] * len(tags)
         return {
-            "field": self._influx_field,
+            "field": f"{self._influx_field}{'_' + self._field_name if self._field_name else ''}",
             "values": vals,
             "timestamps": obs_time,
             "tags": tags,
@@ -315,9 +342,11 @@ class PreprocessDetNET(PreprocessQA):
     The config entry supports a `process_args` block where the following
     options can be specified:
 
-    tags : list
-        Keys into `metadata.det_info` to record as tags with the Influx line.
-        Added to the default list ["wafer_slot", "tel_tube", "wafer.bandpass"].
+    tags : dict
+        The values are keys into `metadata.det_info` to record as tags with
+        the Influx line. The keys are addded to the default list
+        ["wafer_slot", "tel_tube", "bandpass"] with "bandpass" being taken
+        from "wafer.bandpass" or "det_cal.bandpass" if the former isn't found.
     noise_aman : str
         The name of the axis manager that holds the white noise array.
         (default 'noise')
@@ -336,22 +365,34 @@ def __init__(
         # bypass the PreprocessQA __init__
         super(PreprocessQA, self).__init__(*args, **kwargs)
         # extract parameters
-        self._tags = process_args.get("tags", [])
+        self._tags = process_args.get("tags", {})
         self._noise_aman = process_args.get("noise_aman", "noise")
-        self._unit_factor = process_args.get("unit_factor", 1e6)
+        self._field_name = process_args.get("field_name", "")
+        self._unit_factor = process_args.get("unit_factor", 1)
 
     def _process(self, meta):
 
         # record one metric per wafer_slot per bandpass
         # extract these tags for the metric
-        tag_keys = ["wafer_slot", "tel_tube", "wafer.bandpass"]
-        tag_keys += [t for t in self._tags if t not in tag_keys]
+        tag_keys = {
+            "wafer_slot": "wafer_slot",
+            "tel_tube": "tel_tube",
+        }
+
+        if _has_tag(meta.det_info, 'wafer.bandpass'):
+            bandpasses = meta.det_info.wafer.bandpass
+            tag_keys["bandpass"] = "wafer.bandpass"
+        else:
+            bandpasses = meta.det_info.det_cal.bandpass
+            tag_keys["bandpass"] = "det_cal.bandpass"
+
+        tag_keys.update(self._tags)
         tags = []
         vals = []
-        for bp in np.unique(meta.det_info.wafer.bandpass):
+        for bp in np.unique(bandpasses):
             for ws in np.unique(meta.det_info.wafer_slot):
                 subset = np.where(
-                    (meta.det_info.wafer_slot == ws) & (meta.det_info.wafer.bandpass == bp)
+                    (meta.det_info.wafer_slot == ws) & (bandpasses == bp)
                 )[0]
 
                 white_noise = meta.preprocess[self._noise_aman].white_noise[subset] * self._unit_factor
@@ -361,14 +402,14 @@ def _process(self, meta):
                     vals.append(np.sqrt(1.0 / np.nansum(1.0 / (white_noise[good_indices])**2)) * np.sqrt(len(good_indices)))
 
                     tags_base = {
-                        k: _get_tag(meta.det_info, k, subset[0]) for k in tag_keys if _has_tag(meta.det_info, k)
+                        k: _get_tag(meta.det_info, i, subset[0]) for k, i in tag_keys.items() if _has_tag(meta.det_info, i)
                     }
                     tags_base["telescope"] = meta.obs_info.telescope
                     tags.append(tags_base)
 
         obs_time = [meta.obs_info.timestamp] * len(tags)
         return {
-            "field": self._influx_field,
+            "field": f"{self._influx_field}{'_' + self._field_name if self._field_name else ''}",
             "values": vals,
             "timestamps": obs_time,
             "tags": tags,