Merge pull request #340 from dmgav/srx-metadata

Additional metadata for SRX

Author: dmgav

pyxrf/core/tests/test_map_processing.py

@@ -47,9 +47,10 @@ def test_dask_client_create(tmpdir):
 
     # Set the number of workers to some strange number (11 is unusual)
     #   (pass another kwarg in addition to default)
-    client = dask_client_create(n_workers=11)
+    n_workers_requested = 3
+    client = dask_client_create(n_workers=n_workers_requested)
     n_workers = len(client.scheduler_info()["workers"])
-    assert n_workers == 11, "The number of workers was set incorrectly"
+    assert n_workers == n_workers_requested, "The number of workers was set incorrectly"
     client.close()
 
     assert not os.path.exists(dask_worker_space_path), "Temporary directory was created in the current directory"

pyxrf/model/fileio.py

@@ -165,7 +165,6 @@ def _get_pyxrf_version_str(self):
         """
 
         # Determine the current version of PyXRF
-        global pyxrf_version
         pyxrf_version_str = pyxrf_version
         if pyxrf_version_str[0].lower() != "v":
             pyxrf_version_str = f"v{pyxrf_version_str}"
@@ -1511,7 +1510,12 @@ def read_hdf_APS(
                 # Convert ndarrays to lists (they were lists before they were saved)
                 if isinstance(value, np.ndarray):
                     value = list(value)
-                mdata[key] = value
+                if "|" in key:
+                    k1, k2 = key.split("|", 1)
+                    mdata.setdefault(k1, {})
+                    mdata[k1][k2] = value
+                else:
+                    mdata[key] = value
 
         data = f["xrfmap"]
         fname = file_name.split(".")[0]

pyxrf/model/load_data_from_db.py

@@ -669,6 +669,27 @@ def _get_metadata_value_from_descriptor_document(hdr, *, data_key, stream_name="
     return value
 
 
+def _get_metadata_value_from_descriptor_document_by_pattern(hdr, *, pattern, stream_name="baseline"):
+    """
+    Returns all variables in the the first occurrence of each variable with the name containing
+    the patter in specified document stream. Returns empty dictionary if the variable is not found.
+    """
+    value_dict = {}
+    docs = hdr.documents(stream_name=stream_name)
+    for name, doc in docs:
+        if (name != "event") or ("descriptor" not in doc):
+            continue
+        try:
+            values = {k: v for k, v in doc["data"].items if re.search(pattern, k)}
+            for k, v in values.items():
+                if k not in value_dict:
+                    value_dict[k] = v
+        except Exception:
+            pass
+
+    return value_dict
+
+
 def _get_metadata_all_from_descriptor_document(hdr, *, data_key, stream_name="baseline"):
     """
     Returns the list of the recorded values of variables with the name ``data_key`` in
@@ -703,6 +724,21 @@ def _get_metadata_value_from_descriptor_document_tiled(hdr, *, data_key, stream_
     return value
 
 
+def _get_metadata_value_from_descriptor_document_by_pattern_tiled(hdr, *, pattern, stream_name="baseline"):
+    """
+    Returns all variables in the the first occurrence of each variable with the name containing
+    the patter in specified document stream. Returns empty dictionary if the variable is not found.
+    """
+    value_dict = {}
+    docs = hdr[stream_name]["data"]
+
+    for k, v in docs.items():
+        if re.search(pattern, k):
+            value_dict[k] = v.compute()[0]
+
+    return value_dict
+
+
 def _get_metadata_all_from_descriptor_document_tiled(hdr, *, data_key, stream_name="baseline"):
     """
     Returns the list of the recorded values of variables with the name ``data_key`` in
@@ -2280,6 +2316,24 @@ def map_data2D_srx_new_tiled(
     if v is not None:
         mdata["instrument_beam_current"] = v
 
+    v = _get_metadata_all_from_descriptor_document_tiled(
+        hdr, data_key="nano_det_sample2detector", stream_name="baseline"
+    )
+    if v is not None:
+        mdata["instrument_sample_to_detector"] = v
+
+    v = _get_metadata_value_from_descriptor_document_by_pattern_tiled(
+        hdr, pattern="attenuators", stream_name="baseline"
+    )
+    if v:
+        mdata["instrument_attenuators_config"] = v
+
+    v = _get_metadata_value_from_descriptor_document_by_pattern_tiled(
+        hdr, pattern="preamp", stream_name="baseline"
+    )
+    if v:
+        mdata["instrument_preamps_config"] = v
+
     for ax in ["X", "Y", "Z"]:
         v = _get_metadata_all_from_descriptor_document_tiled(
             hdr, data_key=f"nanoKB_interferometer_pos{ax}", stream_name="baseline"
@@ -3982,6 +4036,27 @@ def incorrect_type_msg(channel, data_type):
         if "file_software" not in metadata_prepared:
             metadata_prepared.update(metadata_software_version)
 
+        metadata_prepared2 = metadata_prepared
+        metadata_prepared = {}
+        for k, v in metadata_prepared2.items():
+            if isinstance(v, dict):
+                for k2, v2 in v.items():
+                    metadata_prepared[k + "|" + k2] = v2
+            else:
+                metadata_prepared[k] = v
+
+        metadata_prepared2 = metadata_prepared
+        metadata_prepared = {}
+        for k, v in metadata_prepared2.items():
+            if isinstance(v, np.int64):
+                metadata_prepared[k] = int(v)
+            elif isinstance(v, np.float64):
+                metadata_prepared[k] = float(v)
+            elif isinstance(v, np.str_):
+                metadata_prepared[k] = str(v)
+            else:
+                metadata_prepared[k] = v
+
         if metadata_prepared:
             # We assume, that metadata does not contain repeated keys. Otherwise the
             #   entry with the last occurrence of the key will override the previous ones.

pyxrf/model/scan_metadata.py

@@ -1,3 +1,4 @@
+import pprint
 import re
 import textwrap
 
@@ -64,6 +65,9 @@ def values(self):
     def items(self):
         return self._values.items()
 
+    def setdefault(self, key, default=None):
+        return self._values.setdefault(key, default)
+
     def update(self, source_dict):
         self._values.update(source_dict)
 
@@ -131,6 +135,9 @@ def cap(s):
                         # Now remove spaces at the beginning of the line, since the key will be
                         #   printed instead of the spaces
                         val = val.lstrip()
+                    elif isinstance(v, dict):
+                        val = "\n" + pprint.pformat(v)
+                        val = val.replace("\n", "\n    ")
                     else:
                         val = v
 
@@ -221,6 +228,9 @@ def _gen_default_descriptions(self):
             "instrument_mono_incident_energy": "incident energy",
             "instrument_beam_current": "ring current, mA",
             "instrument_detectors": "detectors",
+            "instrument_sample_to_detector": "sample-to-detector distance, mm",
+            "instrument_attenuators_config": "attenuators configuration",
+            "instrument_preamps_config": "preamp configuration",
             "sample_name": "sample name",
             "experiment_plan_name": "plan name",
             "experiment_plan_type": "plan type",