Merge pull request #45 from BAMresearch/BaseData-modification

Base data modification, simplified to remove the scalar and offset as it made universal maths very complex. Pipeline processes can work on multiple scalar BaseData operations before applying them to the main BaseData arrays. 

No dunder math methods have been implemented here yet, I'm testing various uncertainty handling approaches, so for now we will leave the implementation to the processing modules themselves. auto_uncertainties is a strong contender at the moment.

Author: toqduj

.gitignore

@@ -21,7 +21,7 @@ develop-eggs
 lib
 lib64
 venv*/
-.venv
+.venv*
 pyvenv*/
 pip-wheel-metadata/
 

docs/full_flow.drawio

@@ -0,0 +1,43 @@
+# /usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+__coding__ = "utf-8"
+__author__ = "Brian R. Pauw"
+__license__ = "BSD3"
+__copyright__ = "Copyright 2025, The MoDaCor team"
+__date__ = "24/05/2025"
+__version__ = "20250524.1"
+__status__ = "Development"  # "Development", "Production"
+from typing import Any, Dict
+
+import numpy as np
+import pint
+from attrs import define, field
+from attrs import validators as v
+
+from modacor import ureg
+from modacor.administration.licenses import BSD3Clause as __license__  # noqa: F401
+
+from .validators import arrays_are_equal_shape
+
+# end of header and standard imports
+
+
+@define
+class SourceData:
+    """
+    SourceData is used for a data value or array loaded from an IoSource.
+    Punitive defaults have been set to encourage compliance.
+    """
+
+    # data, units and variance are required
+    value: np.ndarray = field(validator=v.instance_of(np.ndarray))
+    units: pint.Unit = field(validator=v.instance_of(ureg.Unit))
+    variance: pint.Unit = field(validator=v.instance_of(ureg.Unit))
+    attributes: Dict[str, Any] = field(factory=dict)
+
+    def __attrs_post_init__(self):
+        if not arrays_are_equal_shape(self.data, self.variance):
+            raise ValueError("Data and variance arrays must have the same shape.")

src/modacor/dataclasses/basedata.py

@@ -5,6 +5,8 @@
 from numbers import Integral
 from typing import Any, Type
 
+import numpy as np
+
 from modacor import ureg
 
 from .databundle import DataBundle
@@ -49,6 +51,18 @@ def check_data(
     return True
 
 
+def arrays_are_equal_shape(
+    array1: np.ndarray,
+    array2: np.ndarray,
+) -> bool:
+    """
+    Check if two arrays have the same shape.
+    """
+    if array1.shape != array2.shape:
+        return False
+    return True
+
+
 def check_data_element_and_units(
     data: DataBundle,
     data_element_name: str,

src/modacor/dataclasses/source_data.py

@@ -31,18 +31,17 @@
 from os.path import abspath
 
 import h5py
-import numpy as np
 
-from modacor.dataclasses.messagehandler import *
+from modacor.dataclasses.basedata import BaseData
+from modacor.dataclasses.messagehandler import MessageHandler
 
 from ..io_source import IoSource
-from ..io_sources import IoSources
 
 
 class HDFLoader(IoSource):
-    def __init__(self, source_reference: str, logging_level = WARNING):
+    def __init__(self, source_reference: str, logging_level=WARNING):
         super().__init__(source_reference)
-        self.hdf_logger = MessageHandler(level = logging_level, name = 'hdf5logger')
+        self.hdf_logger = MessageHandler(level=logging_level, name="hdf5logger")
         self._file_path = None
         self._file_reference = None
         self._file_datasets = []
@@ -74,7 +73,20 @@ def _close_file(self):
             raise OSError(error)
 
     def _find_datasets(self, path_name, path_object):
-        """An internal function to be used to walk the tree of an HDF5 file and return a list of the datasets within"""
+        """
+        An internal function to be used to walk the tree of an HDF5 file and return a list of
+        the datasets within
+        """
         if isinstance(self._file_reference[path_name], h5py._hl.dataset.Dataset):
             self._file_datasets.append(path_name)
             self._file_datasets_shapes[path_name] = self._file_reference[path_name].shape
+
+    def get_data(self, data_key: str) -> BaseData:
+        raise (NotImplementedError("get_data method not yet implemented in HDFLoader class."))
+
+    def get_static_metadata(self, data_key):
+        raise (
+            NotImplementedError(
+                "get_static_metadata method not yet implemented in HDFLoader class."
+            )
+        )

src/modacor/dataclasses/validators.py

@@ -30,9 +30,10 @@
 
 from typing import Any
 
-import numpy as np
 from attrs import define, field
 
+from modacor.dataclasses.source_data import SourceData
+
 
 def default_config() -> dict[str, Any]:
     """
@@ -77,9 +78,9 @@ class IoSource:
 
     configuration: dict[str, Any] = field(factory=default_config)
 
-    def get_data(self, index: int, data_key: str) -> np.ndarray:
+    def get_data(self, index: int, data_key: str) -> SourceData:
         """
-        Get data from the IO source using the provided data key.
+        Get data and metadata from the IO source using the provided data key.
 
         Parameters
         ----------

src/modacor/io/hdf/hdf_loader.py

@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2025 MoDaCor Authors
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without
+#    specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+__license__ = "BSD-3-Clause"
+__copyright__ = "Copyright 2025 MoDaCor Authors"
+__status__ = "Alpha"

src/modacor/io/io_source.py

@@ -0,0 +1,96 @@
+# /usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from modacor.dataclasses.source_data import SourceData
+
+__coding__ = "utf-8"
+__author__ = "Brian R. Pauw"
+__license__ = "BSD3"
+__copyright__ = "Copyright 2025, The MoDaCor team"
+__date__ = "24/05/2025"
+__version__ = "20250524.1"
+__status__ = "Development"  # "Development", "Production"
+from logging import WARNING
+
+import h5py
+import numpy as np
+
+from modacor.administration.licenses import BSD3Clause as __license__  # noqa: F401
+from modacor.dataclasses.messagehandler import MessageHandler
+
+from ..io_source import IoSource
+
+# end of header and standard imports
+
+
+class StaticMetadata(IoSource):
+    """
+    This IoSource is used to load and make experiment metadata available to
+    the processing pipeline modules.
+    It can be filled in with information such as wavelength,
+    geometry and other relevant information which is needed in multiple
+    processing steps.
+    The metadata can be loaded from a yaml file with mappings. this is set in the configuraiton
+    The entries are returned as BaseData elements, with units and uncertainties.
+    """
+
+    _data_cache: dict[str, SourceData] = None
+    _static_metadata_cache: dict[str, Any] = None
+
+    def __init__(self, source_reference: str, logging_level=WARNING):
+        super().__init__(source_reference)
+        self.logger = MessageHandler(level=logging_level, name="StaticMetadata")
+        self._data_cache = {}  # for values with units and uncertainties
+        self._static_metadata_cache = {}  # for other elements such as strings and tags
+
+    def _load_from_yaml(self, file_path: Path) -> None:
+        """
+        Load static metadata from a YAML file.
+        This method should be implemented to parse the YAML file and populate
+        the _data_cache with SourceData objects.
+        """
+        assert file_path.exists(), f"Static metadataa file {file_path} does not exist."
+        with open(file_path, "r") as f:
+            data = yaml.safe_load(f)
+
+        for key, entry in data.items():
+            if isinstance(entry, dict):
+                if all(k in entry for k in ("value", "units", "variance")):
+                    self._data_cache[key] = SourceData(
+                        value=np.array(entry.pop("value", [])),
+                        units=entry.pop("units", "rankine"),
+                        variance=np.array(entry.pop("variance", [])),
+                        attributes=entry if entry else {},
+                    )
+                else:
+                    # invalid entry, raise an error or log it
+                    self.logger.error(
+                        f"Invalid entry for key '{key}': {entry}. Expected 'value', 'units', and 'variance'."
+                    )
+            else:
+                # Store other metadata as static metadata
+                self._static_metadata_cache[key] = entry
+
+    def get_static_metadata(self, data_key: str) -> Any:
+        if data_key not in self._static_metadata_cache:
+            self.logger.error(f"Static metadata key '{data_key}' not in cache.")
+            return None
+
+        return self._static_metadata_cache.get(data_key)
+
+    def get_data(self, data_key: str) -> SourceData:
+        """
+        Get the data from the HDF5 file.
+        """
+        if data_key not in self._data_cache:
+            self.logger.error(f"Data key '{data_key}' not in static metadata cache.")
+            return None
+
+        return self._data_cache.get(data_key)

src/modacor/io/static_metadata/__init__.py

@@ -17,7 +17,7 @@
 from modacor.math.variance_calculations import divide
 
 
-class MultiplyByVariable(ProcessStep):
+class MultiplyByScalar(ProcessStep):
     """
     Adding Poisson uncertainties to the data
     """