Modify DataReaderObs to get base_yyyy... from stream config (ecmwf#794)

* modify DataReaderObs to get base_yyyy... from stream config, and set it in the ctor, with default of 19700101. Use it in _setup_sample_index. Remove loading obs_id attr. Add igra.yml with example usage.

* add license to igra config

* update to ISO base_datetime, parse to read idx from zarr

Author: shmh40

config/streams/igra/igra.yml

@@ -0,0 +1,34 @@
+# (C) Copyright 2025 WeatherGenerator contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+IGRA :
+  type : obs
+  filenames : ['igra.zarr']
+  base_datetime : '1750-01-01T00:00:00'
+  loss_weight : 1.0
+  #masking_rate_none : 0.05
+  token_size : 64
+  tokenize_spacetime : True
+  max_num_targets: -1
+  embed : 
+    net : transformer
+    num_tokens : 1
+    num_heads : 2
+    dim_embed : 256
+    num_blocks : 2
+  embed_target_coords : 
+    net : linear
+    dim_embed : 256
+  target_readout :
+    type : 'obs_value'  # token or obs_value
+    num_layers : 2
+    num_heads : 4
+  pred_head : 
+    ens_size : 1
+    num_layers : 1

src/weathergen/datasets/data_reader_obs.py

@@ -33,7 +33,12 @@ def __init__(self, tw_handler: TimeWindowHandler, filename: Path, stream_info: d
         self.z = zarr.open(filename, mode="r")
         self.data = self.z["data"]
         self.dt = self.z["dates"]  # datetime only
-        self.hrly_index = self.z["idx_197001010000_1"]
+        self.base_datetime = stream_info.get("base_datetime", "1970-01-01T00:00:00")
+        format_str = "%Y-%m-%dT%H:%M:%S"
+        self.base_datetime = datetime.datetime.strptime(str(self.base_datetime), format_str)
+        # To read idx convert to a string, format e.g.: 197001010000
+        base_date_str = self.base_datetime.strftime("%Y%m%d%H%M")
+        self.hrly_index = self.z[f"idx_{base_date_str}_1"]
         self.colnames = self.data.attrs["colnames"]
 
         data_colnames = [col for col in self.colnames if "obsvalue" in col]
@@ -63,7 +68,7 @@ def __init__(self, tw_handler: TimeWindowHandler, filename: Path, stream_info: d
         self.geoinfo_idx = list(range(self.coords_idx[-1] + 1, data_idx[0]))
         self.geoinfo_channels = [self.colnames[i] for i in self.geoinfo_idx]
 
-        # load additional properties (mean, var, obs_id)
+        # load additional properties (mean, var)
         self._load_properties()
         self.mean = np.array(self.properties["means"])  # [data_idx]
         self.stdev = np.sqrt(np.array(self.properties["vars"]))  # [data_idx])
@@ -140,19 +145,13 @@ def _setup_sample_index(self) -> None:
         )
         step_hrs = int(self.time_window_handler.t_window_step.item().total_seconds()) // 3600
 
-        # TODO: move to ctor
-        base_yyyymmddhhmm = 197001010000
-
-        # Derive new index based on hourly backbone index
-        format_str = "%Y%m%d%H%M%S"
-        base_dt = datetime.datetime.strptime(str(base_yyyymmddhhmm), format_str)
         self.start_dt = self.time_window_handler.t_start.item()
         self.end_dt = self.time_window_handler.t_end.item()
 
         ## Calculate the number of hours between start of hourly base index
         #  and the requested sample index
-        diff_in_hours_start = int((self.start_dt - base_dt).total_seconds() / 3600)
-        diff_in_hours_end = int((self.end_dt - base_dt).total_seconds() / 3600)
+        diff_in_hours_start = int((self.start_dt - self.base_datetime).total_seconds() / 3600)
+        diff_in_hours_end = int((self.end_dt - self.base_datetime).total_seconds() / 3600)
 
         end_range_1 = min(diff_in_hours_end, self.hrly_index.shape[0] - 1)
         self.indices_start = self.hrly_index[diff_in_hours_start:end_range_1:step_hrs]
@@ -201,7 +200,6 @@ def _load_properties(self) -> None:
 
         self.properties["means"] = self.data.attrs["means"]
         self.properties["vars"] = self.data.attrs["vars"]
-        self.properties["obs_id"] = self.data.attrs["obs_id"]
 
     @override
     def _get(self, idx: int, channels_idx: list[int]) -> ReaderData: