progress

Author: bouweandela

esmvalcore/_provenance.py

@@ -11,8 +11,6 @@
 from PIL.PngImagePlugin import PngInfo
 from prov.model import ProvDerivation, ProvDocument
 
-from esmvalcore.io.protocol import DataElement
-
 from ._version import __version__
 
 logger = logging.getLogger(__name__)
@@ -111,7 +109,7 @@ class TrackedFile:
 
     def __init__(
         self,
-        filename: Path | DataElement,
+        filename,
         attributes=None,
         ancestors=None,
         prov_filename=None,
@@ -120,8 +118,8 @@ def __init__(
 
         Arguments
         ---------
-        filename:
-            Path to the file on disk.
+        filename: :obj:`pathlib.Path` or :obj:`esmvalcore.io.protocol.DataElement`
+            Path or data element containing the data described by the provenance.
         attributes: dict
             Dictionary with facets describing the file. If set to None, this
             will be read from the file when provenance is initialized.
@@ -139,6 +137,8 @@ def __init__(
             self.prov_filename = self._filename
         else:
             self.prov_filename = prov_filename
+        # TODO: ensure global attributes are recorded for input data if they're
+        # not netcdf files.
         self.attributes = copy.deepcopy(attributes)
 
         self.provenance = None
@@ -171,20 +171,21 @@ def copy_provenance(self):
         if self.provenance is None:
             msg = f"Provenance of {self} not initialized"
             raise ValueError(msg)
-        new = TrackedFile(self.filename, self.attributes)
+        new = TrackedFile(Path(self.filename), self.attributes)
         new.provenance = copy.deepcopy(self.provenance)
         new.entity = new.provenance.get_record(self.entity.identifier)[0]
         new.activity = new.provenance.get_record(self.activity.identifier)[0]
         return new
 
     @property
-    def filename(self):
-        """Filename."""
+    def filename(self) -> str:
+        """Name of data described by this provenance document."""
         return self._filename
 
     @property
     def provenance_file(self):
-        """Filename of provenance."""
+        """Filename of provenance file."""
+        # This may not work well if filename is the instance_id.
         return os.path.splitext(self.filename)[0] + "_provenance.xml"
 
     def initialize_provenance(self, activity):

esmvalcore/_recipe/recipe.py

@@ -443,10 +443,7 @@ def _get_common_attributes(
 
     # Ensure that attributes start_year and end_year are always available if at
     # least one of the input datasets defines it
-    if "timerange" in attributes:
-        start_year, end_year = _parse_period(attributes["timerange"])
-        attributes["start_year"] = int(str(start_year[0:4]))
-        attributes["end_year"] = int(str(end_year[0:4]))
+    _set_start_end_year(attributes)
 
     return attributes
 
@@ -710,7 +707,7 @@ def _get_preprocessor_products(
     )
 
     for product in products:
-        _set_start_end_year(product)
+        _set_start_end_year(product.attributes)
         product.check()
 
     return products
@@ -770,18 +767,18 @@ def _configure_multi_product_preprocessor(
 
     for product in multimodel_products | ensemble_products:
         product.check()
-        _set_start_end_year(product)
+        _set_start_end_year(product.attributes)
 
 
-def _set_start_end_year(product: PreprocessorFile) -> None:
+def _set_start_end_year(attributes: dict[str, Any]) -> None:
     """Set the attributes `start_year` and `end_year`.
 
     These attributes are used by many diagnostic scripts in ESMValTool.
     """
-    if "timerange" in product.attributes:
-        start_year, end_year = _parse_period(product.attributes["timerange"])
-        product.attributes["start_year"] = int(str(start_year[0:4]))
-        product.attributes["end_year"] = int(str(end_year[0:4]))
+    if "timerange" in attributes:
+        start_year, end_year = _parse_period(attributes["timerange"])
+        attributes["start_year"] = int(str(start_year[0:4]))
+        attributes["end_year"] = int(str(end_year[0:4]))
 
 
 def _update_preproc_functions(

esmvalcore/_task.py

@@ -351,7 +351,7 @@ def __init__(self, prev_preproc_dir, preproc_dir, name):
         for prov_filename, attributes in prev_metadata.items():
             # Update the filename in case the output directory was moved
             # since the original run
-            filename = str(prev_preproc_dir / Path(prov_filename).name)
+            filename = prev_preproc_dir / Path(prov_filename).name
             attributes["filename"] = filename
             product = TrackedFile(
                 filename,

esmvalcore/config/_config.py

@@ -94,7 +94,7 @@ def warn_if_old_extra_facets_exist() -> None:
         )
 
 
-def load_config_developer(cfg_file):
+def load_config_developer(cfg_file) -> dict:
     """Read the developer's configuration file."""
     with open(cfg_file, encoding="utf-8") as file:
         cfg = yaml.safe_load(file)
@@ -118,6 +118,7 @@ def load_config_developer(cfg_file):
         CFG[project] = settings
 
     read_cmor_tables(cfg_file)
+    return cfg
 
 
 def get_project_config(project):

esmvalcore/config/_config_object.py

@@ -13,6 +13,7 @@
 import yaml
 
 import esmvalcore
+from esmvalcore.config._config import load_config_developer
 from esmvalcore.config._config_validators import (
     _deprecated_options_defaults,
     _deprecators,
@@ -145,6 +146,10 @@ def _load_user_config(
 
         try:
             new.update(mapping)
+            # Add known projects from config-developer file while we still have it.
+            for project in load_config_developer(new["config_developer_file"]):
+                if project not in new["projects"]:
+                    new["projects"][project] = {}
             new.check_missing()
         except InvalidConfigParameter as exc:
             msg = (
@@ -364,7 +369,10 @@ def load_from_dirs(self, dirs: Iterable[str | Path]) -> None:
         new_config_dict = self._get_config_dict_from_dirs(dirs)
         self.clear()
         self.update(new_config_dict)
-
+        # Add known projects from config-developer file while we still have it.
+        for project in load_config_developer(self["config_developer_file"]):
+            if project not in self["projects"]:
+                self["projects"][project] = {}
         self.check_missing()
 
     def reload(self) -> None:

esmvalcore/config/_data_sources.py

@@ -11,34 +11,54 @@
 logger = logging.getLogger(__name__)
 
 
-def _get_data_sources(session: Session) -> list[DataSource]:
+def _get_data_sources(
+    session: Session,
+    project: str | None,
+) -> list[DataSource]:
     """Get the list of available data sources.
 
     Arguments
     ---------
     session:
         The configuration.
+    project:
+        If specified, only data sources for this project are returned.
 
     Returns
     -------
     :obj:`list` of :obj:`DataSource`:
         A list of available data sources.
 
+    Raises
+    ------
+    TypeError:
+        If a data source in the configuration is not of type `DataSource`.
+    KeyError:
+        If the project or its settings are not found in the configuration.
+
     """
     data_sources: list[DataSource] = []
-    for project, project_settings in session["projects"].items():
+    if project is not None and project not in session["projects"]:
+        msg = f"Unknown project '{project}', please configure it under 'projects'."
+        raise KeyError(msg)
+    settings = (
+        session["projects"]
+        if project is None
+        else {project: session["projects"][project]}
+    )
+    for project_, project_settings in settings.items():
         if "data" not in project_settings:
-            logger.info("Using legacy data sources for project '%s'", project)
+            logger.info("Using legacy data sources for project '%s'", project_)
             # Use legacy data sources from config-user.yml.
-            legacy_local_sources = esmvalcore.local._get_data_sources(project)  # noqa: SLF001
+            legacy_local_sources = esmvalcore.local._get_data_sources(project_)  # noqa: SLF001
             data_sources.extend(legacy_local_sources)
             if (
                 session["search_esgf"] != "never"
-                and project in esmvalcore.esgf.facets.FACETS
+                and project_ in esmvalcore.esgf.facets.FACETS
             ):
                 data_source = esmvalcore.esgf.ESGFDataSource(
-                    name="legacy",
-                    project=project,
+                    name="legacy-esgf",
+                    project=project_,
                     priority=2,
                     download_dir=session["download_dir"],
                 )
@@ -56,16 +76,26 @@ def _get_data_sources(session: Session) -> list[DataSource]:
             priority = kwargs.pop("priority", 1)
             data_source = cls(
                 name=name,
-                project=project,
+                project=project_,
                 priority=priority,
                 **kwargs,
             )
             if not isinstance(data_source, DataSource):
                 msg = (
                     "Expected a data source of type `esmvalcore.io.protocol.DataSource`, "
-                    f"but your configuration for project '{project}' contains "
+                    f"but your configuration for project '{project_}' contains "
                     f"'{data_source}' of type '{type(data_source)}'."
                 )
                 raise TypeError(msg)
             data_sources.append(data_source)
+
+    if not data_sources:
+        if project is None:
+            msg = "No data sources found. Check your configuration under 'projects'"
+        else:
+            msg = (
+                f"No data sources found for project '{project}'. "
+                f"Check your configuration under 'projects: {project}: data'"
+            )
+        raise KeyError(msg)
     return data_sources

esmvalcore/config/configurations/dkrz-data.yml

@@ -0,0 +1,62 @@
+projects:
+  CMIP6:
+    data:
+      archive-dkrz:
+        type: "esmvalcore.local.DataSource"
+        rootpath: /work/bd0854/DATA/ESMValTool2/CMIP6_DKRZ
+        dirname_template: "{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}"
+        filename_template: "{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc"
+  CMIP5:
+    data:
+      archive-dkrz:
+        type: "esmvalcore.local.DataSource"
+        rootpath: /work/bd0854/DATA/ESMValTool2/CMIP5_DKRZ
+        dirname_template: "{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}/{short_name}"
+        filename_template: "{short_name}_{mip}_{dataset}_{exp}_{ensemble}*.nc"
+  CMIP3:
+    data:
+      archive-dkrz:
+        type: "esmvalcore.local.DataSource"
+        rootpath: /work/bd0854/DATA/ESMValTool2/CMIP3
+        dirname_template: "{exp}/{modeling_realm}/{frequency}/{short_name}/{dataset}/{ensemble}"
+        filename_template: "{short_name}_*.nc"
+  CORDEX:
+    data:
+      archive-dkrz:
+        type: "esmvalcore.local.DataSource"
+        rootpath: /work/ik1017/C3SCORDEX/data/c3s-cordex/output
+        dirname_template: "{domain}/{institute}/{driver}/{exp}/{ensemble}/{institute}-{dataset}/{rcm_version}/{mip}/{short_name}/{version}"
+        filename_template: "{short_name}_{domain}_{driver}_{exp}_{ensemble}_{institute}-{dataset}_{rcm_version}_{mip}*.nc"
+  obs4MIPs:
+    data:
+      archive-dkrz:
+        type: "esmvalcore.local.DataSource"
+        rootpath: /work/bd0854/DATA/ESMValTool2/OBS
+        dirname_template: "Tier{tier}/{dataset}"
+        filename_template: "{short_name}_*.nc"
+  native6:
+    data:
+      archive-dkrz:
+        type: "esmvalcore.local.DataSource"
+        rootpath: /work/bd0854/DATA/ESMValTool2/RAWOBS
+        dirname_template: "Tier{tier}/{dataset}/{version}/{frequency}/{short_name}"
+        filename_template: "*.nc"
+      era5-archive-dkrz:
+        type: "esmvalcore.local.DataSource"
+        rootpath: /pool/data/ERA5
+        dirname_template: "{family}/{level}/{type}/{tres}/{grib_id}"
+        filename_template: "{family}{level}{typeid}_{tres}_*_{grib_id}.grb"
+  OBS6:
+    data:
+      archive-dkrz:
+        type: "esmvalcore.local.DataSource"
+        rootpath: /work/bd0854/DATA/ESMValTool2/OBS
+        dirname_template: "Tier{tier}/{dataset}"
+        filename_template: "{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc"
+  OBS:
+    data:
+      archive-dkrz:
+        type: "esmvalcore.local.DataSource"
+        rootpath: /work/bd0854/DATA/ESMValTool2/OBS
+        dirname_template: "Tier{tier}/{dataset}"
+        filename_template: "{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc"

esmvalcore/dataset.py

@@ -756,7 +756,7 @@ def version(file: DataElement) -> str:
         self._file_globs = []
         files: dict[str, DataElement] = {}
         for data_source in sorted(
-            _get_data_sources(self.session),
+            _get_data_sources(self.session, self.facets["project"]),  # type: ignore[arg-type]
             key=lambda ds: ds.priority,
         ):
             if data_source.project == self.facets["project"]:
@@ -985,7 +985,8 @@ def _update_timerange(self) -> None:
             dataset.facets.pop("timerange")
             dataset.supplementaries = []
             check.data_availability(dataset)
-            intervals = [_get_start_end_date(f) for f in dataset.files]
+            # TODO: read start and end time from facets
+            intervals = [_get_start_end_date(f) for f in dataset.files]  # type: ignore[arg-type]
 
             min_date = min(interval[0] for interval in intervals)
             max_date = max(interval[1] for interval in intervals)

esmvalcore/esgf/_download.py

@@ -22,7 +22,7 @@
 import yaml
 from humanfriendly import format_size, format_timespan
 
-from esmvalcore.config._config import CFG
+from esmvalcore.config import CFG
 from esmvalcore.io.protocol import DataElement
 from esmvalcore.local import LocalFile
 from esmvalcore.typing import Facets

esmvalcore/esgf/_search.py

@@ -2,7 +2,7 @@
 
 import itertools
 import logging
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from functools import lru_cache
 from pathlib import Path
 
@@ -398,12 +398,9 @@ class ESGFDataSource(DataSource):
     download_dir: Path
     """The destination directory where data will be downloaded."""
 
-    debug_info: str = ""
+    debug_info: str = field(init=False, default="")
     """A string containing debug information when no data is found."""
 
-    def __post__init__(self):
-        self.debug_info = ""
-
     def find_data(self, **facets: FacetValue) -> list[ESGFFile]:
         """Find data.