Move preprocessor output filename configuration to new config

Author: bouweandela

esmvalcore/_recipe/recipe.py

@@ -27,7 +27,6 @@
     GRIB_FORMATS,
     _dates_to_timerange,
     _get_multiproduct_filename,
-    _get_output_file,
     _parse_period,
     _truncate_dates,
 )
@@ -38,6 +37,7 @@
     MULTI_MODEL_FUNCTIONS,
     PreprocessingTask,
     PreprocessorFile,
+    _get_preprocessor_filename,
 )
 from esmvalcore.preprocessor._area import _update_shapefile_path
 from esmvalcore.preprocessor._multimodel import _get_stat_identifier
@@ -678,11 +678,7 @@ def _get_preprocessor_products(
         _schedule_for_download(input_datasets)
         _log_input_files(input_datasets)
         logger.info("Found input files for %s", dataset.summary(shorten=True))
-
-        filename = _get_output_file(
-            dataset.facets,
-            dataset.session.preproc_dir,
-        )
+        filename = _get_preprocessor_filename(dataset)
         product = PreprocessorFile(
             filename=filename,
             attributes=dataset.facets,

esmvalcore/config/_config_object.py

@@ -11,7 +11,6 @@
 import dask.config
 
 import esmvalcore
-from esmvalcore.config._config import load_config_developer
 from esmvalcore.config._config_validators import (
     _deprecated_options_defaults,
     _deprecators,
@@ -129,10 +128,6 @@ def load_from_dirs(self, dirs: Iterable[str | Path]) -> None:
         new_config_dict = self._get_config_dict_from_dirs(dirs)
         self.clear()
         self.update(new_config_dict)
-        # Add known projects from config-developer file while we still have it.
-        for project in load_config_developer(self["config_developer_file"]):
-            if project not in self["projects"]:
-                self["projects"][project] = {}
         self.check_missing()
 
     def reload(self) -> None:

esmvalcore/config/_config_validators.py

@@ -375,6 +375,7 @@ def validate_projects(
     options_for_project: dict[str, Callable[[Any], Any]] = {
         "data": validate_dict,  # TODO: try to create data sources here
         "extra_facets": validate_dict,
+        "preprocessor_filename_template": validate_string,
     }
     for project, project_config in mapping.items():
         for option, val in project_config.items():

esmvalcore/config/configurations/defaults/preprocessor_filename_template.yml

@@ -0,0 +1,33 @@
+# Templates for the filenames used to write preprocessor output.
+projects:
+  # ESGF projects.
+  CMIP3:
+    preprocessor_filename_template: "{project}_{institute}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}"
+  CMIP5:
+    preprocessor_filename_template: "{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}"
+  CMIP6:
+    preprocessor_filename_template: "{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{grid}"
+  CORDEX:
+    preprocessor_filename_template: "{project}_{institute}_{dataset}_{rcm_version}_{driver}_{domain}_{mip}_{exp}_{ensemble}_{short_name}"
+  obs4MIPs:
+    preprocessor_filename_template: "{project}_{dataset}_{short_name}"
+  # Observational and reanalysis data that has been CMORized by ESMValTool according to the CMIP5 standard.
+  OBS:
+    preprocessor_filename_template: "{project}_{dataset}_{type}_{version}_{mip}_{short_name}"
+  # Observational and reanalysis data that has been CMORized by ESMValTool according to the CMIP6 standard.
+  OBS6:
+    preprocessor_filename_template: "{project}_{dataset}_{type}_{version}_{mip}_{short_name}"
+  # Observational and reanalysis data that can be read in its native format by ESMValCore.
+  native6:
+    preprocessor_filename_template: "{project}_{dataset}_{type}_{version}_{mip}_{short_name}"
+  # Data from various climate models in their native output format.
+  ACCESS:
+    preprocessor_filename_template: "{project}_{dataset}_{mip}_{exp}_{institute}_{sub_dataset}_{freq_attribute}_{short_name}"
+  CESM:
+    preprocessor_filename_template: "{project}_{dataset}_{case}_{gcomp}_{scomp}_{type}_{mip}_{short_name}"
+  EMAC:
+    preprocessor_filename_template: "{project}_{dataset}_{exp}_{channel}_{mip}_{short_name}"
+  ICON:
+    preprocessor_filename_template: "{project}_{dataset}_{exp}_{var_type}_{mip}_{short_name}"
+  IPSLCM:
+    preprocessor_filename_template: "{dataset}_{account}_{model}_{status}_{exp}_{simulation}_{freq}_{short_name}"

esmvalcore/dataset.py

@@ -27,8 +27,8 @@
 )
 from esmvalcore.config._data_sources import _get_data_sources
 from esmvalcore.exceptions import InputFilesNotFound, RecipeError
-from esmvalcore.io.local import _dates_to_timerange, _get_output_file
-from esmvalcore.preprocessor import preprocess
+from esmvalcore.io.local import _dates_to_timerange
+from esmvalcore.preprocessor import _get_preprocessor_filename, preprocess
 
 if TYPE_CHECKING:
     from collections.abc import Iterable, Iterator, Sequence
@@ -815,7 +815,7 @@ def load(self) -> Cube:
             supplementary_cube = supplementary_dataset._load()  # noqa: SLF001
             supplementary_cubes.append(supplementary_cube)
 
-        output_file = _get_output_file(self.facets, self.session.preproc_dir)
+        output_file = _get_preprocessor_filename(self)
         cubes = preprocess(
             [cube],
             "add_supplementary_variables",
@@ -833,7 +833,7 @@ def _load(self) -> Cube:
             msg = check.get_no_data_message(self)
             raise InputFilesNotFound(msg)
 
-        output_file = _get_output_file(self.facets, self.session.preproc_dir)
+        output_file = _get_preprocessor_filename(self)
         fix_dir_prefix = Path(
             self.session._fixed_file_dir,  # noqa: SLF001
             self._get_joined_summary_facets("_", join_lists=True) + "_",

esmvalcore/io/local.py

@@ -58,7 +58,6 @@
 from netCDF4 import Dataset
 
 import esmvalcore.io.protocol
-from esmvalcore.config._config import get_project_config
 from esmvalcore.exceptions import RecipeError
 from esmvalcore.iris_helpers import ignore_warnings_context
 
@@ -697,27 +696,6 @@ def _templates_to_regex(self) -> str:
         return pattern
 
 
-def _get_output_file(variable: dict[str, Any], preproc_dir: Path) -> Path:
-    """Return the full path to the output (preprocessed) file."""
-    cfg = get_project_config(variable["project"])
-
-    # Join different experiment names
-    if isinstance(variable.get("exp"), (list, tuple)):
-        variable = dict(variable)
-        variable["exp"] = "-".join(variable["exp"])
-    outfile = _replace_tags(cfg["output_file"], variable)[0]
-    if "timerange" in variable:
-        timerange = variable["timerange"].replace("/", "-")
-        outfile = Path(f"{outfile}_{timerange}")
-    outfile = Path(f"{outfile}.nc")
-    return Path(
-        preproc_dir,
-        variable.get("diagnostic", ""),
-        variable.get("variable_group", ""),
-        outfile,
-    )
-
-
 def _get_multiproduct_filename(attributes: dict, preproc_dir: Path) -> Path:
     """Get ensemble/multi-model filename depending on settings."""
     relevant_keys = [

esmvalcore/local.py

@@ -9,16 +9,18 @@
 
 import logging
 import os.path
+import textwrap
 import warnings
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from esmvalcore.config import CFG
 from esmvalcore.config._config import get_ignored_warnings, get_project_config
 from esmvalcore.io.local import (
     LocalDataSource,
     LocalFile,
     _filter_versions_called_latest,
+    _replace_tags,
     _select_latest_version,
 )
 
@@ -255,3 +257,39 @@ def find_files(
             globs.extend(data_source._get_glob_patterns(**facets))  # noqa: SLF001
         return files, sorted(globs)
     return files
+
+
+_GET_OUTPUT_PATH_WARNED: set[str] = set()
+
+
+def _get_output_file(variable: dict[str, Any], preproc_dir: Path) -> Path:
+    """Return the full path to the output (preprocessed) file."""
+    project = variable["project"]
+    cfg = get_project_config(project)
+    if project not in _GET_OUTPUT_PATH_WARNED:
+        _GET_OUTPUT_PATH_WARNED.add(project)
+        msg = textwrap.dedent(
+            f"""
+            Defining 'output_file' in config-develop.yml is deprecated and will be removed in version 2.16.0. Please use the following configuration instead:
+            projects:
+              {variable["project"]}:
+                preprocessor_filename_template: "{cfg["output_file"]}"
+            """.rstrip(),
+        )
+        logger.warning(msg)
+
+    # Join different experiment names
+    if isinstance(variable.get("exp"), (list, tuple)):
+        variable = dict(variable)
+        variable["exp"] = "-".join(variable["exp"])
+    outfile = _replace_tags(cfg["output_file"], variable)[0]
+    if "timerange" in variable:
+        timerange = variable["timerange"].replace("/", "-")
+        outfile = Path(f"{outfile}_{timerange}")
+    outfile = Path(f"{outfile}.nc")
+    return Path(
+        preproc_dir,
+        variable.get("diagnostic", ""),
+        variable.get("variable_group", ""),
+        outfile,
+    )

esmvalcore/preprocessor/__init__.py

@@ -5,6 +5,9 @@
 import copy
 import inspect
 import logging
+import re
+from collections.abc import Sequence
+from pathlib import Path
 from pprint import pformat
 from typing import TYPE_CHECKING, Any
 
@@ -14,7 +17,10 @@
 from esmvalcore._task import BaseTask
 from esmvalcore.cmor.check import cmor_check_data, cmor_check_metadata
 from esmvalcore.cmor.fix import fix_data, fix_file, fix_metadata
+from esmvalcore.exceptions import RecipeError
+from esmvalcore.io.local import _parse_period
 from esmvalcore.io.protocol import DataElement
+from esmvalcore.local import _get_output_file
 from esmvalcore.preprocessor._area import (
     area_statistics,
     extract_named_regions,
@@ -102,14 +108,14 @@
 from esmvalcore.preprocessor._weighting import weighting_landsea_fraction
 
 if TYPE_CHECKING:
-    from collections.abc import Callable, Iterable, Sequence
-    from pathlib import Path
+    from collections.abc import Callable, Iterable
 
     import prov.model
     from dask.delayed import Delayed
     from iris.cube import CubeList
 
     from esmvalcore.dataset import Dataset
+    from esmvalcore.typing import FacetValue
 
 logger = logging.getLogger(__name__)
 
@@ -254,6 +260,71 @@
 }
 
 
+def _get_preprocessor_filename(dataset: Dataset) -> Path:
+    """Get a filename for storing a preprocessed dataset.
+
+    Parameters
+    ----------
+    dataset:
+        The dataset that will be preprocessed.
+
+    Returns
+    -------
+    :
+        A path for storing a preprocessed file.
+    """
+
+    def is_facet_value(value: Any) -> bool:  # noqa: ANN401
+        """Check if a value is of type `esmvalcore.typing.FacetValue`."""
+        return isinstance(value, str | int) or (
+            isinstance(value, Sequence)
+            and all(isinstance(v, str) for v in value)
+        )
+
+    default_template = "_".join(
+        f"{{{k}}}"
+        for k in sorted(dataset.minimal_facets)
+        if is_facet_value(dataset.minimal_facets[k])
+        and k
+        not in ("timerange", "diagnostic", "variable_group", "preprocessor")
+    )
+    template = (
+        dataset.session["projects"]
+        .get(dataset.facets["project"], {})
+        .get("preprocessor_filename_template", default_template)
+    )
+    if template is default_template:
+        try:
+            # Use config-developer.yml for backward compatibility, remove in v2.16.
+            return _get_output_file(
+                dataset.facets,
+                dataset.session.preproc_dir,
+            )
+        except RecipeError:
+            pass
+
+    def normalize(value: FacetValue) -> str:
+        """Normalize a facet value to a string that can be used in a filename."""
+        if isinstance(value, str | int):
+            return re.sub("[^a-zA-Z0-9]+", "-", str(value))[:25]
+        return "-".join(normalize(v) for v in value)
+
+    normalized_facets = {
+        k: normalize(v) for k, v in dataset.facets.items() if is_facet_value(v)
+    }
+    filename = template.format(**normalized_facets)
+    if "timerange" in dataset.facets:
+        start_time, end_time = _parse_period(dataset.facets["timerange"])
+        filename += f"_{start_time}-{end_time}"
+    filename += ".nc"
+    return Path(
+        dataset.session.preproc_dir,
+        dataset.facets.get("diagnostic", ""),  # type: ignore[arg-type]
+        dataset.facets.get("variable_group", ""),  # type: ignore[arg-type]
+        filename,
+    )
+
+
 def _get_itype(step: str) -> str:
     """Get the input type of a preprocessor function."""
     function = globals()[step]
@@ -520,7 +591,7 @@ def __init__(
         filename: Path,
         attributes: dict[str, Any] | None = None,
         settings: dict[str, Any] | None = None,
-        datasets: list | None = None,
+        datasets: list[Dataset] | None = None,
     ) -> None:
         if datasets is not None:
             # Load data using a Dataset

tests/integration/io/test_local.py

@@ -13,10 +13,9 @@
 from esmvalcore.io.local import (
     LocalDataSource,
     LocalFile,
-    _get_output_file,
     _parse_period,
 )
-from esmvalcore.local import _select_drs, find_files
+from esmvalcore.local import _get_output_file, _select_drs, find_files
 
 # Load test configuration
 with open(