Add an interface for adding new data sources and add support for intake-esgf as a first example

.circleci/config.yml

@@ -66,7 +66,7 @@ commands:
       - run:
           name: Install git+ssh
           environment:
-            DEBIAN_FRONTEND: noninteractive  # needed to install tzdata
+            DEBIAN_FRONTEND: noninteractive # needed to install tzdata
           command: apt update && apt install -y git ssh
       - checkout
       - check_changes
@@ -141,6 +141,7 @@ jobs:
             . /opt/conda/etc/profile.d/conda.sh
             mkdir /logs
             conda activate esmvaltool
+            pip install intake-esgf 'globus-sdk<4' # TODO: remove before merging
             pip install --no-deps .[test] > /logs/install.txt 2>&1
             pip check
       - test_and_report:
@@ -155,7 +156,7 @@ jobs:
           name: Install gpg (required by codecov orb)
           command: apt update && apt install -y gpg
       - codecov/upload:
-          files: 'test-reports/coverage.xml'
+          files: "test-reports/coverage.xml"
           disable_search: true
 
   test_installation_from_source_test_mode:

.gitignore

@@ -1,5 +1,6 @@
 # Autogenerated files
 _sidebar.rst.inc
+jupyter_execute/
 
 # Distribution / packaging
 .Python

doc/api/esmvalcore.esgf.rst

@@ -1,18 +1,10 @@
 Find and download files from ESGF
 =================================
 
-This module provides the function :py:func:`esmvalcore.esgf.find_files`
-for searching for files on ESGF using the ESMValTool vocabulary.
-It returns :py:class:`esmvalcore.esgf.ESGFFile` objects, which have a convenient
-:py:meth:`esmvalcore.esgf.ESGFFile.download` method for downloading the files.
-
-See :ref:`config-esgf` for instructions on configuring this module.
-
 esmvalcore.esgf
 ---------------
-.. autofunction:: esmvalcore.esgf.find_files
-.. autofunction:: esmvalcore.esgf.download
-.. autoclass:: esmvalcore.esgf.ESGFFile
+.. automodule:: esmvalcore.esgf
+    :noindex:
 
 esmvalcore.esgf.facets
 ----------------------

doc/api/esmvalcore.io.intake_esgf.rst

@@ -0,0 +1,5 @@
+esmvalcore.io.intake_esgf
+=========================
+
+.. automodule:: esmvalcore.io.intake_esgf
+    :no-inherited-members:

doc/api/esmvalcore.io.protocol.rst

@@ -0,0 +1,5 @@
+esmvalcore.io.protocol
+======================
+
+.. automodule:: esmvalcore.io.protocol
+    :no-inherited-members:

doc/api/esmvalcore.io.rst

@@ -0,0 +1,18 @@
+Access data from any source
+===========================
+
+ESMValCore supports a modular system for reading data from various data sources.
+In the future, this module may be extended with support for writing output data.
+
+The interface is defined in the :mod:`esmvalcore.io.protocol` module and
+the other modules here provide an implementation for a particular data source.
+
+.. toctree::
+   :maxdepth: 1
+
+   esmvalcore.io.protocol
+   esmvalcore.io.intake_esgf
+
+esmvalcore.io
+-------------
+.. automodule:: esmvalcore.io

doc/api/esmvalcore.rst

@@ -14,6 +14,7 @@ library. This section documents the public API of ESMValCore.
    esmvalcore.dataset
    esmvalcore.esgf
    esmvalcore.exceptions
+   esmvalcore.io
    esmvalcore.iris_helpers
    esmvalcore.local
    esmvalcore.preprocessor

doc/conf.py

@@ -461,6 +461,7 @@
     'dask': ('https://docs.dask.org/en/stable/', None),
     'distributed': ('https://distributed.dask.org/en/stable/', None),
     'iris': ('https://scitools-iris.readthedocs.io/en/stable/', None),
+    'intake_esgf': ('https://intake-esgf.readthedocs.io/en/stable/', None),
     'esmf_regrid': ('https://iris-esmf-regrid.readthedocs.io/en/stable/', None),
     'matplotlib': ('https://matplotlib.org/stable/', None),
     'ncdata': ('https://ncdata.readthedocs.io/en/stable/', None),

doc/configurations

@@ -0,0 +1 @@
+../esmvalcore/config/configurations

environment.yml

@@ -9,7 +9,7 @@ dependencies:
   - cartopy
   - cf-units
   - cftime
-  - dask >=2025  # github.com/ESMValGroup/ESMValCore/issues/2503
+  - dask >=2025 # github.com/ESMValGroup/ESMValCore/issues/2503
   - dask-jobqueue
   - distributed
   - esgf-pyclient >=0.3.1
@@ -20,13 +20,15 @@ dependencies:
   - fire
   - geopy
   - humanfriendly
+  - intake-esgf
+  - globus-sdk <4 # https://github.com/esgf2-us/intake-esgf/issues/150
   - intake-esm
-  - iris >=3.12.2  # https://github.com/SciTools/iris/issues/6417
+  - iris >=3.12.2 # https://github.com/SciTools/iris/issues/6417
   - iris-esmf-regrid >=0.11.0
-  - iris-grib >=0.20.0  # github.com/ESMValGroup/ESMValCore/issues/2535
-  - isodate >=0.7.0  # incompatible with very old 0.6.1
+  - iris-grib >=0.20.0 # github.com/ESMValGroup/ESMValCore/issues/2535
+  - isodate >=0.7.0 # incompatible with very old 0.6.1
   - jinja2
-  - libnetcdf !=4.9.1  # to avoid hdf5 warnings; only on conda-forge
+  - libnetcdf !=4.9.1 # to avoid hdf5 warnings; only on conda-forge
   - nc-time-axis
   - ncdata
   - nested-lookup

esmvalcore/_main.py

@@ -159,6 +159,97 @@ class Config:
     files.
     """
 
+    def __init__(self) -> None:
+        from rich.console import Console
+
+        self.console = Console()
+
+    def show(
+        self,
+        filter: tuple[str] | None = ("extra_facets",),  # noqa: A002
+    ) -> None:
+        """Show the current configuration.
+
+        Parameters
+        ----------
+        filter:
+            Filter this list of keys. By default, the `extra_facets`
+            key is filtered out, as it can be very large.
+
+        """
+        import yaml
+        from nested_lookup import nested_delete
+        from rich.syntax import Syntax
+
+        from esmvalcore.config import CFG
+
+        cfg = dict(CFG)
+        if filter:
+            for key in filter:
+                cfg = nested_delete(cfg, key)
+        exclude_msg = (
+            ", excluding the keys " + ", ".join(f"'{f}'" for f in filter)
+            if filter
+            else ""
+        )
+        self.console.print(f"# Current configuration{exclude_msg}:")
+        self.console.print(
+            Syntax(
+                yaml.safe_dump(cfg),
+                "yaml",
+                background_color="default",
+            ),
+        )
+
+    def list(self) -> None:
+        """List all available example configuration files."""
+        import importlib.resources
+
+        import esmvalcore.config
+
+        config_dir = (
+            importlib.resources.files(esmvalcore.config) / "configurations"
+        )
+        self.console.print("Available configuration files:")
+        available_files = sorted(
+            f.name
+            for f in config_dir.iterdir()
+            if f.suffix == ".yml"  # type: ignore[attr-defined]
+        )
+        self.console.print("\n".join(f"- {f}" for f in available_files))
+
+    def copy(
+        self,
+        source_file: str,
+        target_file: Path | None = None,
+        overwrite: bool = False,
+    ) -> None:
+        """Copy one of the available example configuration files to the configuration directory."""
+        import importlib.resources
+
+        import esmvalcore.config
+
+        target_dir = esmvalcore.config._config_object._get_user_config_dir()  # noqa: SLF001
+        target_file = target_dir / (
+            source_file if target_file is None else target_file
+        )
+        config_dir = (
+            importlib.resources.files(esmvalcore.config) / "configurations"
+        )
+        available_files = sorted(
+            f.name
+            for f in config_dir.iterdir()
+            if f.suffix == ".yml"  # type: ignore[attr-defined]
+        )
+        if source_file not in available_files:
+            msg = (
+                f"Configuration file {source_file} not found, choose from "
+                f"{', '.join(available_files)}"
+            )
+            raise FileNotFoundError(msg)
+        with importlib.resources.as_file(config_dir / source_file) as file:
+            self._copy_config_file(file, target_file, overwrite=overwrite)
+
     @staticmethod
     def _copy_config_file(
         in_file: Path,
@@ -184,7 +275,7 @@ def _copy_config_file(
             logger.info("Creating folder %s", target_folder)
             target_folder.mkdir(parents=True, exist_ok=True)
 
-        logger.info("Copying file %s to path %s.", in_file, out_file)
+        logger.info("Copying file %s to path %s", in_file, out_file)
         shutil.copy2(in_file, out_file)
         logger.info("Copy finished.")