Add more docs and command line tools

Author: bouweandela

doc/api/esmvalcore.esgf.rst

@@ -1,15 +1,6 @@
 Find and download files from ESGF
 =================================
 
-This module provides the function :py:func:`esmvalcore.esgf.find_files`
-for searching for files on ESGF using the ESMValTool vocabulary.
-It returns :py:class:`esmvalcore.esgf.ESGFFile` objects, which have a convenient
-:py:meth:`esmvalcore.esgf.ESGFFile.download` method for downloading the file
-as well as a :func:`esmvalcore.esgf.download` function for downloading multiple
-files in parallel.
-
-See :ref:`config-esgf` for instructions on configuring this module.
-
 esmvalcore.esgf
 ---------------
 .. automodule:: esmvalcore.esgf

doc/api/esmvalcore.io.intake_esgf.rst

@@ -1,5 +1,5 @@
-Load data using intake-esgf
-===========================
+esmvalcore.io.intake_esgf
+=========================
 
 .. automodule:: esmvalcore.io.intake_esgf
     :no-inherited-members:

doc/api/esmvalcore.io.protocol.rst

@@ -1,5 +1,5 @@
-Protocols for accessing data
-============================
+esmvalcore.io.protocol
+======================
 
 .. automodule:: esmvalcore.io.protocol
     :no-inherited-members:

esmvalcore/_main.py

@@ -159,6 +159,97 @@ class Config:
     files.
     """
 
+    def __init__(self) -> None:
+        from rich.console import Console
+
+        self.console = Console()
+
+    def show(
+        self,
+        filter: tuple[str] | None = ("extra_facets",),  # noqa: A002
+    ) -> None:
+        """Show the current configuration.
+
+        Parameters
+        ----------
+        filter:
+            Filter this list of keys. By default, the `extra_facets`
+            key is filtered out, as it can be very large.
+
+        """
+        import yaml
+        from nested_lookup import nested_delete
+        from rich.syntax import Syntax
+
+        from esmvalcore.config import CFG
+
+        cfg = dict(CFG)
+        if filter:
+            for key in filter:
+                cfg = nested_delete(cfg, key)
+        exclude_msg = (
+            ", excluding the keys " + ", ".join(f"'{f}'" for f in filter)
+            if filter
+            else ""
+        )
+        self.console.print(f"# Current configuration{exclude_msg}:")
+        self.console.print(
+            Syntax(
+                yaml.safe_dump(cfg),
+                "yaml",
+                background_color="default",
+            ),
+        )
+
+    def list(self) -> None:
+        """List all available example configuration files."""
+        import importlib.resources
+
+        import esmvalcore.config
+
+        config_dir = (
+            importlib.resources.files(esmvalcore.config) / "configurations"
+        )
+        self.console.print("Available configuration files:")
+        available_files = sorted(
+            f.name
+            for f in config_dir.iterdir()
+            if f.suffix == ".yml"  # type: ignore[attr-defined]
+        )
+        self.console.print("\n".join(f"- {f}" for f in available_files))
+
+    def copy(
+        self,
+        source_file: str,
+        target_file: Path | None = None,
+        overwrite: bool = False,
+    ) -> None:
+        """Copy one of the available example configuration files to the configuration directory."""
+        import importlib.resources
+
+        import esmvalcore.config
+
+        target_dir = esmvalcore.config._config_object._get_user_config_dir()  # noqa: SLF001
+        target_file = target_dir / (
+            source_file if target_file is None else target_file
+        )
+        config_dir = (
+            importlib.resources.files(esmvalcore.config) / "configurations"
+        )
+        available_files = sorted(
+            f.name
+            for f in config_dir.iterdir()
+            if f.suffix == ".yml"  # type: ignore[attr-defined]
+        )
+        if source_file not in available_files:
+            msg = (
+                f"Configuration file {source_file} not found, choose from "
+                f"{', '.join(available_files)}"
+            )
+            raise FileNotFoundError(msg)
+        with importlib.resources.as_file(config_dir / source_file) as file:
+            self._copy_config_file(file, target_file, overwrite=overwrite)
+
     @staticmethod
     def _copy_config_file(
         in_file: Path,
@@ -184,7 +275,7 @@ def _copy_config_file(
             logger.info("Creating folder %s", target_folder)
             target_folder.mkdir(parents=True, exist_ok=True)
 
-        logger.info("Copying file %s to path %s.", in_file, out_file)
+        logger.info("Copying file %s to path %s", in_file, out_file)
         shutil.copy2(in_file, out_file)
         logger.info("Copy finished.")
 

esmvalcore/cmor/_fixes/icon/_base_fixes.py

@@ -324,7 +324,7 @@ def _get_grid_from_rootpath(self, grid_name: str) -> CubeList | None:
         """Try to get grid from the ICON rootpath."""
         glob_patterns: list[Path] = []
         for data_source in _get_data_sources(self.session, "ICON"):  # type: ignore[arg-type]
-            if isinstance(data_source, esmvalcore.local.DataSource):
+            if isinstance(data_source, esmvalcore.local.LocalDataSource):
                 glob_patterns.extend(
                     data_source.get_glob_patterns(**self.extra_facets),
                 )

esmvalcore/cmor/check.py

@@ -14,6 +14,7 @@
 import iris.exceptions
 import iris.util
 import numpy as np
+import yaml
 
 from esmvalcore.cmor._utils import (
     _get_alternative_generic_lev_coord,
@@ -50,6 +51,12 @@ class CheckLevels(IntEnum):
     """Do not fail for any discrepancy with CMOR standards."""
 
 
+yaml.representer.SafeRepresenter.add_representer(
+    CheckLevels,
+    lambda dumper, data: dumper.represent_str(data.name.lower()),
+)
+
+
 class CMORCheckError(Exception):
     """Exception raised when a cube does not pass the CMORCheck."""
 

esmvalcore/config/_config_object.py

@@ -51,9 +51,7 @@ def _get_user_config_dir() -> Path:
                 f"ESMVALTOOL_CONFIG_DIR environment variable: "
                 f"{user_config_dir} is not an existing directory"
             )
-            raise NotADirectoryError(
-                msg,
-            )
+            raise NotADirectoryError(msg)
         return user_config_dir
     return Path.home() / ".config" / "esmvaltool"
 

esmvalcore/config/_config_validators.py

@@ -347,7 +347,7 @@ def validate_projects(value: Any) -> Any:
     """Validate projects mapping."""
     mapping = validate_dict(value)
     options_for_project: dict[str, Callable[[Any], Any]] = {
-        "data": validate_dict,
+        "data": validate_dict,  # TODO: try to create data sources here
         "extra_facets": validate_dict,
     }
     for project, project_config in mapping.items():

esmvalcore/config/configurations/access-data.yml

@@ -2,12 +2,12 @@ projects:
   ACCESS:
     data:
       access-sub-dataset:
-        type: "esmvalcore.local.DataSource"
+        type: "esmvalcore.local.LocalDataSource"
         rootpath: ~/climate_data
         dirname_template: "{dataset}/{sub_dataset}/{exp}/{modeling_realm}/netCDF"
         filename_template: "{sub_dataset}.{freq_attribute}-*.nc"
       access-ocean:
-        type: "esmvalcore.local.DataSource"
+        type: "esmvalcore.local.LocalDataSource"
         rootpath: ~/climate_data
         dirname_template: "{dataset}/{sub_dataset}/{exp}/{modeling_realm}/netCDF"
         filename_template: "ocean_{freq_attribute}.nc-*"

esmvalcore/config/configurations/badc-data.yml

@@ -2,49 +2,49 @@ projects:
   CMIP6:
     data:
       badc:
-        type: "esmvalcore.local.DataSource"
+        type: "esmvalcore.local.LocalDataSource"
         rootpath: /badc/cmip6/data
         dirname_template: "{project}/{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}"
         filename_template: "{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc"
   CMIP5:
     data:
       badc:
-        type: "esmvalcore.local.DataSource"
+        type: "esmvalcore.local.LocalDataSource"
         rootpath: /badc/cmip5/data
         dirname_template: "{project.lower}/{product}/{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}"
         filename_template: "{short_name}_{mip}_{dataset}_{exp}_{ensemble}*.nc"
   CMIP3:
     data:
       badc:
-        type: "esmvalcore.local.DataSource"
+        type: "esmvalcore.local.LocalDataSource"
         rootpath: /badc/cmip3_drs/data
         dirname_template: "{project.lower}/output/{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{short_name}/{ensemble}/{version}"
         filename_template: "{short_name}_*.nc"
   CORDEX:
     data:
       badc:
-        type: "esmvalcore.local.DataSource"
+        type: "esmvalcore.local.LocalDataSource"
         rootpath: /badc/cordex/data
         dirname_template: "{project}/output/{domain}/{institute}/{driver}/{exp}/{ensemble}/{institute}-{dataset}/{rcm_version}/{mip}/{short_name}/{version}"
         filename_template: "{short_name}_{domain}_{driver}_{exp}_{ensemble}_{institute}-{dataset}_{rcm_version}_{mip}*.nc"
   obs4MIPs:
     data:
       badc:
-        type: "esmvalcore.local.DataSource"
+        type: "esmvalcore.local.LocalDataSource"
         rootpath: /gws/nopw/j04/esmeval/obsdata-v2
         dirname_template: "Tier{tier}/{dataset}"
         filename_template: "{short_name}_*.nc"
   OBS6:
     data:
       badc:
-        type: "esmvalcore.local.DataSource"
+        type: "esmvalcore.local.LocalDataSource"
         rootpath: /gws/nopw/j04/esmeval/obsdata-v2
         dirname_template: "Tier{tier}/{dataset}"
         filename_template: "{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc"
   OBS:
     data:
       badc:
-        type: "esmvalcore.local.DataSource"
+        type: "esmvalcore.local.LocalDataSource"
         rootpath: /gws/nopw/j04/esmeval/obsdata-v2
         dirname_template: "Tier{tier}/{dataset}"
         filename_template: "{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc"