Merge pull request #227 from Climate-REF/pmp-reference

Author: lewisjared

.github/workflows/ci-integration.yaml

@@ -41,7 +41,7 @@ jobs:
       - name: Run tests
         run: |
           make fetch-test-data
-          uv run python scripts/fetch-ilamb-data.py ilamb.txt
+          uv run ref datasets fetch-data ilamb
           make test
       # Upload the scratch and results directories as artifacts
       - name: Upload scratch artifacts

Makefile

@@ -94,7 +94,7 @@ test-metrics-esmvaltool:  ## run the tests
 
 .PHONY: test-metrics-ilamb
 test-metrics-ilamb:  ## run the tests
-	uv run --package cmip_ref_metrics_ilamb python ./scripts/fetch-ilamb-data.py test.txt
+	uv run ref datasets fetch-data --registry ilamb-test
 	uv run --package cmip_ref_metrics_ilamb \
 		pytest packages/ref-metrics-ilamb \
 		-r a -v --doctest-modules --cov=packages/ref-metrics-ilamb/src --cov-report=term --cov-append
@@ -170,13 +170,13 @@ virtual-environment:  ## update virtual environment, create a new one if it does
 .PHONY: fetch-test-data
 fetch-test-data:  ## Download any data needed by the test suite
 	uv run ref datasets fetch-sample-data
-	uv run python ./scripts/fetch-ilamb-data.py test.txt
+	uv run ref datasets fetch-data --registry ilamb-test
 
 .PHONY: fetch-ref-data
 fetch-ref-data:  ## Download reference data needed by providers and (temporarily) not in obs4mips
-	uv run python ./scripts/fetch-ilamb-data.py ilamb.txt
-	uv run python ./scripts/fetch-ilamb-data.py iomb.txt
+	uv run ref datasets fetch-data --registry ilamb
+	uv run ref datasets fetch-data --registry iomb
 
 .PHONY: update-sample-data-registry
 update-sample-data-registry:  ## Update the sample data registry
-	curl --output packages/ref/src/cmip_ref/datasets/sample_data.txt https://raw.githubusercontent.com/Climate-REF/ref-sample-data/refs/heads/main/registry.txt
+	curl --output packages/ref/src/cmip_ref/dataset_registry/sample_data.txt https://raw.githubusercontent.com/Climate-REF/ref-sample-data/refs/heads/main/registry.txt

README.md

@@ -73,23 +73,22 @@ The REF is designed to enable Modelling Centers to quickly evaluate their data a
 The data under test here may not be published to ESGF yet,
 but the REF can still be used to evaluate it.
 
-For the tutorials and test suite,
-we maintain a set of test data that can be used to evaluate the REF.
-These datasets can be fetched using
+The REF requires some reference data to be available to run the metrics. Some of the reference datasets needed by the REF are available on ESGF yet. The following command will download the reference datasets needed by the REF and store them in a local directory (`datasets/obs4ref`) as well as some sample CMIP6 datasets that we used in our test suite:
 
 ```bash
-ref datasets fetch-sample-data
+ref datasets fetch-data --registry obs4ref --output-dir datasets/obs4ref
+ref datasets fetch-data --registry sample-data --output-dir datasets/sample-data
 ```
 
 These datasets can then be ingested into the REF and the metrics solved using:
 
 ```bash
-uv run ref datasets ingest --source-type cmip6 ./tests/test-data/sample-data/CMIP6/
-uv run ref datasets ingest --source-type obs4mips ./tests/test-data/sample-data/obs4MIPs/
+uv run ref datasets ingest --source-type cmip6 datasets/sample-data/CMIP6/
+uv run ref datasets ingest --source-type obs4mips datasets/obs4ref
 ref solve
 ```
 
-The executed metrics can then be viewed using the `ref executions list-groups` command.
+The executed metrics can then be viewed using the `ref executions list-groups` and `ref executions inspect` commands.
 
 ### As a devops engineer
 

changelog/227.feature.md

@@ -0,0 +1,3 @@
+Enabled metric providers to register registries of datasets for download.
+This unifies the fetching of datasets across the REF via the `ref datasets fetch-data` CLI command.
+Added registries for the datasets that haven't been published to obs4MIPs yet (`obs4REF`) as well as PMP annual cycle datasets.

packages/ref-core/src/cmip_ref_core/dataset_registry.py

@@ -0,0 +1,149 @@
+"""
+Data registries for non-published reference data
+
+These data are placeholders until these data have been added to obs4MIPs.
+The AR7 FT REF requires that reference datasets are openly licensed before it is included
+in any published data catalogs.
+"""
+
+import importlib.resources
+import os
+import pathlib
+import shutil
+
+import pooch
+from loguru import logger
+
+
+def fetch_all_files(registry: pooch.Pooch, output_dir: pathlib.Path | None, symlink: bool = False) -> None:
+    """
+    Fetch all files associated with a pooch registry and write them to an output directory.
+
+    Pooch fetches, caches and validates the downloaded files.
+    Subsequent calls to this function will not refetch any previously downloaded files.
+
+    Parameters
+    ----------
+    registry
+        Pooch directory containing a set of files that should be fetched.
+    output_dir
+        The root directory to write the files to.
+
+        The directory will be created if it doesn't exist,
+        and matching files will be overwritten.
+
+        If no directory is provided, the files will be fetched from the remote server,
+        but not copied anywhere.
+    symlink
+        If True, symlink all files to this directory.
+        Otherwise, perform a copy.
+    """
+    if output_dir:
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+    for key in registry.registry.keys():
+        fetch_file = registry.fetch(key)
+
+        if output_dir is None:
+            # Just warm the cache and move onto the next file
+            continue
+
+        linked_file = output_dir / key
+        linked_file.parent.mkdir(parents=True, exist_ok=True)
+        if not linked_file.exists():  # pragma: no cover
+            if symlink:
+                logger.info(f"Linking {key} to {linked_file}")
+
+                os.symlink(fetch_file, linked_file)
+            else:
+                logger.info(f"Copying {key} to {linked_file}")
+                shutil.copy(fetch_file, linked_file)
+        else:
+            logger.info(f"File {linked_file} already exists. Skipping.")
+
+
+class DatasetRegistryManager:
+    """
+    A collection of reference datasets registries
+
+    The REF requires additional reference datasets
+    in addition to obs4MIPs data which can be downloaded via ESGF.
+    Each provider may have different sets of reference data that are needed.
+    These are provider-specific datasets are datasets not yet available in obs4MIPs,
+    or are post-processed from obs4MIPs.
+
+    A dataset registry consists of a file that contains a list of files and checksums,
+    in combination with a base URL that is used to fetch the files.
+    [Pooch](https://www.fatiando.org/pooch/latest/) is used within the DataRegistry
+    to manage the caching, downloading and validation of the files.
+
+    All datasets that are registered here are expected to be openly licensed and freely available.
+    """
+
+    def __init__(self) -> None:
+        self._registries: dict[str, pooch.Pooch] = {}
+
+    def __getitem__(self, item: str) -> pooch.Pooch:
+        """
+        Get a registry by name
+        """
+        return self._registries[item]
+
+    def keys(self) -> list[str]:
+        """
+        Get the list of registry names
+        """
+        return list(self._registries.keys())
+
+    def register(  # noqa: PLR0913
+        self,
+        name: str,
+        base_url: str,
+        package: str,
+        resource: str,
+        cache_name: str | None = None,
+        version: str | None = None,
+    ) -> None:
+        """
+        Register a new dataset registry
+
+        This will create a new Pooch registry and add it to the list of registries.
+        This is typically used by a provider to register a new collections of datasets at runtime.
+
+        Parameters
+        ----------
+        name
+            Name of the registry
+
+            This is used to identify the registry
+        base_url
+            Commmon URL prefix for the files
+        package
+            Name of the package containing the registry resource.
+        resource
+            Name of the resource in the package that contains a list of files and checksums.
+
+            This must be formatted in a way that is expected by pooch.
+        version
+            The version of the data.
+
+            Changing the version will invalidate the cache and force a re-download of the data.
+        cache_name
+            Name to use to generate the cache directory.
+
+            This defaults to the value of `name` if not provided.
+        """
+        if cache_name is None:
+            cache_name = "ref"
+
+        registry = pooch.create(
+            path=pooch.os_cache(cache_name),
+            base_url=base_url,
+            version=version,
+            env="REF_METRICS_DATA_DIR",
+        )
+        registry.load_registry(str(importlib.resources.files(package) / resource))
+        self._registries[name] = registry
+
+
+dataset_registry_manager = DatasetRegistryManager()

packages/ref-core/src/cmip_ref_core/dataset_registry/__init__.py

@@ -54,6 +54,7 @@ def capture_logging() -> None:
     logger.disable("matplotlib.ticker")
     logger.disable("matplotlib.font_manager")
     logger.disable("pyproj.transformer")
+    logger.disable("pint.facets.plain.registry")
 
 
 def add_log_handler(**kwargs: Any) -> None:

packages/ref-core/src/cmip_ref_core/dataset_registry/pmp_reference.txt

@@ -16,6 +16,12 @@ def test_load_from_file(datadir):
     assert len(cv.dimensions)
 
 
+def test_load_from_file_failed(tmp_path):
+    (tmp_path / "cv_sample.yaml").touch()
+    with pytest.raises(AttributeError, match="'NoneType' object has no attribute 'keys'"):
+        CV.load_from_file(tmp_path / "cv_sample.yaml")
+
+
 def test_validate(cv, cmec_metric):
     cv.validate_metrics(cmec_metric)