Docs update

.bumpversion-helm.toml

@@ -1,24 +1,4 @@
 [tool.bumpversion]
-current_version = "0.9.1"
+current_version = "2026.02.0"
 commit = true
 tag = false
-
-[[tool.bumpversion.files]]
-filename = "helm/Chart.yaml"
-search = 'version: {current_version}'
-replace = 'version: {new_version}'
-
-[[tool.bumpversion.files]]
-filename = "helm/Chart.yaml"
-search = 'appVersion: "{current_version}"'
-replace = 'appVersion: "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "versions.toml"
-search = 'chart-version = "{current_version}"'
-replace = 'chart-version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "versions.toml"
-search = 'app-version = "{current_version}"'
-replace = 'app-version = "{new_version}"'

.github/workflows/nightly.yml

@@ -27,7 +27,7 @@ jobs:
           - name: "latest-core"
             description: "Test with latest core from main branch"
           - name: "latest-all"
-            description: "Test with latest everything from main branches"
+            description: "Test with latest core and available providers from main branches"
     name: ${{ matrix.name }}
     steps:
       - uses: actions/checkout@v4
@@ -69,4 +69,4 @@ jobs:
           "
 
       - name: Run provider discovery test
-        run: uv run pytest tests/integration/test_cmip7_aft.py::test_provider_discovery -v
+        run: uv run pytest tests/integration/test_cmip7_aft.py::test_provider_discovery -v --slow

.github/workflows/packaging.yaml

@@ -16,26 +16,9 @@ permissions:
   id-token: write
 
 jobs:
-  changes:
-    name: Detect Changes
-    runs-on: ubuntu-latest
-    outputs:
-      helm: ${{ steps.filter.outputs.helm }}
-    steps:
-      - uses: actions/checkout@v6
-      # - uses: dorny/paths-filter@v3
-      #   id: filter
-      #   with:
-      #     filters: |
-      #       helm:
-      #         - 'helm/**'
-
-
   helm:
     name: Helm Chart
     runs-on: ubuntu-latest
-    needs: [changes]
-    if: needs.changes.outputs.helm == 'true' || github.event_name != 'pull_request'
     permissions:
       packages: write
     outputs:
@@ -80,8 +63,7 @@ jobs:
     name: Test Helm Deployment
     runs-on: ubuntu-latest
     timeout-minutes: 30
-    if: github.event_name == 'pull_request' && needs.changes.outputs.helm == 'true'
-    needs: [changes, helm]
+    needs: [helm]
     steps:
     - uses: actions/checkout@v6
     - name: Cache Sample Data (Restore)
@@ -104,7 +86,6 @@ jobs:
       run: |
         helm install test oci://ghcr.io/climate-ref/charts/climate-ref-aft \
           --version=${{ needs.helm.outputs.generated-semver }} \
-          --set defaults.image.tag=pr-${{ github.event.pull_request.number }} \
           -f helm/ci/gh-actions-values.yaml
 
         sleep 60
@@ -146,7 +127,10 @@ jobs:
         kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref -v solve --timeout 720 --one-per-provider \
           --diagnostic global-mean-timeseries \
           --diagnostic annual-cycle \
-          --diagnostic gpp-wecann
+          --diagnostic gpp-wecann \
+          --provider esmvaltool \
+          --provider pmp \
+          --provider ilamb
     - name: Capture Worker Logs on Failure
       if: failure()
       run: |
@@ -159,9 +143,6 @@ jobs:
         echo "=== ILAMB Worker Logs ==="
         kubectl logs -l app.kubernetes.io/component=ilamb --tail=500 || true
         echo ""
-        echo "=== Example Worker Logs ==="
-        kubectl logs -l app.kubernetes.io/component=example --tail=500 || true
-        echo ""
         echo "=== Orchestrator Worker Logs ==="
         kubectl logs -l app.kubernetes.io/component=orchestrator --tail=500 || true
         echo ""

.gitignore

@@ -33,5 +33,10 @@ test-outputs/
 helm/charts/
 helm/Chart.lock
 
+# Docker
+docker/_data
 # uv
 uv.lock
+
+# Environment management
+.envrc

CHANGELOG.md

@@ -7,7 +7,7 @@ This project uses [Calendar Versioning](https://calver.org/) with the format `YY
 
 <!-- towncrier release notes start -->
 
-## [2026.02] - Unreleased
+## [2026.02.0] - Unreleased
 
 ### Fixed
 
@@ -28,11 +28,11 @@ This project uses [Calendar Versioning](https://calver.org/) with the format `YY
 
 ### Component Versions
 
-| Component | Version |
-|-----------|---------|
-| climate-ref-core | >=0.9.0 |
-| climate-ref | >=0.9.0 |
-| climate-ref-celery | >=0.9.0 |
-| climate-ref-esmvaltool | >=0.9.0 |
-| climate-ref-pmp | >=0.9.0 |
-| climate-ref-ilamb | >=0.9.0 |
+| Component              | Version |
+| ---------------------- | ------- |
+| climate-ref-core       | 0.11.1  |
+| climate-ref            | 0.11.1  |
+| climate-ref-celery     | 0.11.1  |
+| climate-ref-esmvaltool | 0.11.1  |
+| climate-ref-pmp        | 0.11.1  |
+| climate-ref-ilamb      | 0.11.1  |

README.md

@@ -6,26 +6,27 @@ evaluation pipeline built on [Climate REF](https://github.com/Climate-REF/climat
 
 ## What This Repository Contains
 
-| Directory | Purpose |
-|-----------|---------|
-| `helm/` | Helm chart for Kubernetes deployment |
-| `docker/` | Docker Compose for local development and testing |
-| `tests/` | Cross-provider integration tests |
-| `scripts/` | Smoke tests and deployment helpers |
-| `versions.toml` | Version manifest pinning all component versions |
+| Directory       | Purpose                                          |
+| --------------- | ------------------------------------------------ |
+| `helm/`         | Helm chart for Kubernetes deployment             |
+| `docker/`       | Docker Compose for local development and testing |
+| `tests/`        | Cross-provider integration tests                 |
+| `scripts/`      | Smoke tests and deployment helpers               |
+| `versions.toml` | Version manifest pinning all component versions  |
 
 ## Components
 
 The AFT deployment brings together independently versioned packages:
 
-| Package | Repository | Description |
-|---------|------------|-------------|
-| `climate-ref-core` | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | Core library with base classes and interfaces |
-| `climate-ref` | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | Main application, CLI, database, solver |
-| `climate-ref-celery` | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | Celery executor for distributed execution |
-| `climate-ref-esmvaltool` | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | ESMValTool diagnostic provider |
-| `climate-ref-pmp` | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | PCMDI Metrics Package diagnostic provider |
-| `climate-ref-ilamb` | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | ILAMB diagnostic provider |
+| Package                  | Repository                                                            | Description                                   |
+| ------------------------ | --------------------------------------------------------------------- | --------------------------------------------- |
+| `climate-ref-core`       | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | Core library with base classes and interfaces |
+| `climate-ref`            | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | Main application, CLI, database, solver       |
+| `climate-ref-celery`     | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | Celery executor for distributed execution     |
+| `climate-ref-esmvaltool` | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | ESMValTool diagnostic provider                |
+| `climate-ref-pmp`        | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | PCMDI Metrics Package diagnostic provider     |
+| `climate-ref-ilamb`      | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | ILAMB diagnostic provider                     |
+| `climate-ref-frontend`   | [Climate-REF/ref-ap](https://github.com/Climate-REF/ref-app)          | API + Frontend                                |
 
 Note: we intend to split the providers out into their own repositories in the coming weeks.
 
@@ -55,7 +56,7 @@ bash scripts/smoke-test.sh
 helm install ref ./helm -f helm/local-test-values.yaml
 
 # Or from the OCI registry
-helm install ref oci://ghcr.io/climate-ref/charts/climate-ref-aft --version 0.9.1
+helm install ref oci://ghcr.io/climate-ref/charts/climate-ref-aft --version 0.11.1
 ```
 
 ### Integration Tests

docker/docker-compose.local.yaml

@@ -0,0 +1,86 @@
+# Optional overrides for local development.
+#
+# Mount pre-fetched data or a local results directory into the stack
+# so you can skip the slow `ref providers` step.
+#
+# Usage:
+#   docker compose \
+#     -f docker/docker-compose.yaml \
+#     -f docker/docker-compose.local.yaml \
+#     up -d
+#
+# Customise the paths below to match your local machine.
+
+services:
+  # Mount local data into the workers so they can access it at /ref/data.
+  # The ref-init service is not overridden here; it still creates ref.toml
+  # on the shared named volume.
+  climate-ref:
+    volumes:
+      - ref:/ref
+      # Cached input data
+      - ${REF_DATASET_CACHE_DIR:-./_data/cache}:/ref/cache/climate_ref
+      # Conda environments
+      # This saves time if we are frequently rebuilding the stack
+      - ${REF_SOFTWARE_ROOT:-./_data/software}:/ref/software
+      # Pre-fetched input datasets (CMIP6, obs4REF, etc.)
+      # Example: ./local-data/sample-data contains CMIP6/ and obs4REF/ subdirs
+      - ${REF_DATA_DIR:-./_data/sample-data}:/data:ro
+      # Persist results across restarts
+      # - ${REF_RESULTS_DIR:-./_data/results}:/ref/results
+
+  ref-init:
+    volumes:
+      - ref:/ref
+      - ${REF_DATASET_CACHE_DIR:-./_data/cache}:/ref/cache/climate_ref
+      - ${REF_SOFTWARE_ROOT:-./_data/software}:/ref/software
+
+  ref-app:
+    volumes:
+      - ref:/ref
+      # Give the API read access to results
+      - ${REF_RESULTS_DIR:-./_data/results}:/ref/results:ro
+
+  climate-ref-esmvaltool:
+   volumes:
+      - ref:/ref
+      # Cached input data
+      - ${REF_DATASET_CACHE_DIR:-./_data/cache}:/ref/cache/climate_ref
+      # Conda environments
+      # This saves time if we are frequently rebuilding the stack
+      - ${REF_SOFTWARE_ROOT:-./_data/software}:/ref/software
+      # Pre-fetched input datasets (CMIP6, obs4REF, etc.)
+      # Example: ./local-data/sample-data contains CMIP6/ and obs4REF/ subdirs
+      - ${REF_DATA_DIR:-./_data/sample-data}:/data:ro
+      # Persist results across restarts
+      # - ${REF_RESULTS_DIR:-./_data/results}:/ref/results
+
+  climate-ref-pmp:
+   volumes:
+      - ref:/ref
+      # Cached input data
+      - ${REF_DATASET_CACHE_DIR:-./_data/cache}:/ref/cache/climate_ref
+      # Conda environments
+      # This saves time if we are frequently rebuilding the stack
+      - ${REF_SOFTWARE_ROOT:-./_data/software}:/ref/software
+      # Pre-fetched input datasets (CMIP6, obs4REF, etc.)
+      # Example: ./local-data/sample-data contains CMIP6/ and obs4REF/ subdirs
+      - ${REF_DATA_DIR:-./_data/sample-data}:/data:ro
+      # Persist results across restarts
+      # - ${REF_RESULTS_DIR:-./_data/results}:/ref/results
+
+  climate-ref-ilamb:
+   volumes:
+      - ref:/ref
+      # Cached input data
+      - ${REF_DATASET_CACHE_DIR:-./_data/cache}:/ref/cache/climate_ref
+      # Conda environments
+      # This saves time if we are frequently rebuilding the stack
+      - ${REF_SOFTWARE_ROOT:-./_data/software}:/ref/software
+      # Pre-fetched input datasets (CMIP6, obs4REF, etc.)
+      # Example: ./local-data/sample-data contains CMIP6/ and obs4REF/ subdirs
+      - ${REF_DATA_DIR:-./_data/sample-data}:/data:ro
+      # Persist results across restarts
+      # - ${REF_RESULTS_DIR:-./_data/results}:/ref/results
+volumes:
+  ref:

docker/docker-compose.yaml

@@ -6,6 +6,16 @@
 # Usage:
 #   docker compose -f docker/docker-compose.yaml up -d
 #   bash scripts/smoke-test.sh
+#
+# To mount local data directories (e.g. pre-fetched input data), layer
+# the local override file on top:
+#
+#   docker compose \
+#     -f docker/docker-compose.yaml \
+#     -f docker/docker-compose.local.yaml \
+#     up -d
+#
+# See docker/docker-compose.local.yaml for available mount options.
 
 name: climate-ref-aft
 services:
@@ -26,18 +36,52 @@ services:
     ports:
       - "5432:5432"
 
+  # One-shot service to initialise the shared /ref volume.
+  # Creates config, subdirectories, and populates providers.
+  ref-init:
+    image: ghcr.io/climate-ref/climate-ref:main
+    platform: linux/amd64
+    depends_on:
+      - postgres
+      - redis
+    environment:
+      - REF_DATABASE_URL=postgresql://postgres:example@postgres:5432/postgres
+      - REF_CONFIGURATION=/ref
+      - COLUMNS=150
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - |
+        # Create required subdirectories
+        mkdir -p /ref/log /ref/scratch /ref/results /ref/software
+        if [ ! -f /ref/ref.toml ]; then
+          echo "Initialising /ref/ref.toml..."
+          ref config list > /ref/ref.toml
+        fi
+        # Run setup commands to populate the database and providers
+        ref providers setup
+        echo "ref-init complete"
+    volumes:
+      - ref:/ref
+
   ref-app:
     image: ghcr.io/climate-ref/climate-ref-frontend:main
     platform: linux/amd64
     restart: always
     depends_on:
-      - postgres
-      - redis
+      ref-init:
+        condition: service_completed_successfully
+      postgres:
+        condition: service_started
+      redis:
+        condition: service_started
     environment:
       - REF_DATABASE_URL=postgresql://postgres:example@postgres:5432/postgres
       - ENVIRONMENT=local
       - SECRET_KEY=local-dev-secret
-      - REF_CONFIGURATION=/app/.ref
+      - REF_CONFIGURATION=/ref
+      - COLUMNS=150
+    volumes:
+      - ref:/ref
     ports:
       - "8000:8000"
 
@@ -66,6 +110,7 @@ services:
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
       - REF_EXECUTOR=climate_ref_celery.executor.CeleryExecutor
       - REF_DATABASE_URL=postgresql://postgres:example@postgres:5432/postgres
+      - COLUMNS=150
     command: celery start-worker --loglevel INFO
     volumes:
       - ref:/ref
@@ -86,11 +131,6 @@ services:
       service: climate-ref
     command: celery start-worker --provider ilamb --loglevel INFO
 
-  climate-ref-example:
-    extends:
-      service: climate-ref
-    command: celery start-worker --provider example --loglevel INFO
-
 volumes:
   ref:
   postgres-data:

helm/Chart.yaml

@@ -6,11 +6,11 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.9.1
+version: 0.11.1
 
 # This is the version number of the application being deployed.
 # This should match the chart version for coupled versioning.
-appVersion: "0.9.1"
+appVersion: "0.11.1"
 
 dependencies:
 - name: dragonfly