feat: Build and publish Helm chart in CI

.github/actions/build-container/action.yml

@@ -26,13 +26,12 @@ runs:
         cache-to: type=gha,mode=max
         context: .
         file: ${{ inputs.dockerfile }}
-        push: ${{ ! github.event.pull_request.head.repo.fork }}
+        push: true
         sbom: true
         tags: ${{ steps.metadata.outputs.tags }}
         labels: ${{ steps.metadata.outputs.labels }}
     - name: Attest to REF image
       uses: actions/attest-build-provenance@v2
-      if: ${{ ! github.event.pull_request.head.repo.fork }}
       with:
         subject-name: ghcr.io/${{ github.repository_owner }}/${{ inputs.container-name }}
         subject-digest: ${{ steps.push.outputs.digest }}

.github/workflows/bump.yaml

@@ -20,7 +20,7 @@ on:
 jobs:
   bump_version:
     name: "Bump version and create changelog"
-    if: "!startsWith(github.event.head_commit.message, 'bump:')"
+    if: ! startsWith(github.event.head_commit.message, 'bump:')
     runs-on: ubuntu-latest
     env:
       CI_COMMIT_EMAIL: "ci-runner@climate-ref.invalid"

.github/workflows/packaging.yaml

@@ -0,0 +1,154 @@
+name: Packaging
+
+on:
+  pull_request:
+  workflow_dispatch:
+  push:
+    branches:
+      - "main"
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+  packages: write
+  attestations: write
+  id-token: write
+
+jobs:
+  containers:
+    name: Containers
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: ./.github/actions/build-container
+        with:
+          container-name: climate-ref
+          dockerfile: packages/climate-ref/Dockerfile
+  helm:
+    name: Helm Chart
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    outputs:
+      generated-semver: ${{ steps.semantic-version.outputs.generated-semver }}
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+    - name: Install jq
+      run: |
+        sudo apt-get install --yes jq
+    - name: Install yq
+      run: |
+        pip install yq
+    - name: Generate SemVer
+      id: semantic-version
+      run: |
+        CHART_VERSION=$(yq -r '.version' helm/Chart.yaml)
+        LOCAL_SEGMENT=+pr-${{ github.event.pull_request.number }}
+        GENERATED_VERSION=${CHART_VERSION}${LOCAL_SEGMENT}
+        yq -Y -i ".version = \"$GENERATED_VERSION\"" helm/Chart.yaml
+        echo "generated-semver=$GENERATED_VERSION" >> $GITHUB_OUTPUT
+    - name: Chart | Push
+      uses: appany/helm-oci-chart-releaser@v0.5.0
+      with:
+        name: ref
+        repository: climate-ref/charts
+        tag: ${{ steps.semantic-version.outputs.generated-semver }}
+        path: helm
+        registry: ghcr.io
+        registry_username: ${{ github.actor }}
+        registry_password: ${{ secrets.GITHUB_TOKEN }}
+        update_dependencies: 'true'
+
+  test:
+    name: Test Helm Deployment
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    needs: [containers, helm]
+    steps:
+    - uses: actions/checkout@v4
+    - name: Cache Sample Data (Restore)
+      id: cache-sample-data-restore
+      uses: actions/cache/restore@v4
+      with:
+        path: ${{ github.workspace }}/cache/ref-config
+        key: ${{ runner.os }}-sample-data
+        enableCrossOsArchive: true
+    - name: Set permissions for cached data
+      run: |
+        sudo install -d --owner=1000 --group=1000 ${GITHUB_WORKSPACE}/cache/ref-config
+    - name: Start minikube
+      uses: medyagh/setup-minikube@latest
+      with:
+        mount-path: '${{ github.workspace }}/cache/ref-config:/cache/ref-config'
+    - name: Set up Helm
+      uses: azure/setup-helm@v4.3.0
+    - name: Install Chart
+      run: |
+        helm install test oci://ghcr.io/climate-ref/charts/ref \
+          --version=${{ needs.helm.outputs.generated-semver }} \
+          --set climate-ref.image.tag=pr-${{ github.event.pull_request.number }} \
+          -f helm/ci/gh-actions-values.yaml
+
+        sleep 60
+        kubectl get pods
+        echo ""
+        kubectl describe pod -l app.kubernetes.io/component=pmp
+        echo ""
+        kubectl logs -l app.kubernetes.io/component=pmp
+    - name: Run Migrations
+      run: |
+        kubectl exec deployment/test-ref-orchestrator -- ref config list
+    - name: Initialize Providers (pmp)
+      run: |
+        # Imports ilamb3 which tries to create /home/app/.config/ilamb3 on import, no way to tell it to live somewhere else
+        kubectl exec deployment/test-ref-pmp -- ref providers create-env --provider pmp
+    - name: Initialize Providers (emsvaltool)
+      run: |
+        kubectl exec deployment/test-ref-esmvaltool -- ref providers create-env --provider esmvaltool
+    - name: Fetch Test Data
+      run: |
+        kubectl exec deployment/test-ref-orchestrator -- ref datasets fetch-data --registry sample-data --output-directory /ref/sample-data
+
+    - name: Cache Sample Data (Save)
+      uses: actions/cache/save@v4
+      with:
+        path: ${{ github.workspace }}/cache/ref-config
+        key: ${{ runner.os }}-sample-data
+
+    - name: Ingest Test Data (CMIP6)
+      run: |
+        kubectl exec deployment/test-ref-orchestrator -- ref -v datasets ingest --source-type cmip6 /ref/sample-data/CMIP6
+    - name: Ingest Test Data (obs4mips)
+      run: |
+        kubectl exec deployment/test-ref-orchestrator -- ref -v datasets ingest --source-type obs4mips /ref/sample-data/obs4REF
+    - name: Simple Solve
+      run: |
+        kubectl exec deployment/test-ref-orchestrator -- ref -v solve --timeout 180 --one-per-provider
+    - name: Capture Worker Logs on Failure
+      if: failure()
+      run: |
+        echo "=== PMP Worker Logs ==="
+        kubectl logs -l app.kubernetes.io/component=pmp --tail=500 || true
+        echo ""
+        echo "=== ESMValTool Worker Logs ==="
+        kubectl logs -l app.kubernetes.io/component=esmvaltool --tail=500 || true
+        echo ""
+        echo "=== ILAMB Worker Logs ==="
+        kubectl logs -l app.kubernetes.io/component=ilamb --tail=500 || true
+        echo ""
+        echo "=== Example Worker Logs ==="
+        kubectl logs -l app.kubernetes.io/component=example --tail=500 || true
+        echo ""
+        echo "=== Orchestrator Worker Logs ==="
+        kubectl logs -l app.kubernetes.io/component=orchestrator --tail=500 || true
+        echo ""
+        echo "=== Dragonfly Logs ==="
+        kubectl logs -l app.kubernetes.io/name=dragonfly --tail=200 || true

.gitignore

@@ -164,3 +164,6 @@ packages/*/NOTICE
 
 # User-specific catalog paths (test data)
 *.paths.yaml
+
+# Helm dependencies
+helm/charts/*

.pre-commit-config.yaml

@@ -25,6 +25,7 @@ repos:
       - id: check-yaml
         args:
         - --unsafe
+        exclude: helm/templates
       - id: debug-statements
       - id: detect-private-key
       - id: end-of-file-fixer

changelog/492.feature.md

@@ -0,0 +1 @@
+Added Helm chart for Kubernetes deployment with automated CI/CD pipeline for building and publishing the chart to GitHub Container Registry, including deployment templates for provider workloads (ESMValTool, PMP, ILAMB), Flower monitoring UI, Dragonfly Redis dependency, and comprehensive integration testing in minikube.

docs/installation.md

@@ -7,7 +7,7 @@ This was a deliberate decision to make it easy to make the framework easy to ins
 Some of the [diagnostic providers](nutshell.md) require additional dependencies in order to run an execution.
 For these providers, the REF can automatically create a new Conda environment and install the required dependencies in this standalone environment.
 Each of these provider-specific environments are decoupled to allow for potentially incompatible dependencies.
-This uses a bundled version of the [micromamba](https://github.com/mamba-org/micromamba-releases)
+This uses a bundled version of [micromamba](https://github.com/mamba-org/micromamba-releases)
 to create and manage the environments so no additional dependencies are required.
 
 /// admonition | HPC Users
@@ -65,7 +65,7 @@ The conda-forge packages are a work in progress and are not yet available.
 See [#80](https://github.com/Climate-REF/climate-ref/issues/80) for more information.
 ///
 
-We intend on providing a recipe on conda-force. Once complete you can install `climate-ref` using `mamba` or `conda`:
+We intend on providing a recipe on conda-forge. Once complete you can install `climate-ref` using `mamba` or `conda`:
 
 ```bash
 mamba install -c conda-forge climate-ref
@@ -96,7 +96,7 @@ If you want to use the latest development version, you can build the Docker imag
 ```bash
 git clone https://github.com/Climate-REF/climate-ref.git
 cd climate-ref
-docker-compose build
+docker build
 ```
 
 If you require the full-stack of services recommended for a production deployment, you can use the `docker-compose` file to start the services.
@@ -125,7 +125,7 @@ See the [development documentation](development.md) for more information on how
 /// admonition | Windows support
     type: warning
 
-Window's doesn't support some of the packages required by the [diagnostic providers](nutshell.md),
+Windows doesn't support some of the packages required by the [diagnostic providers](nutshell.md),
 so we only support MacOS and Linux.
 Windows users are recommended to use [WSL](https://learn.microsoft.com/en-us/windows/wsl/install)
 or a Linux VM if they wish to use the REF.

helm/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

helm/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: dragonfly
+  repository: oci://ghcr.io/dragonflydb/dragonfly/helm
+  version: v1.33.1
+digest: sha256:822a6b075df8042b2829e2f52c0c1aff9a9e7e35ae34e87185e10fd473b29411
+generated: "2026-01-20T21:50:46.454792+11:00"

helm/Chart.yaml

@@ -0,0 +1,21 @@
+apiVersion: v2
+name: ref
+description: A Helm chart for Kubernetes
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.2.0"
+
+dependencies:
+- name: dragonfly
+  version: v1.33.1
+  repository: oci://ghcr.io/dragonflydb/dragonfly/helm
+  condition: dragonfly.enabled