packaging.yaml

name: Packaging

on:
  pull_request:
  workflow_dispatch:
  push:
    branches:
      - "main"
    tags:
      - "v*"

# Cancel a PR's in-flight runs when it is pushed to again.
# Pushes to main and tags fall back to the run id, so a publish is never cancelled.
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: read
  packages: write
  attestations: write
  id-token: write

jobs:
  helm:
    name: Helm Chart
    runs-on: ubuntu-latest
    permissions:
      packages: write
    outputs:
      generated-semver: ${{ steps.semantic-version.outputs.generated-semver }}
    steps:
    - uses: actions/checkout@v7
    - uses: actions/setup-python@v7
    - name: Install yq
      run: |
        pip install yq
    - name: Generate SemVer
      id: semantic-version
      run: |
        CHART_VERSION=$(yq -r '.version' helm/Chart.yaml)
        if [ "${{ github.event_name }}" = "pull_request" ]; then
          LOCAL_SEGMENT=+pr-${{ github.event.pull_request.number }}
        elif [ "${{ github.ref_type }}" = "tag" ]; then
          LOCAL_SEGMENT=""
        else
          SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
          LOCAL_SEGMENT=+${SHORT_SHA}
        fi
        GENERATED_VERSION=${CHART_VERSION}${LOCAL_SEGMENT}
        yq -Y -i ".version = \"$GENERATED_VERSION\"" helm/Chart.yaml
        echo "generated-semver=$GENERATED_VERSION" >> $GITHUB_OUTPUT
    - name: Chart | Push
      uses: appany/helm-oci-chart-releaser@v0.5.0
      with:
        name: climate-ref-aft
        repository: climate-ref/charts
        tag: ${{ steps.semantic-version.outputs.generated-semver }}
        path: helm
        registry: ghcr.io
        registry_username: ${{ github.actor }}
        registry_password: ${{ secrets.GITHUB_TOKEN }}
        update_dependencies: 'true'

  test-minimal:
    name: Test Minimal Deployment
    runs-on: ubuntu-latest
    timeout-minutes: 10
    needs: [helm]
    steps:
    - uses: actions/checkout@v7
    - name: Start minikube
      uses: medyagh/setup-minikube@latest
    - name: Set up Helm
      uses: azure/setup-helm@v5.0.1
    - name: Install Chart (minimal)
      run: |
        helm install test oci://ghcr.io/climate-ref/charts/climate-ref-aft \
          --version=${{ needs.helm.outputs.generated-semver }} \
          -f helm/ci/minimal-values.yaml
    - name: Wait for pods
      run: |
        echo "Waiting for deployments to be ready..."
        # Init containers hold every worker until the broker hostname resolves,
        # and DNS can take a couple of minutes to come up on a fresh minikube.
        kubectl wait --for=condition=available deployment --all --timeout=300s
        kubectl get pods
    - name: Verify all pods healthy
      run: |
        # Check no pods are in CrashLoopBackOff or Error state
        UNHEALTHY=$(kubectl get pods --no-headers | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff' || true)
        if [ -n "$UNHEALTHY" ]; then
          echo "Unhealthy pods found:"
          echo "$UNHEALTHY"
          echo ""
          echo "=== Pod logs ==="
          for pod in $(echo "$UNHEALTHY" | awk '{print $1}'); do
            echo "--- $pod ---"
            kubectl logs "$pod" --tail=50 || true
          done
          exit 1
        fi
        echo "All pods healthy."
    - name: Capture logs on failure
      if: failure()
      run: |
        kubectl get pods
        kubectl describe pods
        for pod in $(kubectl get pods --no-headers | awk '{print $1}'); do
          echo "=== $pod ==="
          kubectl logs "$pod" --tail=100 || true
        done

  test:
    name: Test Helm Deployment
    runs-on: ubuntu-latest
    timeout-minutes: 30
    needs: [helm]
    steps:
    - uses: actions/checkout@v7
    - name: Cache Sample Data (Restore)
      id: cache-sample-data-restore
      uses: actions/cache/restore@v6
      with:
        path: ${{ github.workspace }}/cache/ref-config
        key: ${{ runner.os }}-sample-data
        enableCrossOsArchive: true
    - name: Set permissions for cached data
      run: |
        # The pods run as uid 1000, but the cache action restores files owned by the runner user,
        # so everything the previous run cached (e.g. /ref/log) must be handed back
        # before the migrate hook writes to it.
        sudo install -d --owner=1000 --group=1000 ${GITHUB_WORKSPACE}/cache/ref-config
        sudo chown -R 1000:1000 ${GITHUB_WORKSPACE}/cache/ref-config
    - name: Start minikube
      uses: medyagh/setup-minikube@latest
      with:
        cpus: max
        memory: 12g
        mount-path: '${{ github.workspace }}/cache/ref-config:/cache/ref-config'
    - name: Set up Helm
      uses: azure/setup-helm@v5.0.1
    - name: Install Chart
      run: |
        helm install test oci://ghcr.io/climate-ref/charts/climate-ref-aft \
          --version=${{ needs.helm.outputs.generated-semver }} \
          -f helm/ci/gh-actions-values.yaml

        # Only the orchestrator is waited on.
        # The api pod is expected to be unhealthy until providers are set up, so `deployment --all` would block.
        kubectl wait deployment/test-climate-ref-aft-orchestrator \
          --for=condition=available --timeout=600s

        kubectl get pods || true
        echo ""
        kubectl describe pod -l app.kubernetes.io/component=pmp || true
        echo ""
        kubectl logs -l app.kubernetes.io/component=pmp || true
    - name: Run Migrations
      run: |
        kubectl exec deployment/test-climate-ref-aft-orchestrator -- uv run ref db migrate
    - name: Initialize Providers
      run: |
        # First, set up all providers without fetching data (handles conda envs)
        kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref providers setup --skip-data --skip-validate
        # Fetch data for providers except esmvaltool (ERA5 data is too large for CI)
        kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref providers setup --provider pmp
        kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref providers setup --provider ilamb
    - name: Fetch Test Data
      run: |
        kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref datasets fetch-data --registry sample-data --output-directory /ref/sample-data

    - name: Make cached data readable for the save
      run: |
        # The pods write as uid 1000 and some files land mode 600,
        # which the runner user cannot read.
        # tar then silently drops them from the archive and the cache goes hollow.
        sudo chmod -R a+rX ${GITHUB_WORKSPACE}/cache/ref-config

    - name: Cache Sample Data (Save)
      uses: actions/cache/save@v6
      with:
        path: ${{ github.workspace }}/cache/ref-config
        key: ${{ runner.os }}-sample-data

    - name: Ingest Test Data (CMIP6)
      run: |
        kubectl exec deployment/test-climate-ref-aft-orchestrator -- \
          ref -v datasets ingest --source-type cmip6 /ref/sample-data/CMIP6
    - name: Ingest Test Data (obs4mips)
      run: |
        kubectl exec deployment/test-climate-ref-aft-orchestrator -- \
          ref -v datasets ingest --source-type obs4mips /ref/sample-data/obs4REF
    - name: Simple Solve
      run: |
        # Use a fixed set of fast diagnostics to keep CI times predictable
        kubectl exec deployment/test-climate-ref-aft-orchestrator -- \
          ref -v solve --timeout 720 --one-per-provider \
          --diagnostic global-mean-timeseries \
          --diagnostic annual-cycle \
          --diagnostic gpp-wecann \
          --provider esmvaltool \
          --provider pmp \
          --provider ilamb
    - name: Validate API can read compute results
      shell: bash
      run: |
        set -euo pipefail
        # ref-app eagerly imports providers from /ref/software at startup.
        # Provider setup happens after the api Deployment is created,
        # so the initial api pod may have crashed before /ref/software was populated.
        # Force a rollout so a fresh pod starts against the now-populated /ref.
        kubectl rollout restart deployment/test-climate-ref-aft-api
        kubectl rollout status  deployment/test-climate-ref-aft-api --timeout=300s

        # Run scripts/lib/api_check.py via the orchestrator pod to check the API responses
        API_BASE='http://test-climate-ref-aft-api/api/v1'
        check() {
          kubectl exec -i deployment/test-climate-ref-aft-orchestrator -- \
            python3 - "${API_BASE}$1" "${2:-0}" \
            < scripts/lib/api_check.py
        }

        check /utils/health-check/
        check /cmip7-aft-diagnostics/ 1
        check /executions/ 1
    - name: Capture Migrate Logs on Failure
      if: failure()
      # The migrate hook runs before any deployment exists,
      # so a pre-install failure leaves only its pods to inspect.
      run: |
        kubectl describe job/test-climate-ref-aft-migrate || true
        kubectl logs -l job-name=test-climate-ref-aft-migrate --tail=200 || true
    - name: Capture API logs on failure
      if: failure()
      run: |
        echo "=== API Logs ==="
        kubectl logs -l app.kubernetes.io/component=api --tail=200 --all-containers=true || true
        echo ""
        echo "=== API describe ==="
        kubectl describe deployment/test-climate-ref-aft-api || true
    - name: Capture Broker State on Failure
      if: failure()
      # The recurring failure mode is a task submitted to a queue and never traced by its worker.
      run: |
        kubectl exec -i deployment/test-climate-ref-aft-orchestrator -- \
          uv run python - < scripts/lib/broker_state.py || true
    - name: Capture Worker Logs on Failure
      if: failure()
      run: |
        echo "=== PMP Worker Logs ==="
        kubectl logs -l app.kubernetes.io/component=pmp --tail=500 || true
        echo ""
        echo "=== ESMValTool Worker Logs ==="
        kubectl logs -l app.kubernetes.io/component=esmvaltool --tail=500 || true
        echo ""
        # The worker prints nothing while a diagnostic runs.
        # ESMValTool logs to files under the execution's scratch directory,
        # so these tails show where a hung execution stalled.
        echo "=== ESMValTool execution logs ==="
        kubectl exec deployment/test-climate-ref-aft-esmvaltool -- sh -c \
          'find /ref/scratch/esmvaltool -name "main_log.txt" | while read -r f; do echo "--- $f"; tail -n 100 "$f"; done' || true
        echo ""
        echo "=== ILAMB Worker Logs ==="
        kubectl logs -l app.kubernetes.io/component=ilamb --tail=500 || true
        echo ""
        echo "=== Orchestrator Worker Logs ==="
        kubectl logs -l app.kubernetes.io/component=orchestrator --tail=500 || true
        echo ""
        echo "=== Flower Logs ==="
        kubectl logs -l app.kubernetes.io/component=flower --tail=500 || true
        echo ""
        echo "=== Dragonfly Logs ==="
        kubectl logs -l app.kubernetes.io/name=dragonfly --tail=500 || true