chore: align REF configuration and migrate to SemVer #23
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Packaging | |
| on: | |
| pull_request: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - "main" | |
| tags: | |
| - "v*" | |
| permissions: | |
| contents: read | |
| packages: write | |
| attestations: write | |
| id-token: write | |
| jobs: | |
| helm: | |
| name: Helm Chart | |
| runs-on: ubuntu-latest | |
| permissions: | |
| packages: write | |
| outputs: | |
| generated-semver: ${{ steps.semantic-version.outputs.generated-semver }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| - name: Install yq | |
| run: | | |
| pip install yq | |
| - name: Generate SemVer | |
| id: semantic-version | |
| run: | | |
| CHART_VERSION=$(yq -r '.version' helm/Chart.yaml) | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| LOCAL_SEGMENT=+pr-${{ github.event.pull_request.number }} | |
| elif [ "${{ github.ref_type }}" = "tag" ]; then | |
| LOCAL_SEGMENT="" | |
| else | |
| SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7) | |
| LOCAL_SEGMENT=+${SHORT_SHA} | |
| fi | |
| GENERATED_VERSION=${CHART_VERSION}${LOCAL_SEGMENT} | |
| yq -Y -i ".version = \"$GENERATED_VERSION\"" helm/Chart.yaml | |
| echo "generated-semver=$GENERATED_VERSION" >> $GITHUB_OUTPUT | |
| - name: Chart | Push | |
| uses: appany/helm-oci-chart-releaser@v0.5.0 | |
| with: | |
| name: climate-ref-aft | |
| repository: climate-ref/charts | |
| tag: ${{ steps.semantic-version.outputs.generated-semver }} | |
| path: helm | |
| registry: ghcr.io | |
| registry_username: ${{ github.actor }} | |
| registry_password: ${{ secrets.GITHUB_TOKEN }} | |
| update_dependencies: 'true' | |
| test-minimal: | |
| name: Test Minimal Deployment | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| needs: [helm] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Start minikube | |
| uses: medyagh/setup-minikube@latest | |
| - name: Set up Helm | |
| uses: azure/setup-helm@v4.3.1 | |
| - name: Install Chart (minimal) | |
| run: | | |
| helm install test oci://ghcr.io/climate-ref/charts/climate-ref-aft \ | |
| --version=${{ needs.helm.outputs.generated-semver }} \ | |
| -f helm/ci/minimal-values.yaml | |
| - name: Wait for pods | |
| run: | | |
| echo "Waiting for deployments to be ready..." | |
| kubectl wait --for=condition=available deployment --all --timeout=120s | |
| kubectl get pods | |
| - name: Verify all pods healthy | |
| run: | | |
| # Check no pods are in CrashLoopBackOff or Error state | |
| UNHEALTHY=$(kubectl get pods --no-headers | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff' || true) | |
| if [ -n "$UNHEALTHY" ]; then | |
| echo "Unhealthy pods found:" | |
| echo "$UNHEALTHY" | |
| echo "" | |
| echo "=== Pod logs ===" | |
| for pod in $(echo "$UNHEALTHY" | awk '{print $1}'); do | |
| echo "--- $pod ---" | |
| kubectl logs "$pod" --tail=50 || true | |
| done | |
| exit 1 | |
| fi | |
| echo "All pods healthy." | |
| - name: Capture logs on failure | |
| if: failure() | |
| run: | | |
| kubectl get pods | |
| kubectl describe pods | |
| for pod in $(kubectl get pods --no-headers | awk '{print $1}'); do | |
| echo "=== $pod ===" | |
| kubectl logs "$pod" --tail=100 || true | |
| done | |
| test: | |
| name: Test Helm Deployment | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| needs: [helm] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Cache Sample Data (Restore) | |
| id: cache-sample-data-restore | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: ${{ github.workspace }}/cache/ref-config | |
| key: ${{ runner.os }}-sample-data | |
| enableCrossOsArchive: true | |
| - name: Set permissions for cached data | |
| run: | | |
| sudo install -d --owner=1000 --group=1000 ${GITHUB_WORKSPACE}/cache/ref-config | |
| - name: Start minikube | |
| uses: medyagh/setup-minikube@latest | |
| with: | |
| mount-path: '${{ github.workspace }}/cache/ref-config:/cache/ref-config' | |
| - name: Set up Helm | |
| uses: azure/setup-helm@v4.3.1 | |
| - name: Install Chart | |
| run: | | |
| helm install test oci://ghcr.io/climate-ref/charts/climate-ref-aft \ | |
| --version=${{ needs.helm.outputs.generated-semver }} \ | |
| -f helm/ci/gh-actions-values.yaml | |
| sleep 60 | |
| kubectl get pods | |
| echo "" | |
| kubectl describe pod -l app.kubernetes.io/component=pmp | |
| echo "" | |
| kubectl logs -l app.kubernetes.io/component=pmp | |
| - name: Run Migrations | |
| run: | | |
| kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref config list | |
| - name: Initialize Providers | |
| run: | | |
| # Imports ilamb3 which tries to create /home/app/.config/ilamb3 on import, no way to tell it to live somewhere else | |
| # First, set up all providers without fetching data (handles conda envs) | |
| kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref providers setup --skip-data --skip-validate | |
| # Fetch data for providers except esmvaltool (ERA5 data is too large for CI) | |
| kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref providers setup --provider pmp | |
| kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref providers setup --provider ilamb | |
| - name: Fetch Test Data | |
| run: | | |
| kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref datasets fetch-data --registry sample-data --output-directory /ref/sample-data | |
| - name: Cache Sample Data (Save) | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: ${{ github.workspace }}/cache/ref-config | |
| key: ${{ runner.os }}-sample-data | |
| - name: Ingest Test Data (CMIP6) | |
| run: | | |
| kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref -v datasets ingest --source-type cmip6 /ref/sample-data/CMIP6 | |
| - name: Ingest Test Data (obs4mips) | |
| run: | | |
| kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref -v datasets ingest --source-type obs4mips /ref/sample-data/obs4REF | |
| - name: Simple Solve | |
| run: | | |
| # Use a fixed set of fast diagnostics to keep CI times predictable | |
| kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref -v solve --timeout 720 --one-per-provider \ | |
| --diagnostic global-mean-timeseries \ | |
| --diagnostic annual-cycle \ | |
| --diagnostic gpp-wecann \ | |
| --provider esmvaltool \ | |
| --provider pmp \ | |
| --provider ilamb | |
| - name: Capture Worker Logs on Failure | |
| if: failure() | |
| run: | | |
| echo "=== PMP Worker Logs ===" | |
| kubectl logs -l app.kubernetes.io/component=pmp --tail=500 || true | |
| echo "" | |
| echo "=== ESMValTool Worker Logs ===" | |
| kubectl logs -l app.kubernetes.io/component=esmvaltool --tail=500 || true | |
| echo "" | |
| echo "=== ILAMB Worker Logs ===" | |
| kubectl logs -l app.kubernetes.io/component=ilamb --tail=500 || true | |
| echo "" | |
| echo "=== Orchestrator Worker Logs ===" | |
| kubectl logs -l app.kubernetes.io/component=orchestrator --tail=500 || true | |
| echo "" | |
| echo "=== Flower Logs ===" | |
| kubectl logs -l app.kubernetes.io/component=flower --tail=500 || true | |
| echo "" | |
| echo "=== Dragonfly Logs ===" | |
| kubectl logs -l app.kubernetes.io/name=dragonfly --tail=500 || true |