deploy-azd.yml

name: deploy-azd

on:
  workflow_dispatch:
    inputs:
      environment:
        description: Deployment environment name (for azd -e)
        required: true
        default: dev
      location:
        description: Azure location
        required: true
        default: centralus
      projectName:
        description: Project prefix used by naming convention
        required: true
        default: holidaypeakhub
      imageTag:
        description: Image tag to deploy
        required: true
        default: latest
      deployStatic:
        description: Provision static web app resources
        required: true
        type: boolean
        default: true
      uiOnly:
        description: Deploy only the UI using SWA token (skips provision and Azure login jobs)
        required: true
        type: boolean
        default: false
      apiBaseUrl:
        description: Optional API base URL for UI build override
        required: false
        default: ''
      seedDemoData:
        description: Run demo faker seed job after deployment (non-prod only)
        required: true
        type: boolean
        default: true
      deployChangedOnly:
        description: Legacy flag (changed-app-only deployment is always enforced)
        required: true
        type: boolean
        default: true
      forceApimSync:
        description: Force APIM sync and smoke checks even when no changed services are detected
        required: true
        type: boolean
        default: false

permissions:
  id-token: write
  contents: read

concurrency:
  group: deploy-azd-${{ inputs.environment }}
  cancel-in-progress: false

jobs:
  detect-changes:
    if: ${{ !inputs.uiOnly }}
    runs-on: ubuntu-latest
    outputs:
      crud_changed: ${{ steps.detect.outputs.crud_changed }}
      ui_changed: ${{ steps.detect.outputs.ui_changed }}
      agents_changed: ${{ steps.detect.outputs.agents_changed }}
      changed_agents_matrix: ${{ steps.detect.outputs.changed_agents_matrix }}
      changed_agent_services_csv: ${{ steps.detect.outputs.changed_agent_services_csv }}
      changed_aks_services_csv: ${{ steps.detect.outputs.changed_aks_services_csv }}
      lib_changed: ${{ steps.detect.outputs.lib_changed }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Detect changed services
        id: detect
        shell: bash
        run: |
          set -euo pipefail

          DEFAULT_BRANCH="${{ github.event.repository.default_branch }}"
          git fetch origin "$DEFAULT_BRANCH" --depth=1

          CHANGED_FILES=$(git diff --name-only "origin/$DEFAULT_BRANCH...HEAD")

          mapfile -t AGENT_SERVICES < <(python3 - <<'PY'
          import re

          with open('azure.yaml', encoding='utf-8') as f:
            lines = f.readlines()

          in_services = False
          current_service = None
          current_host = None
          services = []

          for raw in lines:
            line = raw.rstrip('\n')
            if not in_services:
              if re.match(r'^services:\s*$', line):
                in_services = True
              continue

            if re.match(r'^[^\s]', line):
              break

            service_match = re.match(r'^  ([a-z0-9\-]+):\s*$', line)
            if service_match:
              if current_service and current_host == 'aks' and current_service != 'crud-service':
                services.append(current_service)
              current_service = service_match.group(1)
              current_host = None
              continue

            host_match = re.match(r'^    host:\s*(\S+)\s*$', line)
            if host_match:
              current_host = host_match.group(1)

          if current_service and current_host == 'aks' and current_service != 'crud-service':
            services.append(current_service)

          for service in services:
            print(service)
          PY
          )

          CRUD_CHANGED=false
          if echo "$CHANGED_FILES" | grep -Eq '^apps/crud-service/'; then
            CRUD_CHANGED=true
          fi

          UI_CHANGED=false
          if echo "$CHANGED_FILES" | grep -Eq '^apps/ui/'; then
            UI_CHANGED=true
          fi

          LIB_CHANGED=false
          if echo "$CHANGED_FILES" | grep -Eq '^lib/'; then
            LIB_CHANGED=true
          fi

          AGENTS_CHANGED=false
          MATRIX='[]'
          CHANGED_AGENTS=()

          for service in "${AGENT_SERVICES[@]}"; do
            if echo "$CHANGED_FILES" | grep -Eq "^apps/$service/"; then
              CHANGED_AGENTS+=("$service")
            fi
          done

          # If lib/ changed, all agent services need rebuild
          if [ "$LIB_CHANGED" = true ]; then
            CHANGED_AGENTS=("${AGENT_SERVICES[@]}")
            CRUD_CHANGED=true
          fi

          if [ ${#CHANGED_AGENTS[@]} -gt 0 ]; then
            AGENTS_CHANGED=true
            MATRIX=$(printf '%s\n' "${CHANGED_AGENTS[@]}" | jq -R . | jq -s 'map({service: .})')
          fi

          CHANGED_AGENT_SERVICES_CSV=""
          if [ ${#CHANGED_AGENTS[@]} -gt 0 ]; then
            CHANGED_AGENT_SERVICES_CSV=$(IFS=,; echo "${CHANGED_AGENTS[*]}")
          fi

          CHANGED_AKS_SERVICES_CSV="$CHANGED_AGENT_SERVICES_CSV"
          if [ "$CRUD_CHANGED" = true ]; then
            if [ -n "$CHANGED_AKS_SERVICES_CSV" ]; then
              CHANGED_AKS_SERVICES_CSV="crud-service,$CHANGED_AKS_SERVICES_CSV"
            else
              CHANGED_AKS_SERVICES_CSV="crud-service"
            fi
          fi

          echo "crud_changed=$CRUD_CHANGED" >> "$GITHUB_OUTPUT"
          echo "ui_changed=$UI_CHANGED" >> "$GITHUB_OUTPUT"
          echo "agents_changed=$AGENTS_CHANGED" >> "$GITHUB_OUTPUT"
          echo "changed_agents_matrix=$MATRIX" >> "$GITHUB_OUTPUT"
          echo "changed_agent_services_csv=$CHANGED_AGENT_SERVICES_CSV" >> "$GITHUB_OUTPUT"
          echo "changed_aks_services_csv=$CHANGED_AKS_SERVICES_CSV" >> "$GITHUB_OUTPUT"
          echo "lib_changed=$LIB_CHANGED" >> "$GITHUB_OUTPUT"

  provision:
    if: ${{ !inputs.uiOnly }}
    needs: detect-changes
    runs-on: ubuntu-latest
    environment: ${{ inputs.environment }}
    outputs:
      AI_SERVICES_NAME: ${{ steps.outputs.outputs.AI_SERVICES_NAME }}
      PROJECT_ENDPOINT: ${{ steps.outputs.outputs.PROJECT_ENDPOINT }}
      PROJECT_NAME: ${{ steps.outputs.outputs.PROJECT_NAME }}
      APIM_GATEWAY_URL: ${{ steps.outputs.outputs.APIM_GATEWAY_URL }}
      COSMOS_ACCOUNT_URI: ${{ steps.outputs.outputs.COSMOS_ACCOUNT_URI }}
      COSMOS_DATABASE: ${{ steps.outputs.outputs.COSMOS_DATABASE }}
      KEY_VAULT_URI: ${{ steps.outputs.outputs.KEY_VAULT_URI }}
      REDIS_HOST: ${{ steps.outputs.outputs.REDIS_HOST }}
      EVENT_HUB_NAMESPACE: ${{ steps.outputs.outputs.EVENT_HUB_NAMESPACE }}
      APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ steps.outputs.outputs.APPLICATIONINSIGHTS_CONNECTION_STRING }}
      POSTGRES_HOST: ${{ steps.outputs.outputs.POSTGRES_HOST }}
      POSTGRES_USER: ${{ steps.outputs.outputs.POSTGRES_USER }}
      POSTGRES_ADMIN_USER: ${{ steps.outputs.outputs.POSTGRES_ADMIN_USER }}
      POSTGRES_AUTH_MODE: ${{ steps.outputs.outputs.POSTGRES_AUTH_MODE }}
      POSTGRES_DATABASE: ${{ steps.outputs.outputs.POSTGRES_DATABASE }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - uses: actions/checkout@v4

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Setup azd
        uses: Azure/setup-azd@v2

      - name: Authenticate azd (OIDC)
        run: |
          azd auth login \
            --client-id "${AZURE_CLIENT_ID}" \
            --tenant-id "${AZURE_TENANT_ID}" \
            --federated-credential-provider github \
            --no-prompt

      - name: Configure azd environment context
        run: |
          azd env set AZURE_SUBSCRIPTION_ID "${AZURE_SUBSCRIPTION_ID}" -e "${{ inputs.environment }}"
          azd env set AZURE_LOCATION "${{ inputs.location }}" -e "${{ inputs.environment }}"
          azd env set AZURE_ENV_NAME "${{ inputs.environment }}" -e "${{ inputs.environment }}"
          azd env set AZURE_RESOURCE_GROUP "${{ inputs.projectName }}-${{ inputs.environment }}-rg" -e "${{ inputs.environment }}"
          azd env set resourceGroupName "${{ inputs.projectName }}-${{ inputs.environment }}-rg" -e "${{ inputs.environment }}"
          azd env set AZURE_AKS_CLUSTER_NAME "${{ inputs.projectName }}-${{ inputs.environment }}-aks" -e "${{ inputs.environment }}"
          azd env set AKS_CLUSTER_NAME "${{ inputs.projectName }}-${{ inputs.environment }}-aks" -e "${{ inputs.environment }}"
          azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT "${{ inputs.projectName }}${{ inputs.environment }}acr.azurecr.io" -e "${{ inputs.environment }}"

      - name: Install kubelogin
        run: |
          for attempt in 1 2 3 4 5; do
            if az aks install-cli --kubelogin-version latest; then
              exit 0
            fi
            echo "kubelogin install failed on attempt ${attempt}; retrying..."
            sleep 15
          done
          echo "kubelogin install failed after retries"
          exit 1

      - name: Ensure hook scripts executable
        run: chmod +x ./.infra/azd/hooks/*.sh

      - name: Ensure AKS cluster is running
        run: |
          AKS_RG="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          AKS_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-aks"

          if az aks show --resource-group "$AKS_RG" --name "$AKS_NAME" >/dev/null 2>&1; then
            POWER_STATE=$(az aks show --resource-group "$AKS_RG" --name "$AKS_NAME" --query "powerState.code" -o tsv 2>/dev/null || true)
            if [ "$POWER_STATE" = "Stopped" ]; then
              echo "AKS cluster is stopped. Starting $AKS_NAME..."
              az aks start --resource-group "$AKS_RG" --name "$AKS_NAME"
              for _ in $(seq 1 40); do
                CURRENT_STATE=$(az aks show --resource-group "$AKS_RG" --name "$AKS_NAME" --query "powerState.code" -o tsv 2>/dev/null || true)
                if [ "$CURRENT_STATE" = "Running" ]; then
                  echo "AKS cluster is running."
                  break
                fi
                sleep 30
              done
            fi
          else
            echo "AKS cluster does not exist yet. Continuing."
          fi

      - name: Install kubelogin
        run: |
          for attempt in 1 2 3 4 5; do
            if az aks install-cli --kubelogin-version latest; then
              exit 0
            fi
            echo "kubelogin install failed on attempt ${attempt}; retrying..."
            sleep 15
          done
          echo "kubelogin install failed after retries"
          exit 1

      - name: Ensure hook scripts executable
        run: chmod +x ./.infra/azd/hooks/*.sh

      - name: Configure azd environment
        run: |
          azd env set AZURE_SUBSCRIPTION_ID "${AZURE_SUBSCRIPTION_ID}" -e "${{ inputs.environment }}"
          azd env set AZURE_LOCATION "${{ inputs.location }}" -e "${{ inputs.environment }}"
          azd env set AZURE_ENV_NAME "${{ inputs.environment }}" -e "${{ inputs.environment }}"
          azd env set AZURE_RESOURCE_GROUP "${{ inputs.projectName }}-${{ inputs.environment }}-rg" -e "${{ inputs.environment }}"
          azd env set resourceGroupName "${{ inputs.projectName }}-${{ inputs.environment }}-rg" -e "${{ inputs.environment }}"
          azd env set AZURE_AKS_CLUSTER_NAME "${{ inputs.projectName }}-${{ inputs.environment }}-aks" -e "${{ inputs.environment }}"
          azd env set AKS_CLUSTER_NAME "${{ inputs.projectName }}-${{ inputs.environment }}-aks" -e "${{ inputs.environment }}"
          azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT "${{ inputs.projectName }}${{ inputs.environment }}acr.azurecr.io" -e "${{ inputs.environment }}"

          azd env set deployShared true -e "${{ inputs.environment }}"
          azd env set deployStatic "${{ inputs.deployStatic }}" -e "${{ inputs.environment }}"
          azd env set environment "${{ inputs.environment }}" -e "${{ inputs.environment }}"
          azd env set location "${{ inputs.location }}" -e "${{ inputs.environment }}"
          azd env set projectName "${{ inputs.projectName }}" -e "${{ inputs.environment }}"

          azd env set IMAGE_PREFIX "ghcr.io/${{ github.repository_owner }}" -e "${{ inputs.environment }}"
          azd env set IMAGE_TAG "${{ inputs.imageTag }}" -e "${{ inputs.environment }}"
          azd env set K8S_NAMESPACE holiday-peak -e "${{ inputs.environment }}"
          azd env set KEDA_ENABLED false -e "${{ inputs.environment }}"

      - name: Ensure deploy principal has AKS RBAC Cluster Admin
        shell: bash
        run: |
          set -euo pipefail

          AKS_RG="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          DEPLOY_SP_OBJECT_ID=$(az ad sp show --id "${AZURE_CLIENT_ID}" --query id -o tsv)
          RG_ID=$(az group show --name "$AKS_RG" --query id -o tsv)

          EXISTING=$(az role assignment list \
            --scope "$RG_ID" \
            --assignee-object-id "$DEPLOY_SP_OBJECT_ID" \
            --query "[?roleDefinitionName=='Azure Kubernetes Service RBAC Cluster Admin'] | length(@)" -o tsv)

          if [ "$EXISTING" = "0" ]; then
            echo "Assigning Azure Kubernetes Service RBAC Cluster Admin to deploy principal at RG scope $AKS_RG."
            az role assignment create \
              --assignee-object-id "$DEPLOY_SP_OBJECT_ID" \
              --assignee-principal-type ServicePrincipal \
              --role "Azure Kubernetes Service RBAC Cluster Admin" \
              --scope "$RG_ID"
          else
            echo "Deploy principal already has Azure Kubernetes Service RBAC Cluster Admin at RG scope $AKS_RG."
          fi

          VERIFIED=$(az role assignment list \
            --scope "$RG_ID" \
            --assignee-object-id "$DEPLOY_SP_OBJECT_ID" \
            --query "[?roleDefinitionName=='Azure Kubernetes Service RBAC Cluster Admin'] | length(@)" -o tsv)

          if [ "$VERIFIED" = "0" ]; then
            echo "Failed to verify Azure Kubernetes Service RBAC Cluster Admin assignment for deploy principal at scope $AKS_RG."
            exit 1
          fi

      - name: Preflight drift remediation (non-prod)
        if: ${{ inputs.environment != 'prod' && inputs.environment != 'production' }}
        shell: bash
        run: |
          set -euo pipefail

          RG_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          KEY_VAULT_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-kv"
          POSTGRES_SERVER_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-postgres"
          DESIRED_LOCATION="$(echo "${{ inputs.location }}" | tr '[:upper:]' '[:lower:]')"

          echo "Running non-prod preflight drift remediation for environment '${{ inputs.environment }}'."

          # Remediate soft-deleted Key Vault name conflicts that can block azd provision.
          if az keyvault show --name "$KEY_VAULT_NAME" --resource-group "$RG_NAME" >/dev/null 2>&1; then
            CURRENT_KV_LOCATION=$(az keyvault show \
              --name "$KEY_VAULT_NAME" \
              --resource-group "$RG_NAME" \
              --query location -o tsv | tr '[:upper:]' '[:lower:]')

            if [ "$CURRENT_KV_LOCATION" != "$DESIRED_LOCATION" ]; then
              LOCATION_SUFFIX=$(echo "$DESIRED_LOCATION" | tr -cd '[:alnum:]' | cut -c1-4)
              KEY_VAULT_OVERRIDE=$(echo "${{ inputs.projectName }}-${{ inputs.environment }}-kv-${LOCATION_SUFFIX}" | tr '[:upper:]' '[:lower:]' | cut -c1-24)
              KEY_VAULT_OVERRIDE="${KEY_VAULT_OVERRIDE%-}"

              echo "Key Vault location mismatch detected for $KEY_VAULT_NAME ($CURRENT_KV_LOCATION vs $DESIRED_LOCATION)."
              echo "Setting keyVaultNameOverride to $KEY_VAULT_OVERRIDE for this deployment."
              azd env set keyVaultNameOverride "$KEY_VAULT_OVERRIDE" -e "${{ inputs.environment }}"
            else
              echo "Key Vault $KEY_VAULT_NAME already exists in $RG_NAME with matching location $CURRENT_KV_LOCATION."
            fi
          else
            DELETED_KV_LOCATION=$(az keyvault list-deleted \
              --query "[?name=='$KEY_VAULT_NAME'] | [0].properties.location" \
              -o tsv)

            if [ -n "$DELETED_KV_LOCATION" ]; then
              echo "Recovering soft-deleted Key Vault $KEY_VAULT_NAME."
              if az keyvault recover --name "$KEY_VAULT_NAME" >/dev/null 2>&1; then
                echo "Recovery initiated for $KEY_VAULT_NAME."
              else
                echo "Recovery failed for $KEY_VAULT_NAME. Attempting purge fallback from $DELETED_KV_LOCATION."
                az keyvault purge --name "$KEY_VAULT_NAME" --location "$DELETED_KV_LOCATION"
              fi

              for _ in $(seq 1 20); do
                if az keyvault show --name "$KEY_VAULT_NAME" --resource-group "$RG_NAME" >/dev/null 2>&1; then
                  echo "Key Vault $KEY_VAULT_NAME is now available in $RG_NAME."
                  break
                fi
                sleep 10
              done
            else
              echo "No soft-deleted Key Vault conflict found for $KEY_VAULT_NAME."
            fi
          fi

          # Start stopped PostgreSQL Flexible Server so provisioning can reconcile state.
          if az postgres flexible-server show --resource-group "$RG_NAME" --name "$POSTGRES_SERVER_NAME" >/dev/null 2>&1; then
            POSTGRES_STATE=$(az postgres flexible-server show \
              --resource-group "$RG_NAME" \
              --name "$POSTGRES_SERVER_NAME" \
              --query state -o tsv)

            if [ "$POSTGRES_STATE" = "Stopped" ]; then
              echo "PostgreSQL server $POSTGRES_SERVER_NAME is stopped. Starting..."
              az postgres flexible-server start --resource-group "$RG_NAME" --name "$POSTGRES_SERVER_NAME"

              POSTGRES_READY=false
              for _ in $(seq 1 40); do
                CURRENT_STATE=$(az postgres flexible-server show \
                  --resource-group "$RG_NAME" \
                  --name "$POSTGRES_SERVER_NAME" \
                  --query state -o tsv)
                if [ "$CURRENT_STATE" = "Ready" ]; then
                  echo "PostgreSQL server is ready."
                  POSTGRES_READY=true
                  break
                fi
                sleep 15
              done

              if [ "$POSTGRES_READY" != "true" ]; then
                echo "PostgreSQL server $POSTGRES_SERVER_NAME did not reach Ready state in time."
                exit 1
              fi
            else
              echo "PostgreSQL server $POSTGRES_SERVER_NAME state is $POSTGRES_STATE. No start needed."
            fi
          else
            echo "PostgreSQL server $POSTGRES_SERVER_NAME does not exist yet. Continuing."
          fi

      - name: Validate truth Event Hubs IaC declarations
        shell: bash
        run: |
          set -euo pipefail

          BICEP_FILE=".infra/modules/shared-infrastructure/shared-infrastructure.bicep"

          for hub in ingest-jobs enrichment-jobs export-jobs hitl-jobs; do
            if ! grep -q "'$hub'" "$BICEP_FILE"; then
              echo "Missing required truth Event Hub declaration in $BICEP_FILE: $hub" >&2
              exit 1
            fi
          done

          for group in ingestion-group enrichment-engine export-engine hitl-service; do
            if ! grep -q "'$group'" "$BICEP_FILE"; then
              echo "Missing required truth consumer group declaration in $BICEP_FILE: $group" >&2
              exit 1
            fi
          done

          echo "Truth Event Hub and consumer group declarations are present in IaC."

      - name: Provision infrastructure
        run: azd provision --no-prompt -e "${{ inputs.environment }}"

      - name: Ensure AKS identity can read Key Vault secrets
        run: |
          AKS_RG="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          AKS_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-aks"
          KEY_VAULT_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-kv"

          KUBELET_OBJECT_ID=$(az aks show \
            --resource-group "$AKS_RG" \
            --name "$AKS_NAME" \
            --query "identityProfile.kubeletidentity.objectId" -o tsv)

          KEY_VAULT_ID=$(az keyvault show \
            --resource-group "$AKS_RG" \
            --name "$KEY_VAULT_NAME" \
            --query id -o tsv)

          EXISTING=$(az role assignment list \
            --assignee-object-id "$KUBELET_OBJECT_ID" \
            --scope "$KEY_VAULT_ID" \
            --query "[?roleDefinitionName=='Key Vault Secrets User'] | length(@)" -o tsv)

          if [ "$EXISTING" = "0" ]; then
            az role assignment create \
              --assignee-object-id "$KUBELET_OBJECT_ID" \
              --assignee-principal-type ServicePrincipal \
              --role "Key Vault Secrets User" \
              --scope "$KEY_VAULT_ID"
          fi

      - name: Export provisioned outputs
        id: outputs
        run: |
          eval "$(azd env get-values -e '${{ inputs.environment }}' | sed 's/^/export /')"
          APIM_GATEWAY_URL_VALUE="${APIM_GATEWAY_URL:-${apimGatewayUrl:-}}"
          {
            echo "AI_SERVICES_NAME=${AI_SERVICES_NAME}"
            echo "PROJECT_ENDPOINT=${PROJECT_ENDPOINT}"
            echo "PROJECT_NAME=${PROJECT_NAME}"
            echo "APIM_GATEWAY_URL=${APIM_GATEWAY_URL_VALUE}"
            echo "COSMOS_ACCOUNT_URI=${COSMOS_ACCOUNT_URI}"
            echo "COSMOS_DATABASE=${COSMOS_DATABASE}"
            echo "KEY_VAULT_URI=${KEY_VAULT_URI}"
            echo "REDIS_HOST=${REDIS_HOST}"
            echo "EVENT_HUB_NAMESPACE=${EVENT_HUB_NAMESPACE}"
            echo "APPLICATIONINSIGHTS_CONNECTION_STRING=${APPLICATIONINSIGHTS_CONNECTION_STRING}"
            echo "POSTGRES_HOST=${POSTGRES_HOST}"
            echo "POSTGRES_USER=${POSTGRES_USER}"
            echo "POSTGRES_ADMIN_USER=${POSTGRES_ADMIN_USER}"
            echo "POSTGRES_AUTH_MODE=${POSTGRES_AUTH_MODE}"
            echo "POSTGRES_DATABASE=${POSTGRES_DATABASE}"
          } >> "$GITHUB_OUTPUT"

  deploy-foundry-models:
    runs-on: ubuntu-latest
    needs: provision
    environment: ${{ inputs.environment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - uses: actions/checkout@v4

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Deploy AI model deployments
        run: |
          bash .infra/azd/hooks/deploy-foundry-models.sh \
            "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            "${{ needs.provision.outputs.AI_SERVICES_NAME }}"
        env:
          AZURE_RESOURCE_GROUP: ${{ inputs.projectName }}-${{ inputs.environment }}-rg
          AI_SERVICES_NAME: ${{ needs.provision.outputs.AI_SERVICES_NAME }}

  deploy-crud:
    runs-on: ubuntu-latest
    needs:
      - provision
      - detect-changes
    if: ${{ !inputs.uiOnly && needs.detect-changes.outputs.crud_changed == 'true' }}
    environment: ${{ inputs.environment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - uses: actions/checkout@v4

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Setup azd
        uses: Azure/setup-azd@v2

      - name: Authenticate azd (OIDC)
        run: |
          azd auth login \
            --client-id "${AZURE_CLIENT_ID}" \
            --tenant-id "${AZURE_TENANT_ID}" \
            --federated-credential-provider github \
            --no-prompt

      - name: Configure azd environment context
        run: |
          azd env set AZURE_SUBSCRIPTION_ID "${AZURE_SUBSCRIPTION_ID}" -e "${{ inputs.environment }}"
          azd env set AZURE_LOCATION "${{ inputs.location }}" -e "${{ inputs.environment }}"
          azd env set AZURE_ENV_NAME "${{ inputs.environment }}" -e "${{ inputs.environment }}"
          azd env set AZURE_RESOURCE_GROUP "${{ inputs.projectName }}-${{ inputs.environment }}-rg" -e "${{ inputs.environment }}"
          azd env set resourceGroupName "${{ inputs.projectName }}-${{ inputs.environment }}-rg" -e "${{ inputs.environment }}"
          azd env set AZURE_AKS_CLUSTER_NAME "${{ inputs.projectName }}-${{ inputs.environment }}-aks" -e "${{ inputs.environment }}"
          azd env set AKS_CLUSTER_NAME "${{ inputs.projectName }}-${{ inputs.environment }}-aks" -e "${{ inputs.environment }}"
          azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT "${{ inputs.projectName }}${{ inputs.environment }}acr.azurecr.io" -e "${{ inputs.environment }}"

      - name: Install kubelogin
        run: |
          for attempt in 1 2 3 4 5; do
            if az aks install-cli --kubelogin-version latest; then
              exit 0
            fi
            echo "kubelogin install failed on attempt ${attempt}; retrying..."
            sleep 15
          done
          echo "kubelogin install failed after retries"
          exit 1

      - name: Ensure hook scripts executable
        run: chmod +x ./.infra/azd/hooks/*.sh

      - name: Configure azd environment context
        run: |
          azd env set AZURE_SUBSCRIPTION_ID "${AZURE_SUBSCRIPTION_ID}" -e "${{ inputs.environment }}"
          azd env set AZURE_LOCATION "${{ inputs.location }}" -e "${{ inputs.environment }}"
          azd env set AZURE_ENV_NAME "${{ inputs.environment }}" -e "${{ inputs.environment }}"
          azd env set AZURE_RESOURCE_GROUP "${{ inputs.projectName }}-${{ inputs.environment }}-rg" -e "${{ inputs.environment }}"
          azd env set resourceGroupName "${{ inputs.projectName }}-${{ inputs.environment }}-rg" -e "${{ inputs.environment }}"
          azd env set AZURE_AKS_CLUSTER_NAME "${{ inputs.projectName }}-${{ inputs.environment }}-aks" -e "${{ inputs.environment }}"
          azd env set AKS_CLUSTER_NAME "${{ inputs.projectName }}-${{ inputs.environment }}-aks" -e "${{ inputs.environment }}"
          azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT "${{ inputs.projectName }}${{ inputs.environment }}acr.azurecr.io" -e "${{ inputs.environment }}"

      - name: Get AKS credentials
        run: |
          az aks get-credentials \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-aks" \
            --overwrite-existing

      - name: Resolve AKS managed identity client ID
        run: |
          AKS_MI_CLIENT_ID=$(az aks show \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-aks" \
            --query "identityProfile.kubeletidentity.clientId" -o tsv)
          echo "WORKLOAD_AZURE_CLIENT_ID=${AKS_MI_CLIENT_ID}" >> "$GITHUB_ENV"

      - name: Resolve ingress class
        shell: bash
        run: |
          set -euo pipefail
          kubectl get ingressclass -o wide || true

          if [ -n "${INGRESS_CLASS_NAME:-}" ] && kubectl get ingressclass "${INGRESS_CLASS_NAME}" >/dev/null 2>&1; then
            echo "Using preconfigured ingress class: ${INGRESS_CLASS_NAME}"
            echo "INGRESS_CLASS_NAME=${INGRESS_CLASS_NAME}" >> "$GITHUB_ENV"
            exit 0
          fi

          for cls in webapprouting.kubernetes.azure.com nginx azure-application-gateway; do
            if kubectl get ingressclass "$cls" >/dev/null 2>&1; then
              echo "Using detected ingress class: $cls"
              echo "INGRESS_CLASS_NAME=$cls" >> "$GITHUB_ENV"
              exit 0
            fi
          done

          echo "No supported IngressClass found. Enable AKS Web App Routing or provide INGRESS_CLASS_NAME." >&2
          exit 1

      - name: Preflight ACR data-plane accessibility
        shell: bash
        run: |
          set -euo pipefail

          RG_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          ACR_NAME=$(echo "${{ inputs.projectName }}${{ inputs.environment }}acr" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]')

          if ! az acr show --resource-group "$RG_NAME" --name "$ACR_NAME" >/dev/null 2>&1; then
            echo "ACR preflight could not find registry '$ACR_NAME' in resource group '$RG_NAME'." >&2
            echo "Verify naming convention or set AZURE_CONTAINER_REGISTRY_ENDPOINT in azd env before deploy." >&2
            exit 1
          fi

          ACR_LOGIN_SERVER=$(az acr show \
            --resource-group "$RG_NAME" \
            --name "$ACR_NAME" \
            --query "loginServer" -o tsv)

          STATUS_CODE=$(curl -sS -o /tmp/acr-v2-probe.txt -w "%{http_code}" --max-time 25 "https://${ACR_LOGIN_SERVER}/v2/" || true)

          if [ "$STATUS_CODE" = "200" ] || [ "$STATUS_CODE" = "401" ]; then
            echo "ACR preflight passed: ${ACR_LOGIN_SERVER} is reachable from this runner (HTTP $STATUS_CODE)."
            exit 0
          fi

          RUNNER_IP=$(curl -sS --max-time 10 https://api.ipify.org || true)
          echo "ACR preflight failed: ${ACR_LOGIN_SERVER}/v2/ returned HTTP ${STATUS_CODE}." >&2
          if [ "$STATUS_CODE" = "403" ]; then
            echo "Likely cause: ACR firewall/network rules are blocking this GitHub-hosted runner." >&2
          fi
          if [ -n "$RUNNER_IP" ]; then
            echo "Runner egress IP detected: $RUNNER_IP" >&2
            echo "Action: allow this IP (or CIDR) on ACR '$ACR_NAME' or use a self-hosted runner in an allowed network." >&2
            echo "Example: az acr network-rule add --name '$ACR_NAME' --resource-group '$RG_NAME' --ip-address '$RUNNER_IP'" >&2
          else
            echo "Action: check ACR firewall/network rules for GitHub runner egress and retry." >&2
          fi
          exit 1

      - name: Deploy CRUD service
        timeout-minutes: 25
        shell: bash
        run: |
          set -euo pipefail
          max_attempts=4

          for attempt in $(seq 1 "$max_attempts"); do
            echo "Deploy attempt ${attempt}/${max_attempts}"
            if azd deploy --service crud-service --no-prompt -e "${{ inputs.environment }}"; then
              echo "CRUD deploy succeeded."
              exit 0
            fi

            echo "Attempt ${attempt} failed; collecting ingress diagnostics..."
            kubectl get ingressclass -o wide || true
            kubectl get ingress -n holiday-peak -o wide || true
            kubectl get pods -n app-routing-system -o wide || true
            kubectl get pods -n ingress-nginx -o wide || true

            if [ "$attempt" -eq "$max_attempts" ]; then
              echo "CRUD deploy failed after ${max_attempts} attempts." >&2
              exit 1
            fi

            backoff=$((attempt * 45))
            echo "Retrying after ${backoff}s..."
            sleep "$backoff"
          done
        env:
          INGRESS_CLASS_NAME: ${{ env.INGRESS_CLASS_NAME }}
          AZURE_CLIENT_ID: ${{ env.WORKLOAD_AZURE_CLIENT_ID }}
          AZURE_TENANT_ID: ${{ env.AZURE_TENANT_ID }}
          PROJECT_ENDPOINT: ${{ needs.provision.outputs.PROJECT_ENDPOINT }}
          PROJECT_NAME: ${{ needs.provision.outputs.PROJECT_NAME }}
          MODEL_DEPLOYMENT_NAME_FAST: gpt-5-nano
          MODEL_DEPLOYMENT_NAME_RICH: gpt-5-nano
          POSTGRES_AUTH_MODE: ${{ needs.provision.outputs.POSTGRES_AUTH_MODE }}
          COSMOS_ACCOUNT_URI: ${{ needs.provision.outputs.COSMOS_ACCOUNT_URI }}
          COSMOS_DATABASE: ${{ needs.provision.outputs.COSMOS_DATABASE }}
          REDIS_HOST: ${{ needs.provision.outputs.REDIS_HOST }}
          EVENT_HUB_NAMESPACE: ${{ needs.provision.outputs.EVENT_HUB_NAMESPACE }}
          KEY_VAULT_URI: ${{ needs.provision.outputs.KEY_VAULT_URI }}
          APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ needs.provision.outputs.APPLICATIONINSIGHTS_CONNECTION_STRING }}
          POSTGRES_HOST: ${{ needs.provision.outputs.POSTGRES_HOST }}
          POSTGRES_USER: ${{ needs.provision.outputs.POSTGRES_AUTH_MODE == 'password' && needs.provision.outputs.POSTGRES_ADMIN_USER || needs.provision.outputs.POSTGRES_USER }}
          POSTGRES_DATABASE: ${{ needs.provision.outputs.POSTGRES_DATABASE }}

      - name: Gate CRUD readiness (/ready)
        shell: bash
        run: |
          set -euo pipefail

          APIM_BASE="${{ needs.provision.outputs.APIM_GATEWAY_URL }}"
          APIM_BASE="${APIM_BASE%/}"
          APIM_READY_URL="${APIM_BASE}/api/ready"
          SERVICE_READY_URL="http://127.0.0.1:18080/ready"

          check_ready() {
            local url="$1"
            local label="$2"

            for attempt in $(seq 1 24); do
              STATUS_CODE=$(curl -sS -o /tmp/crud-ready.json -w "%{http_code}" "$url" || true)
              if [ "$STATUS_CODE" = "200" ]; then
                echo "[OK] ${label} ready on attempt ${attempt}: ${url}"
                return 0
              fi
              echo "Attempt ${attempt}/24: ${label} returned HTTP ${STATUS_CODE}"
              sleep 10
            done

            echo "[FAIL] ${label} readiness did not return HTTP 200: ${url}" >&2
            cat /tmp/crud-ready.json 2>/dev/null || true
            return 1
          }

          APIM_READY=false
          if [ -n "$APIM_BASE" ]; then
            if check_ready "$APIM_READY_URL" "APIM /api/ready"; then
              APIM_READY=true
            fi
          else
            echo "APIM gateway URL not provided by provision outputs; skipping APIM /api/ready check."
          fi

          kubectl port-forward svc/crud-service 18080:8000 -n holiday-peak >/tmp/crud-port-forward.log 2>&1 &
          PF_PID=$!
          trap 'kill "$PF_PID" >/dev/null 2>&1 || true' EXIT
          sleep 3

          SERVICE_READY=false
          if check_ready "$SERVICE_READY_URL" "service /ready"; then
            SERVICE_READY=true
          fi

          if [ "$APIM_READY" != "true" ] && [ "$SERVICE_READY" != "true" ]; then
            echo "CRUD readiness gate failed: both APIM and direct service /ready checks are unhealthy." >&2
            echo "Port-forward log:" >&2
            cat /tmp/crud-port-forward.log 2>/dev/null || true
            exit 1
          fi

  deploy-ui:
    runs-on: ubuntu-latest
    if: ${{ always() && inputs.deployStatic && !inputs.uiOnly && needs.detect-changes.outputs.ui_changed == 'true' && (needs.deploy-crud.result == 'success' || needs.deploy-crud.result == 'skipped') && (needs.deploy-agents.result == 'success' || needs.deploy-agents.result == 'skipped') && (needs.sync-apim.result == 'success' || needs.sync-apim.result == 'skipped') && (needs.ensure-foundry-agents.result == 'success' || needs.ensure-foundry-agents.result == 'skipped') && (needs.smoke-apim.result == 'success' || needs.smoke-apim.result == 'skipped') }}
    needs:
      - provision
      - detect-changes
      - deploy-crud
      - deploy-agents
      - sync-apim
      - ensure-foundry-agents
      - smoke-apim
    environment: ${{ inputs.environment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - uses: actions/checkout@v4

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Setup azd
        uses: Azure/setup-azd@v2

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'

      - name: Resolve API URL
        id: api
        shell: bash
        run: |
          set -euo pipefail

          RESOURCE_GROUP="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          EXPECTED_APIM_URL="${{ needs.provision.outputs.APIM_GATEWAY_URL }}"

          if [ -z "$EXPECTED_APIM_URL" ]; then
            echo "Provision output APIM_GATEWAY_URL is empty. Cannot safely propagate API URL to UI build." >&2
            exit 1
          fi

          APIM_URL=$(az apim show \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-apim" \
            --resource-group "$RESOURCE_GROUP" \
            --query "gatewayUrl" -o tsv)

          APIM_URL="$(echo "$APIM_URL" | xargs)"
          EXPECTED_APIM_URL="$(echo "$EXPECTED_APIM_URL" | xargs)"

          APIM_URL="${APIM_URL%/}"
          EXPECTED_APIM_URL="${EXPECTED_APIM_URL%/}"

          if [ -z "$APIM_URL" ]; then
            echo "Failed to resolve APIM gatewayUrl for resource group $RESOURCE_GROUP. Ensure APIM exists and is provisioned before deploying UI." >&2
            exit 1
          fi

          if [ "$APIM_URL" != "$EXPECTED_APIM_URL" ]; then
            echo "APIM URL drift detected between provision output and live APIM gateway URL." >&2
            echo "Expected: $EXPECTED_APIM_URL" >&2
            echo "Actual:   $APIM_URL" >&2
            exit 1
          fi

          echo "api_url=$EXPECTED_APIM_URL" >> "$GITHUB_OUTPUT"
          echo "crud_api_url=$EXPECTED_APIM_URL" >> "$GITHUB_OUTPUT"

      - name: Smoke test APIM API health before UI deploy
        shell: bash
        run: |
          set -euo pipefail

          API_BASE_URL="${{ steps.api.outputs.api_url }}"
          API_BASE_URL="${API_BASE_URL%/}"

          smoke_endpoint() {
            local url="$1"
            local label="$2"

            for attempt in $(seq 1 20); do
              STATUS_CODE=$(curl -sS -o /tmp/api-health.json -w "%{http_code}" "$url" || true)
              if [ "$STATUS_CODE" = "200" ]; then
                echo "[OK] $label: $url"
                return 0
              fi
              echo "Attempt $attempt/20: $label returned HTTP $STATUS_CODE"
              sleep 10
            done

            echo "[FAIL] $label: $url" >&2
            cat /tmp/api-health.json 2>/dev/null || true
            return 1
          }

          smoke_endpoint "${API_BASE_URL}/api/health" "crud-health"
          smoke_endpoint "${API_BASE_URL}/api/products?limit=1" "crud-products"
          smoke_endpoint "${API_BASE_URL}/api/categories" "crud-categories"

      - name: Resolve SWA deployment token
        id: swa
        shell: bash
        run: |
          set -euo pipefail

          RESOURCE_GROUP="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          EXPECTED_SWA_NAME="${{ inputs.projectName }}-ui-${{ inputs.environment }}"

          SWA_NAME=""
          if az staticwebapp show --name "$EXPECTED_SWA_NAME" --resource-group "$RESOURCE_GROUP" >/dev/null 2>&1; then
            SWA_NAME="$EXPECTED_SWA_NAME"
          else
            SWA_NAME=$(az staticwebapp list \
              --resource-group "$RESOURCE_GROUP" \
              --query "[?starts_with(defaultHostname, '${EXPECTED_SWA_NAME}.')].name | [0]" -o tsv)

            if [ -z "$SWA_NAME" ]; then
              COUNT=$(az staticwebapp list --resource-group "$RESOURCE_GROUP" --query "length(@)" -o tsv)
              if [ "$COUNT" = "1" ]; then
                SWA_NAME=$(az staticwebapp list --resource-group "$RESOURCE_GROUP" --query "[0].name" -o tsv)
              fi
            fi
          fi

          if [ -z "$SWA_NAME" ]; then
            echo "Could not deterministically resolve Static Web App in $RESOURCE_GROUP." >&2
            echo "Expected name: $EXPECTED_SWA_NAME" >&2
            echo "Action: align SWA naming with IaC convention or keep exactly one SWA in the resource group." >&2
            exit 1
          fi

          SWA_TOKEN=$(az staticwebapp secrets list \
            --name "$SWA_NAME" \
            --resource-group "$RESOURCE_GROUP" \
            --query "properties.apiKey" -o tsv)

          if [ -z "$SWA_TOKEN" ]; then
            echo "Failed to resolve deployment token for SWA $SWA_NAME"
            exit 1
          fi

          echo "::add-mask::$SWA_TOKEN"
          echo "swa_name=$SWA_NAME" >> "$GITHUB_OUTPUT"
          echo "swa_token=$SWA_TOKEN" >> "$GITHUB_OUTPUT"

      - name: Deploy UI via Static Web Apps action
        uses: Azure/static-web-apps-deploy@v1
        with:
          action: upload
          azure_static_web_apps_api_token: ${{ steps.swa.outputs.swa_token }}
          app_location: apps/ui
          output_location: ''
          skip_api_build: true
          app_build_command: yarn install --frozen-lockfile && yarn build
        env:
          NEXT_PUBLIC_API_URL: ${{ steps.api.outputs.api_url }}
          NEXT_PUBLIC_CRUD_API_URL: ${{ steps.api.outputs.crud_api_url }}
          NEXT_PUBLIC_ENTRA_CLIENT_ID: ${{ secrets.NEXT_PUBLIC_ENTRA_CLIENT_ID }}
          NEXT_PUBLIC_ENTRA_TENANT_ID: ${{ secrets.NEXT_PUBLIC_ENTRA_TENANT_ID }}

      - name: Smoke test UI host and API health after deploy
        shell: bash
        run: |
          set -euo pipefail

          RESOURCE_GROUP="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          SWA_NAME="${{ steps.swa.outputs.swa_name }}"
          API_BASE_URL="${{ steps.api.outputs.api_url }}"

          SWA_HOSTNAME=$(az staticwebapp show \
            --name "$SWA_NAME" \
            --resource-group "$RESOURCE_GROUP" \
            --query "defaultHostname" -o tsv)

          SWA_HOSTNAME="$(echo "$SWA_HOSTNAME" | xargs)"
          if [ -z "$SWA_HOSTNAME" ]; then
            echo "Could not resolve Static Web App hostname for $SWA_NAME" >&2
            exit 1
          fi

          UI_URL="https://$SWA_HOSTNAME"
          for attempt in $(seq 1 20); do
            STATUS_CODE=$(curl -sS -o /tmp/ui-homepage.html -w "%{http_code}" "$UI_URL" || true)
            if [ "$STATUS_CODE" = "200" ]; then
              echo "UI homepage is reachable: $UI_URL"
              break
            fi

            if [ "$attempt" -eq 20 ]; then
              echo "UI smoke test failed with HTTP $STATUS_CODE at $UI_URL" >&2
              exit 1
            fi
            sleep 10
          done

          API_HEALTH_URL="${API_BASE_URL%/}/api/health"
          STATUS_CODE=$(curl -sS -o /tmp/ui-api-health.json -w "%{http_code}" "$API_HEALTH_URL" || true)
          if [ "$STATUS_CODE" != "200" ]; then
            echo "API health check failed after UI deploy with HTTP $STATUS_CODE at $API_HEALTH_URL" >&2
            cat /tmp/ui-api-health.json 2>/dev/null || true
            exit 1
          fi

          PRODUCTS_URL="${API_BASE_URL%/}/api/products?limit=1"
          STATUS_CODE=$(curl -sS -o /tmp/ui-api-products.json -w "%{http_code}" "$PRODUCTS_URL" || true)
          if [ "$STATUS_CODE" != "200" ]; then
            echo "Products smoke check failed after UI deploy with HTTP $STATUS_CODE at $PRODUCTS_URL" >&2
            cat /tmp/ui-api-products.json 2>/dev/null || true
            exit 1
          fi

          CATEGORIES_URL="${API_BASE_URL%/}/api/categories"
          STATUS_CODE=$(curl -sS -o /tmp/ui-api-categories.json -w "%{http_code}" "$CATEGORIES_URL" || true)
          if [ "$STATUS_CODE" != "200" ]; then
            echo "Categories smoke check failed after UI deploy with HTTP $STATUS_CODE at $CATEGORIES_URL" >&2
            cat /tmp/ui-api-categories.json 2>/dev/null || true
            exit 1
          fi

  deploy-ui-token:
    runs-on: ubuntu-latest
    if: ${{ inputs.deployStatic && inputs.uiOnly }}
    environment: ${{ inputs.environment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - uses: actions/checkout@v4

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'

      - name: Resolve API URL
        id: api
        shell: bash
        run: |
          set -euo pipefail

          RESOURCE_GROUP="${{ inputs.projectName }}-${{ inputs.environment }}-rg"

          APIM_URL=$(az apim show \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-apim" \
            --resource-group "$RESOURCE_GROUP" \
            --query "gatewayUrl" -o tsv)

          APIM_URL="$(echo "$APIM_URL" | xargs)"
          APIM_URL="${APIM_URL%/}"

          if [ -z "$APIM_URL" ]; then
            echo "Could not resolve APIM gateway URL for UI-only deployment." >&2
            exit 1
          fi

          RESOLVED_API_URL="$APIM_URL"

          if [ -n "${{ inputs.apiBaseUrl }}" ]; then
            OVERRIDE_API_URL="$(echo "${{ inputs.apiBaseUrl }}" | xargs)"
            OVERRIDE_API_URL="${OVERRIDE_API_URL%/}"

            if [ "$OVERRIDE_API_URL" != "$APIM_URL" ]; then
              echo "apiBaseUrl override drift detected. Provided URL does not match live APIM gateway URL." >&2
              echo "Provided: $OVERRIDE_API_URL" >&2
              echo "Expected: $APIM_URL" >&2
              exit 1
            fi

            RESOLVED_API_URL="$OVERRIDE_API_URL"
          fi

          if [ -z "$RESOLVED_API_URL" ]; then
            echo "UI-only deployment could not resolve API base URL. Provide 'apiBaseUrl' input or ensure the default APIM host is valid." >&2
            exit 1
          fi

          echo "api_url=$RESOLVED_API_URL" >> "$GITHUB_OUTPUT"
          echo "crud_api_url=$RESOLVED_API_URL" >> "$GITHUB_OUTPUT"

      - name: Smoke test APIM API health before UI deploy
        shell: bash
        run: |
          set -euo pipefail

          API_BASE_URL="${{ steps.api.outputs.api_url }}"
          API_BASE_URL="${API_BASE_URL%/}"

          smoke_endpoint() {
            local url="$1"
            local label="$2"
            STATUS_CODE=$(curl -sS -o /tmp/ui-token-health-pre.json -w "%{http_code}" "$url" || true)
            if [ "$STATUS_CODE" != "200" ]; then
              echo "$label check failed before UI-only deployment with HTTP $STATUS_CODE at $url" >&2
              cat /tmp/ui-token-health-pre.json 2>/dev/null || true
              exit 1
            fi
          }

          smoke_endpoint "${API_BASE_URL}/api/health" "APIM health"
          smoke_endpoint "${API_BASE_URL}/api/products?limit=1" "Products"
          smoke_endpoint "${API_BASE_URL}/api/categories" "Categories"

      - name: Deploy UI via Static Web Apps action (token mode)
        uses: Azure/static-web-apps-deploy@v1
        with:
          action: upload
          azure_static_web_apps_api_token: ${{ secrets.SWA_DEPLOYMENT_TOKEN }}
          app_location: apps/ui
          output_location: ''
          skip_api_build: true
          app_build_command: yarn install --frozen-lockfile && yarn build
        env:
          NEXT_PUBLIC_API_URL: ${{ steps.api.outputs.api_url }}
          NEXT_PUBLIC_CRUD_API_URL: ${{ steps.api.outputs.crud_api_url }}
          NEXT_PUBLIC_ENTRA_CLIENT_ID: ${{ secrets.NEXT_PUBLIC_ENTRA_CLIENT_ID }}
          NEXT_PUBLIC_ENTRA_TENANT_ID: ${{ secrets.NEXT_PUBLIC_ENTRA_TENANT_ID }}

      - name: Smoke test API health after UI-only deploy
        shell: bash
        run: |
          set -euo pipefail

          API_BASE_URL="${{ steps.api.outputs.api_url }}"
          API_BASE_URL="${API_BASE_URL%/}"

          smoke_endpoint() {
            local url="$1"
            local label="$2"
            STATUS_CODE=$(curl -sS -o /tmp/ui-token-health-post.json -w "%{http_code}" "$url" || true)
            if [ "$STATUS_CODE" != "200" ]; then
              echo "$label check failed after UI-only deployment with HTTP $STATUS_CODE at $url" >&2
              cat /tmp/ui-token-health-post.json 2>/dev/null || true
              exit 1
            fi
          }

          smoke_endpoint "${API_BASE_URL}/api/health" "APIM health"
          smoke_endpoint "${API_BASE_URL}/api/products?limit=1" "Products"
          smoke_endpoint "${API_BASE_URL}/api/categories" "Categories"

  deploy-agents:
    runs-on: ubuntu-latest
    if: ${{ !inputs.uiOnly && needs.detect-changes.outputs.agents_changed == 'true' }}
    needs:
      - provision
      - deploy-crud
      - deploy-foundry-models
      - detect-changes
    environment: ${{ inputs.environment }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.detect-changes.outputs.changed_agents_matrix) }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - uses: actions/checkout@v4

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Setup azd
        uses: Azure/setup-azd@v2

      - name: Authenticate azd (OIDC)
        run: |
          azd auth login \
            --client-id "${AZURE_CLIENT_ID}" \
            --tenant-id "${AZURE_TENANT_ID}" \
            --federated-credential-provider github \
            --no-prompt

      - name: Get AKS credentials
        run: |
          az aks get-credentials \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-aks" \
            --overwrite-existing

      - name: Resolve AKS managed identity client ID
        run: |
          AKS_MI_CLIENT_ID=$(az aks show \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-aks" \
            --query "identityProfile.kubeletidentity.clientId" -o tsv)
          echo "WORKLOAD_AZURE_CLIENT_ID=${AKS_MI_CLIENT_ID}" >> "$GITHUB_ENV"

      - name: Deploy service
        run: azd deploy --service "${{ matrix.service }}" --no-prompt -e "${{ inputs.environment }}"
        env:
          AZURE_CLIENT_ID: ${{ env.WORKLOAD_AZURE_CLIENT_ID }}
          AZURE_TENANT_ID: ${{ env.AZURE_TENANT_ID }}
          PROJECT_ENDPOINT: ${{ needs.provision.outputs.PROJECT_ENDPOINT }}
          PROJECT_NAME: ${{ needs.provision.outputs.PROJECT_NAME }}
          FOUNDRY_AUTO_ENSURE_ON_STARTUP: "true"
          FOUNDRY_STRICT_ENFORCEMENT: "true"
          MODEL_DEPLOYMENT_NAME_FAST: gpt-5-nano
          MODEL_DEPLOYMENT_NAME_RICH: gpt-5-nano
          POSTGRES_AUTH_MODE: ${{ needs.provision.outputs.POSTGRES_AUTH_MODE }}
          COSMOS_ACCOUNT_URI: ${{ needs.provision.outputs.COSMOS_ACCOUNT_URI }}
          COSMOS_DATABASE: ${{ needs.provision.outputs.COSMOS_DATABASE }}
          REDIS_HOST: ${{ needs.provision.outputs.REDIS_HOST }}
          EVENT_HUB_NAMESPACE: ${{ needs.provision.outputs.EVENT_HUB_NAMESPACE }}
          KEY_VAULT_URI: ${{ needs.provision.outputs.KEY_VAULT_URI }}
          APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ needs.provision.outputs.APPLICATIONINSIGHTS_CONNECTION_STRING }}
          POSTGRES_HOST: ${{ needs.provision.outputs.POSTGRES_HOST }}
          POSTGRES_USER: ${{ needs.provision.outputs.POSTGRES_AUTH_MODE == 'password' && needs.provision.outputs.POSTGRES_ADMIN_USER || needs.provision.outputs.POSTGRES_USER }}
          POSTGRES_DATABASE: ${{ needs.provision.outputs.POSTGRES_DATABASE }}

  sync-apim:
    runs-on: ubuntu-latest
    if: ${{ !inputs.uiOnly && (inputs.forceApimSync || needs.detect-changes.outputs.agents_changed == 'true' || needs.detect-changes.outputs.crud_changed == 'true') && (needs.deploy-crud.result == 'success' || needs.deploy-crud.result == 'skipped') && (needs.deploy-agents.result == 'success' || needs.deploy-agents.result == 'skipped') }}
    needs:
      - deploy-crud
      - deploy-agents
      - detect-changes
    environment: ${{ inputs.environment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      CHANGED_SERVICES: ${{ needs.detect-changes.outputs.changed_aks_services_csv }}
    steps:
      - uses: actions/checkout@v4

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Get AKS credentials
        run: |
          az aks get-credentials \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-aks" \
            --overwrite-existing

      - name: Sync APIM agent APIs
        run: |
          bash .infra/azd/hooks/sync-apim-agents.sh --use-ingress \
            --app-gw-name "${{ inputs.projectName }}-${{ inputs.environment }}-appgw"
        env:
          AZURE_RESOURCE_GROUP: ${{ inputs.projectName }}-${{ inputs.environment }}-rg

  smoke-apim:
    runs-on: ubuntu-latest
    if: ${{ !inputs.uiOnly && (inputs.forceApimSync || needs.detect-changes.outputs.agents_changed == 'true' || needs.detect-changes.outputs.crud_changed == 'true') && (needs.sync-apim.result == 'success' || needs.sync-apim.result == 'skipped') && (needs.ensure-foundry-agents.result == 'success' || needs.ensure-foundry-agents.result == 'skipped') }}
    needs:
      - detect-changes
      - sync-apim
      - ensure-foundry-agents
    environment: ${{ inputs.environment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      CHANGED_AGENT_SERVICES: ${{ needs.detect-changes.outputs.changed_agent_services_csv }}
    steps:
      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Resolve APIM gateway URL
        id: apim
        shell: bash
        run: |
          set -euo pipefail

          APIM_URL=$(az apim show \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-apim" \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --query "gatewayUrl" -o tsv)

          APIM_URL="$(echo "$APIM_URL" | xargs)"
          APIM_URL="${APIM_URL%/}"

          if [ -z "$APIM_URL" ]; then
            echo "Failed to resolve APIM gateway URL for smoke tests." >&2
            exit 1
          fi

          echo "gateway_url=$APIM_URL" >> "$GITHUB_OUTPUT"

      - name: Smoke test CRUD and changed agent health through APIM
        shell: bash
        run: |
          set -euo pipefail

          API_BASE="${{ steps.apim.outputs.gateway_url }}"

          smoke_health() {
            local url="$1"
            local label="$2"

            for attempt in $(seq 1 20); do
              STATUS_CODE=$(curl -sS -o /tmp/smoke-response.json -w "%{http_code}" "$url" || true)
              if [ "$STATUS_CODE" = "200" ]; then
                echo "[OK] $label: $url"
                return 0
              fi
              echo "Attempt $attempt/20 failed for $label with HTTP $STATUS_CODE"
              sleep 10
            done

            echo "[FAIL] $label: $url" >&2
            cat /tmp/smoke-response.json 2>/dev/null || true
            return 1
          }

          smoke_health "${API_BASE}/api/health" "crud-health"
          smoke_health "${API_BASE}/api/products?limit=1" "crud-products"
          smoke_health "${API_BASE}/api/categories" "crud-categories"

          if [ -n "${CHANGED_AGENT_SERVICES}" ]; then
            IFS=',' read -r -a agent_services <<< "${CHANGED_AGENT_SERVICES}"
            for svc in "${agent_services[@]}"; do
              [ -z "$svc" ] && continue
              smoke_health "${API_BASE}/agents/${svc}/health" "agent-health-$svc"
            done
          else
            echo "No changed agent services detected; skipping agent health smoke checks."
          fi

  ensure-foundry-agents:
    runs-on: ubuntu-latest
    if: ${{ !inputs.uiOnly && needs.detect-changes.outputs.agents_changed == 'true' && (needs.deploy-agents.result == 'success' || needs.deploy-agents.result == 'skipped') }}
    needs:
      - deploy-agents
      - detect-changes
    environment: ${{ inputs.environment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      CHANGED_SERVICES: ${{ needs.detect-changes.outputs.changed_agent_services_csv }}
    steps:
      - uses: actions/checkout@v4

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Setup Python (for azure.yaml parser)
        uses: actions/setup-python@v5
        with:
          python-version: '3.13'

      - name: Get AKS credentials
        run: |
          az aks get-credentials \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-aks" \
            --overwrite-existing

      - name: Ensure all V2 Foundry agents are provisioned
        run: |
          bash .infra/azd/hooks/ensure-foundry-agents.sh --port-forward
        env:
          K8S_NAMESPACE: holiday-peak

      - name: Verify Foundry readiness across services
        run: |
          set -euo pipefail

          if [ -n "${CHANGED_SERVICES}" ]; then
            SERVICES=$(printf '%s' "${CHANGED_SERVICES}" | tr ',' '\n' | sed '/^crud-service$/d' | sed '/^$/d')
          else
            SERVICES=$(python3 -c "
            import re
            with open('azure.yaml') as f: lines = f.readlines()
            s, cs, ch = False, None, None
            svcs = []
            for l in lines:
              l = l.rstrip()
              if not s:
                if re.match(r'^services:\s*$', l): s = True
                continue
              if re.match(r'^[^\s]', l): break
              m = re.match(r'^  ([a-z0-9\-]+):\s*$', l)
              if m:
                if cs and ch == 'aks' and cs != 'crud-service': svcs.append(cs)
                cs, ch = m.group(1), None
                continue
              h = re.match(r'^    host:\s*(\S+)', l)
              if h: ch = h.group(1)
            if cs and ch == 'aks' and cs != 'crud-service': svcs.append(cs)
            print('\\n'.join(svcs))
            ")
          fi

          FAILED=0
          while IFS= read -r SVC; do
            [ -z "$SVC" ] && continue
            LOCAL_PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("",0)); print(s.getsockname()[1]); s.close()')
            kubectl port-forward "svc/$SVC" "$LOCAL_PORT:8000" -n holiday-peak &
            PF_PID=$!
            sleep 3

            STATUS=$(curl -s "http://localhost:$LOCAL_PORT/health" 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('status','unknown'))" 2>/dev/null || echo "unreachable")
            kill "$PF_PID" 2>/dev/null || true

            if [ "$STATUS" = "ok" ]; then
              echo "  [OK] $SVC"
            else
              echo "  [WARN] $SVC: status=$STATUS"
              FAILED=$((FAILED + 1))
            fi
          done <<< "$SERVICES"

          if [ "$FAILED" -gt 0 ]; then
            echo "$FAILED service(s) did not report status=ok during Foundry readiness checks." >&2
            exit 1
          fi

  seed-demo-data:
    runs-on: ubuntu-latest
    needs:
      - sync-apim
      - smoke-apim
      - ensure-foundry-agents
      - deploy-ui
    if: ${{ always() && inputs.seedDemoData && needs.ensure-foundry-agents.result == 'success' && (needs.smoke-apim.result == 'success' || needs.smoke-apim.result == 'skipped') && (needs.deploy-ui.result == 'success' || needs.deploy-ui.result == 'skipped') && inputs.environment != 'prod' && inputs.environment != 'production' }}
    environment: ${{ inputs.environment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - uses: actions/checkout@v4

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Setup azd
        uses: Azure/setup-azd@v2

      - name: Resolve environment values
        run: |
          eval "$(azd env get-values -e '${{ inputs.environment }}' | sed 's/^/export /')"

          if [ -z "${SERVICE_CRUD_SERVICE_IMAGE_NAME:-}" ]; then
            echo "SERVICE_CRUD_SERVICE_IMAGE_NAME is required for seed job"
            exit 1
          fi

          echo "CRUD_IMAGE=${SERVICE_CRUD_SERVICE_IMAGE_NAME}" >> "$GITHUB_ENV"
          echo "POSTGRES_HOST=${POSTGRES_HOST}" >> "$GITHUB_ENV"
          echo "POSTGRES_USER=${POSTGRES_USER}" >> "$GITHUB_ENV"
          echo "POSTGRES_PASSWORD=${POSTGRES_PASSWORD}" >> "$GITHUB_ENV"
          echo "POSTGRES_DATABASE=${POSTGRES_DATABASE}" >> "$GITHUB_ENV"
          echo "POSTGRES_PORT=${POSTGRES_PORT}" >> "$GITHUB_ENV"
          echo "POSTGRES_SSL=${POSTGRES_SSL}" >> "$GITHUB_ENV"

      - name: Get AKS credentials
        run: |
          az aks get-credentials \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-aks" \
            --overwrite-existing

      - name: Run demo faker seed job
        run: |
          JOB_NAME="crud-demo-seed-${{ github.run_id }}"

          cat <<EOF | kubectl apply -f -
          apiVersion: batch/v1
          kind: Job
          metadata:
            name: ${JOB_NAME}
            namespace: holiday-peak
          spec:
            backoffLimit: 1
            ttlSecondsAfterFinished: 600
            template:
              spec:
                restartPolicy: Never
                containers:
                  - name: seed
                    image: ${CRUD_IMAGE}
                    imagePullPolicy: Always
                    command: ["python", "-m", "crud_service.scripts.seed_demo_data"]
                    env:
                      - name: DEMO_ENVIRONMENT
                        value: "${{ inputs.environment }}"
                      - name: POSTGRES_HOST
                        value: "${POSTGRES_HOST}"
                      - name: POSTGRES_USER
                        value: "${POSTGRES_USER}"
                      - name: POSTGRES_PASSWORD
                        value: "${POSTGRES_PASSWORD}"
                      - name: POSTGRES_DATABASE
                        value: "${POSTGRES_DATABASE}"
                      - name: POSTGRES_PORT
                        value: "${POSTGRES_PORT}"
                      - name: POSTGRES_SSL
                        value: "${POSTGRES_SSL}"
          EOF

          kubectl wait --for=condition=complete "job/${JOB_NAME}" -n holiday-peak --timeout=10m
          kubectl logs "job/${JOB_NAME}" -n holiday-peak