Merge pull request microsoft#76 from microsoft/task/ci-pipeline-quota-check-macae

ci: implement quota check in CI Pipeline to ensure resource availability in supported region

Author: Roopan-Microsoft

.github/workflows/deploy.yml

@@ -14,6 +14,48 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v3
 
+      - name: Run Quota Check
+        id: quota-check
+        run: |
+          export AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}
+          export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}
+          export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}
+          export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
+          export GPT_MIN_CAPACITY="130"
+
+          chmod +x deploy/scripts/checkquota.sh
+          if ! deploy/scripts/checkquota.sh; then
+            # If quota check fails due to insufficient quota, set the flag
+            if grep -q "No region with sufficient quota found" deploy/scripts/checkquota.sh; then
+              echo "QUOTA_FAILED=true" >> $GITHUB_ENV
+            fi
+            exit 1  # Fail the pipeline if any other failure occurs
+          fi
+
+      - name: Send Notification on Quota Failure
+        if: env.QUOTA_FAILED == 'true'
+        run: |
+          RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          EMAIL_BODY=$(cat <<EOF
+          {
+            "body": "<p>Dear Team,</p><p>The quota check has failed, and the pipeline cannot proceed.</p><p><strong>Build URL:</strong> ${RUN_URL}</p><p>Please take necessary action.</p><p>Best regards,<br>Your Automation Team</p>"
+          }
+          EOF
+          )
+
+          curl -X POST "${{ secrets.AUTO_LOGIC_APP_URL }}" \
+            -H "Content-Type: application/json" \
+            -d "$EMAIL_BODY" || echo "Failed to send notification"
+
+      - name: Fail Pipeline if Quota Check Fails
+        if: env.QUOTA_FAILED == 'true'
+        run: exit 1
+
+      - name: Set Deployment Region
+        run: |
+          echo "Selected Region: $VALID_REGION"
+          echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV 
+
       - name: Setup Azure CLI
         run: |
           curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
@@ -45,7 +87,7 @@ jobs:
           rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
           if [ "$rg_exists" = "false" ]; then
             echo "Resource group does not exist. Creating..."
-            az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location eastus || { echo "Error creating resource group"; exit 1; }
+            az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.AZURE_LOCATION }} || { echo "Error creating resource group"; exit 1; }
           else
             echo "Resource group already exists."
           fi
@@ -58,7 +100,7 @@ jobs:
           az deployment group create \
             --resource-group ${{ env.RESOURCE_GROUP_NAME }} \
             --template-file deploy/macae.bicep \
-            --parameters azureOpenAILocation=eastus cosmosLocation=eastus2
+            --parameters azureOpenAILocation=${{env.AZURE_LOCATION }} cosmosLocation=${{env.AZURE_LOCATION }}
 
 
       - name: Send Notification on Failure

deploy/scripts/checkquota.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+# List of Azure regions to check for quota (update as needed)
+REGIONS=("eastus" "westus" "northcentralus" "uksouth" "swedencentral")
+
+SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID}"
+GPT_MIN_CAPACITY="${GPT_MIN_CAPACITY}"
+AZURE_CLIENT_ID="${AZURE_CLIENT_ID}"
+AZURE_TENANT_ID="${AZURE_TENANT_ID}"
+AZURE_CLIENT_SECRET="${AZURE_CLIENT_SECRET}"
+
+# Authenticate using Managed Identity
+echo "Authentication using Managed Identity..."
+if ! az login --service-principal -u "$AZURE_CLIENT_ID" -p "$AZURE_CLIENT_SECRET" --tenant "$AZURE_TENANT_ID"; then
+   echo "❌ Error: Failed to login using Managed Identity."
+   exit 1
+fi
+
+echo "🔄 Validating required environment variables..."
+if [[ -z "$SUBSCRIPTION_ID" || -z "$GPT_MIN_CAPACITY" ]]; then
+    echo "❌ ERROR: Missing required environment variables."
+    exit 1
+fi
+
+echo "🔄 Setting Azure subscription..."
+if ! az account set --subscription "$SUBSCRIPTION_ID"; then
+    echo "❌ ERROR: Invalid subscription ID or insufficient permissions."
+    exit 1
+fi
+echo "✅ Azure subscription set successfully."
+
+# Define models and their minimum required capacities
+declare -A MIN_CAPACITY=(
+    ["OpenAI.Standard.gpt-4o"]=$GPT_MIN_CAPACITY
+)
+
+VALID_REGION=""
+for REGION in "${REGIONS[@]}"; do
+    echo "----------------------------------------"
+    echo "🔍 Checking region: $REGION"
+
+    QUOTA_INFO=$(az cognitiveservices usage list --location "$REGION" --output json)
+    if [ -z "$QUOTA_INFO" ]; then
+        echo "⚠️ WARNING: Failed to retrieve quota for region $REGION. Skipping."
+        continue
+    fi
+
+    INSUFFICIENT_QUOTA=false
+    for MODEL in "${!MIN_CAPACITY[@]}"; do
+        MODEL_INFO=$(echo "$QUOTA_INFO" | awk -v model="\"value\": \"$MODEL\"" '
+            BEGIN { RS="},"; FS="," }
+            $0 ~ model { print $0 }
+        ')
+
+        if [ -z "$MODEL_INFO" ]; then
+            echo "⚠️ WARNING: No quota information found for model: $MODEL in $REGION. Skipping."
+            continue
+        fi
+
+        CURRENT_VALUE=$(echo "$MODEL_INFO" | awk -F': ' '/"currentValue"/ {print $2}' | tr -d ',' | tr -d ' ')
+        LIMIT=$(echo "$MODEL_INFO" | awk -F': ' '/"limit"/ {print $2}' | tr -d ',' | tr -d ' ')
+
+        CURRENT_VALUE=${CURRENT_VALUE:-0}
+        LIMIT=${LIMIT:-0}
+
+        CURRENT_VALUE=$(echo "$CURRENT_VALUE" | cut -d'.' -f1)
+        LIMIT=$(echo "$LIMIT" | cut -d'.' -f1)
+
+        AVAILABLE=$((LIMIT - CURRENT_VALUE))
+
+        echo "✅ Model: $MODEL | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE"
+
+        if [ "$AVAILABLE" -lt "${MIN_CAPACITY[$MODEL]}" ]; then
+            echo "❌ ERROR: $MODEL in $REGION has insufficient quota."
+            INSUFFICIENT_QUOTA=true
+            break
+        fi
+    done
+
+    if [ "$INSUFFICIENT_QUOTA" = false ]; then
+        VALID_REGION="$REGION"
+        break
+    fi
+
+done
+
+if [ -z "$VALID_REGION" ]; then
+    echo "❌ No region with sufficient quota found. Blocking deployment."
+    echo "QUOTA_FAILED=true" >> "$GITHUB_ENV"
+    exit 0
+else
+    echo "✅ Final Region: $VALID_REGION"
+    echo "VALID_REGION=$VALID_REGION" >> "$GITHUB_ENV"
+    exit 0
+fi