Skip to content

Commit b825cc3

Browse files
Roopan-MicrosoftRoopan-Microsoft
authored
Merge pull request #258 from microsoft/task/ci-pipeline-quota-check-kmg-ckmv2
ci: implement quota check in CI Pipeline to ensure resource availability in supported region
2 parents 50b9e52 + 022b2d1 commit b825cc3

File tree

4 files changed

+283
-4
lines changed

4 files changed

+283
-4
lines changed

‎.github/workflows/bicep_deploy.yml‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,49 @@ jobs:
1212
- name: Checkout Code
1313
uses: actions/checkout@v3
1414

15+
- name: Run Quota Check
16+
id: quota-check
17+
run: |
18+
export AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}
19+
export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}
20+
export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}
21+
export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
22+
export GPT_MIN_CAPACITY="130"
23+
export TEXT_EMBEDDING_MIN_CAPACITY="30"
24+
25+
chmod +x infra/scripts/checkquota_ckmv2.sh
26+
if ! infra/scripts/checkquota_ckmv2.sh; then
27+
# If quota check fails due to insufficient quota, set the flag
28+
if grep -q "No region with sufficient quota found" infra/scripts/checkquota_ckmv2.sh; then
29+
echo "QUOTA_FAILED=true" >> $GITHUB_ENV
30+
fi
31+
exit 1 # Fail the pipeline if any other failure occurs
32+
fi
33+
34+
- name: Send Notification on Quota Failure
35+
if: env.QUOTA_FAILED == 'true'
36+
run: |
37+
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
38+
EMAIL_BODY=$(cat <<EOF
39+
{
40+
"body": "<p>Dear Team,</p><p>The quota check has failed, and the pipeline cannot proceed.</p><p><strong>Build URL:</strong> ${RUN_URL}</p><p>Please take necessary action.</p><p>Best regards,<br>Your Automation Team</p>"
41+
}
42+
EOF
43+
)
44+
45+
curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
46+
-H "Content-Type: application/json" \
47+
-d "$EMAIL_BODY" || echo "Failed to send notification"
48+
49+
- name: Fail Pipeline if Quota Check Fails
50+
if: env.QUOTA_FAILED == 'true'
51+
run: exit 1
52+
53+
- name: Set Deployment Region
54+
run: |
55+
echo "Selected Region: $VALID_REGION"
56+
echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV
57+
1558
- name: Setup Azure CLI
1659
run: |
1760
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
@@ -36,7 +79,7 @@ jobs:
3679
3780
- name: Create Resource Group
3881
run: |
39-
az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location eastus
82+
az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.AZURE_LOCATION }}
4083
4184
- name: Generate Unique Solution Prefix
4285
id: generate_solution_prefix
@@ -56,7 +99,7 @@ jobs:
5699
az deployment group create \
57100
--resource-group ${{ env.RESOURCE_GROUP_NAME }} \
58101
--template-file infra/main.bicep \
59-
--parameters solutionPrefix=${{ env.SOLUTION_PREFIX }} location=eastus
102+
--parameters solutionPrefix=${{ env.SOLUTION_PREFIX }} location=${{ env.AZURE_LOCATION }}
60103
61104
- name: Delete Bicep Deployment
62105
if: success()

‎.github/workflows/deploy-KMGeneric.yml‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,49 @@ jobs:
1919
- name: Checkout Code
2020
uses: actions/checkout@v3
2121

22+
- name: Run Quota Check
23+
id: quota-check
24+
run: |
25+
export AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}
26+
export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}
27+
export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}
28+
export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
29+
export GPT_MIN_CAPACITY="130"
30+
export TEXT_EMBEDDING_MIN_CAPACITY="30"
31+
32+
chmod +x infra/scripts/checkquota_km.sh
33+
if ! infra/scripts/checkquota_km.sh; then
34+
# If quota check fails due to insufficient quota, set the flag
35+
if grep -q "No region with sufficient quota found" infra/scripts/checkquota_km.sh; then
36+
echo "QUOTA_FAILED=true" >> $GITHUB_ENV
37+
fi
38+
exit 1 # Fail the pipeline if any other failure occurs
39+
fi
40+
41+
- name: Send Notification on Quota Failure
42+
if: env.QUOTA_FAILED == 'true'
43+
run: |
44+
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
45+
EMAIL_BODY=$(cat <<EOF
46+
{
47+
"body": "<p>Dear Team,</p><p>The quota check has failed, and the pipeline cannot proceed.</p><p><strong>Build URL:</strong> ${RUN_URL}</p><p>Please take necessary action.</p><p>Best regards,<br>Your Automation Team</p>"
48+
}
49+
EOF
50+
)
51+
52+
curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
53+
-H "Content-Type: application/json" \
54+
-d "$EMAIL_BODY" || echo "Failed to send notification"
55+
56+
- name: Fail Pipeline if Quota Check Fails
57+
if: env.QUOTA_FAILED == 'true'
58+
run: exit 1
59+
60+
- name: Set Deployment Region
61+
run: |
62+
echo "Selected Region: $VALID_REGION"
63+
echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV
64+
2265
- name: Setup Azure CLI
2366
run: |
2467
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
@@ -43,7 +86,7 @@ jobs:
4386
4487
- name: Create Resource Group
4588
run: |
46-
az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location eastus2
89+
az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.AZURE_LOCATION }}
4790
4891
- name: Generate Unique Solution Prefix
4992
id: generate_solution_prefix
@@ -68,7 +111,7 @@ jobs:
68111
az deployment group create \
69112
--resource-group ${{ env.RESOURCE_GROUP_NAME }} \
70113
--template-file infra/main.bicep \
71-
--parameters environmentName=${{env.SOLUTION_PREFIX}} contentUnderstandingLocation="West US" secondaryLocation="eastus2" imageTag=${{ steps.determine_tag.outputs.tagname }}
114+
--parameters environmentName=${{env.SOLUTION_PREFIX}} contentUnderstandingLocation="West US" secondaryLocation="${{ env.AZURE_LOCATION }}" imageTag=${{ steps.determine_tag.outputs.tagname }}
72115
73116

74117
- name: Extract AI Services and Key Vault Names

‎infra/scripts/checkquota_ckmv2.sh‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
#!/bin/bash
2+
3+
# List of Azure regions to check for quota (update as needed)
4+
REGIONS=("eastus" "eastus2" "northcentralus")
5+
6+
SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID}"
7+
GPT_MIN_CAPACITY="${GPT_MIN_CAPACITY}"
8+
TEXT_EMBEDDING_MIN_CAPACITY="${TEXT_EMBEDDING_MIN_CAPACITY}"
9+
AZURE_CLIENT_ID="${AZURE_CLIENT_ID}"
10+
AZURE_TENANT_ID="${AZURE_TENANT_ID}"
11+
AZURE_CLIENT_SECRET="${AZURE_CLIENT_SECRET}"
12+
13+
# Authenticate using Managed Identity
14+
echo "Authentication using Managed Identity..."
15+
if ! az login --service-principal -u "$AZURE_CLIENT_ID" -p "$AZURE_CLIENT_SECRET" --tenant "$AZURE_TENANT_ID"; then
16+
echo "❌ Error: Failed to login using Managed Identity."
17+
exit 1
18+
fi
19+
20+
echo "🔄 Validating required environment variables..."
21+
if [[ -z "$SUBSCRIPTION_ID" || -z "$GPT_MIN_CAPACITY" || -z "$TEXT_EMBEDDING_MIN_CAPACITY" ]]; then
22+
echo "❌ ERROR: Missing required environment variables."
23+
exit 1
24+
fi
25+
26+
echo "🔄 Setting Azure subscription..."
27+
if ! az account set --subscription "$SUBSCRIPTION_ID"; then
28+
echo "❌ ERROR: Invalid subscription ID or insufficient permissions."
29+
exit 1
30+
fi
31+
echo "✅ Azure subscription set successfully."
32+
33+
# Define models and their minimum required capacities
34+
declare -A MIN_CAPACITY=(
35+
["OpenAI.Standard.gpt-4"]=$GPT_MIN_CAPACITY #for ckmv2
36+
)
37+
38+
VALID_REGION=""
39+
for REGION in "${REGIONS[@]}"; do
40+
echo "----------------------------------------"
41+
echo "🔍 Checking region: $REGION"
42+
43+
QUOTA_INFO=$(az cognitiveservices usage list --location "$REGION" --output json)
44+
if [ -z "$QUOTA_INFO" ]; then
45+
echo "⚠️ WARNING: Failed to retrieve quota for region $REGION. Skipping."
46+
continue
47+
fi
48+
49+
INSUFFICIENT_QUOTA=false
50+
for MODEL in "${!MIN_CAPACITY[@]}"; do
51+
MODEL_INFO=$(echo "$QUOTA_INFO" | awk -v model="\"value\": \"$MODEL\"" '
52+
BEGIN { RS="},"; FS="," }
53+
$0 ~ model { print $0 }
54+
')
55+
56+
if [ -z "$MODEL_INFO" ]; then
57+
echo "⚠️ WARNING: No quota information found for model: $MODEL in $REGION. Skipping."
58+
continue
59+
fi
60+
61+
CURRENT_VALUE=$(echo "$MODEL_INFO" | awk -F': ' '/"currentValue"/ {print $2}' | tr -d ',' | tr -d ' ')
62+
LIMIT=$(echo "$MODEL_INFO" | awk -F': ' '/"limit"/ {print $2}' | tr -d ',' | tr -d ' ')
63+
64+
CURRENT_VALUE=${CURRENT_VALUE:-0}
65+
LIMIT=${LIMIT:-0}
66+
67+
CURRENT_VALUE=$(echo "$CURRENT_VALUE" | cut -d'.' -f1)
68+
LIMIT=$(echo "$LIMIT" | cut -d'.' -f1)
69+
70+
AVAILABLE=$((LIMIT - CURRENT_VALUE))
71+
72+
echo "✅ Model: $MODEL | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE"
73+
74+
if [ "$AVAILABLE" -lt "${MIN_CAPACITY[$MODEL]}" ]; then
75+
echo "❌ ERROR: $MODEL in $REGION has insufficient quota."
76+
INSUFFICIENT_QUOTA=true
77+
break
78+
fi
79+
done
80+
81+
if [ "$INSUFFICIENT_QUOTA" = false ]; then
82+
VALID_REGION="$REGION"
83+
break
84+
fi
85+
86+
done
87+
88+
if [ -z "$VALID_REGION" ]; then
89+
echo "❌ No region with sufficient quota found. Blocking deployment."
90+
echo "QUOTA_FAILED=true" >> "$GITHUB_ENV"
91+
exit 0
92+
else
93+
echo "✅ Final Region: $VALID_REGION"
94+
echo "VALID_REGION=$VALID_REGION" >> "$GITHUB_ENV"
95+
exit 0
96+
fi

‎infra/scripts/checkquota_km.sh‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/bin/bash
2+
3+
# List of Azure regions to check for quota (update as needed)
4+
REGIONS=("eastus" "eastus2")
5+
6+
SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID}"
7+
GPT_MIN_CAPACITY="${GPT_MIN_CAPACITY}"
8+
TEXT_EMBEDDING_MIN_CAPACITY="${TEXT_EMBEDDING_MIN_CAPACITY}"
9+
AZURE_CLIENT_ID="${AZURE_CLIENT_ID}"
10+
AZURE_TENANT_ID="${AZURE_TENANT_ID}"
11+
AZURE_CLIENT_SECRET="${AZURE_CLIENT_SECRET}"
12+
13+
# Authenticate using Managed Identity
14+
echo "Authentication using Managed Identity..."
15+
if ! az login --service-principal -u "$AZURE_CLIENT_ID" -p "$AZURE_CLIENT_SECRET" --tenant "$AZURE_TENANT_ID"; then
16+
echo "❌ Error: Failed to login using Managed Identity."
17+
exit 1
18+
fi
19+
20+
echo "🔄 Validating required environment variables..."
21+
if [[ -z "$SUBSCRIPTION_ID" || -z "$GPT_MIN_CAPACITY" || -z "$TEXT_EMBEDDING_MIN_CAPACITY" ]]; then
22+
echo "❌ ERROR: Missing required environment variables."
23+
exit 1
24+
fi
25+
26+
echo "🔄 Setting Azure subscription..."
27+
if ! az account set --subscription "$SUBSCRIPTION_ID"; then
28+
echo "❌ ERROR: Invalid subscription ID or insufficient permissions."
29+
exit 1
30+
fi
31+
echo "✅ Azure subscription set successfully."
32+
33+
# Define models and their minimum required capacities
34+
declare -A MIN_CAPACITY=(
35+
["OpenAI.Standard.gpt-4o-mini"]=$GPT_MIN_CAPACITY #km generic
36+
["OpenAI.Standard.text-embedding-ada-002"]=$TEXT_EMBEDDING_MIN_CAPACITY #km generic
37+
)
38+
39+
VALID_REGION=""
40+
for REGION in "${REGIONS[@]}"; do
41+
echo "----------------------------------------"
42+
echo "🔍 Checking region: $REGION"
43+
44+
QUOTA_INFO=$(az cognitiveservices usage list --location "$REGION" --output json)
45+
if [ -z "$QUOTA_INFO" ]; then
46+
echo "⚠️ WARNING: Failed to retrieve quota for region $REGION. Skipping."
47+
continue
48+
fi
49+
50+
INSUFFICIENT_QUOTA=false
51+
for MODEL in "${!MIN_CAPACITY[@]}"; do
52+
MODEL_INFO=$(echo "$QUOTA_INFO" | awk -v model="\"value\": \"$MODEL\"" '
53+
BEGIN { RS="},"; FS="," }
54+
$0 ~ model { print $0 }
55+
')
56+
57+
if [ -z "$MODEL_INFO" ]; then
58+
echo "⚠️ WARNING: No quota information found for model: $MODEL in $REGION. Skipping."
59+
continue
60+
fi
61+
62+
CURRENT_VALUE=$(echo "$MODEL_INFO" | awk -F': ' '/"currentValue"/ {print $2}' | tr -d ',' | tr -d ' ')
63+
LIMIT=$(echo "$MODEL_INFO" | awk -F': ' '/"limit"/ {print $2}' | tr -d ',' | tr -d ' ')
64+
65+
CURRENT_VALUE=${CURRENT_VALUE:-0}
66+
LIMIT=${LIMIT:-0}
67+
68+
CURRENT_VALUE=$(echo "$CURRENT_VALUE" | cut -d'.' -f1)
69+
LIMIT=$(echo "$LIMIT" | cut -d'.' -f1)
70+
71+
AVAILABLE=$((LIMIT - CURRENT_VALUE))
72+
73+
echo "✅ Model: $MODEL | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE"
74+
75+
if [ "$AVAILABLE" -lt "${MIN_CAPACITY[$MODEL]}" ]; then
76+
echo "❌ ERROR: $MODEL in $REGION has insufficient quota."
77+
INSUFFICIENT_QUOTA=true
78+
break
79+
fi
80+
done
81+
82+
if [ "$INSUFFICIENT_QUOTA" = false ]; then
83+
VALID_REGION="$REGION"
84+
break
85+
fi
86+
87+
done
88+
89+
if [ -z "$VALID_REGION" ]; then
90+
echo "❌ No region with sufficient quota found. Blocking deployment."
91+
echo "QUOTA_FAILED=true" >> "$GITHUB_ENV"
92+
exit 0
93+
else
94+
echo "✅ Final Region: $VALID_REGION"
95+
echo "VALID_REGION=$VALID_REGION" >> "$GITHUB_ENV"
96+
exit 0
97+
fi

0 commit comments

Comments
 (0)