Skip to content

Commit ae9438c

Browse files
Merge pull request #238 from microsoft/task/ci-pipeline-quota-check
ci: implement quota check in CI Pipeline to ensure resource availability in supported region
2 parents 2d8fdd2 + ba3b019 commit ae9438c

File tree

3 files changed

+154
-3
lines changed

3 files changed

+154
-3
lines changed

.github/workflows/deploy.yml

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,52 @@ jobs:
2323
run: |
2424
az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }}
2525
26+
- name: Run Quota Check
27+
id: quota-check
28+
run: |
29+
export AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}
30+
export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}
31+
export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}
32+
export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
33+
export GPT_MIN_CAPACITY="30"
34+
export TEXT_EMBEDDING_MIN_CAPACITY="30"
35+
36+
chmod +x scripts/checkquota.sh
37+
if ! scripts/checkquota.sh; then
38+
# If quota check fails due to insufficient quota, set the flag
39+
if grep -q "No region with sufficient quota found" scripts/checkquota.sh; then
40+
echo "QUOTA_FAILED=true" >> $GITHUB_ENV
41+
fi
42+
exit 1 # Fail the pipeline if any other failure occurs
43+
fi
44+
45+
46+
- name: Send Notification on Quota Failure
47+
if: env.QUOTA_FAILED == 'true'
48+
run: |
49+
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
50+
EMAIL_BODY=$(cat <<EOF
51+
{
52+
"body": "<p>Dear Team,</p><p>The quota check has failed, and the pipeline cannot proceed.</p><p><strong>Build URL:</strong> ${RUN_URL}</p><p>Please take necessary action.</p><p>Best regards,<br>Your Automation Team</p>"
53+
}
54+
EOF
55+
)
56+
57+
curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
58+
-H "Content-Type: application/json" \
59+
-d "$EMAIL_BODY" || echo "Failed to send notification"
60+
61+
- name: Fail Pipeline if Quota Check Fails
62+
if: env.QUOTA_FAILED == 'true'
63+
run: exit 1
64+
2665
- name: Install Bicep CLI
2766
run: az bicep install
67+
68+
- name: Set Deployment Region
69+
run: |
70+
echo "Selected Region: $VALID_REGION"
71+
echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV
2872
2973
- name: Generate Resource Group Name
3074
id: generate_rg_name
@@ -44,7 +88,7 @@ jobs:
4488
rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
4589
if [ "$rg_exists" = "false" ]; then
4690
echo "Resource group does not exist. Creating..."
47-
az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location northcentralus || { echo "Error creating resource group"; exit 1; }
91+
az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.AZURE_LOCATION }} || { echo "Error creating resource group"; exit 1; }
4892
else
4993
echo "Resource group already exists."
5094
fi
@@ -72,7 +116,7 @@ jobs:
72116
ApplicationInsightsName="appins-${{ env.SOLUTION_PREFIX }}" \
73117
WebsiteName="webapp-${{ env.SOLUTION_PREFIX }}" \
74118
CosmosDBName="db-cosmos-${{ env.SOLUTION_PREFIX }}" \
75-
CosmosDBRegion="NorthCentralUS" \
119+
CosmosDBRegion="${{ env.AZURE_LOCATION }}" \
76120
AzureSearchService="search-${{ env.SOLUTION_PREFIX }}" \
77121
AzureOpenAIResource="aoai-${{ env.SOLUTION_PREFIX }}" \
78122
WorkspaceName="worksp-${{ env.SOLUTION_PREFIX }}"

.github/workflows/docker-build-and-push.yml

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,17 @@ jobs:
5050

5151
- name: Determine Tag Name Based on Branch
5252
id: determine_tag
53-
run: echo "tagname=${{ github.ref_name == 'main' && 'latest' || github.ref_name == 'dev' && 'dev' || github.ref_name == 'demo' && 'demo' || github.head_ref || 'default' }}" >> $GITHUB_OUTPUT
53+
run: |
54+
if [[ "${{ github.ref_name }}" == "main" ]]; then
55+
echo "tagname=latest" >> $GITHUB_OUTPUT
56+
elif [[ "${{ github.ref_name }}" == "dev" ]]; then
57+
echo "tagname=dev" >> $GITHUB_OUTPUT
58+
elif [[ "${{ github.ref_name }}" == "demo" ]]; then
59+
echo "tagname=demo" >> $GITHUB_OUTPUT
60+
else
61+
echo "tagname=default" >> $GITHUB_OUTPUT
62+
63+
fi
5464
5565
- name: Build Docker Image and optionally push
5666
uses: docker/build-push-action@v6

scripts/checkquota.sh

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/bin/bash
2+
3+
# List of Azure regions to check for quota (update as needed)
4+
REGIONS=("eastus" "westus" "northcentralus" "uksouth" "swedencentral")
5+
6+
SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID}"
7+
GPT_MIN_CAPACITY="${GPT_MIN_CAPACITY}"
8+
TEXT_EMBEDDING_MIN_CAPACITY="${TEXT_EMBEDDING_MIN_CAPACITY}"
9+
AZURE_CLIENT_ID="${AZURE_CLIENT_ID}"
10+
AZURE_TENANT_ID="${AZURE_TENANT_ID}"
11+
AZURE_CLIENT_SECRET="${AZURE_CLIENT_SECRET}"
12+
13+
# Authenticate using Managed Identity
14+
echo "Authentication using Managed Identity..."
15+
if ! az login --service-principal -u "$AZURE_CLIENT_ID" -p "$AZURE_CLIENT_SECRET" --tenant "$AZURE_TENANT_ID"; then
16+
echo "❌ Error: Failed to login using Managed Identity."
17+
exit 1
18+
fi
19+
20+
echo "🔄 Validating required environment variables..."
21+
if [[ -z "$SUBSCRIPTION_ID" || -z "$GPT_MIN_CAPACITY" || -z "$TEXT_EMBEDDING_MIN_CAPACITY" ]]; then
22+
echo "❌ ERROR: Missing required environment variables."
23+
exit 1
24+
fi
25+
26+
echo "🔄 Setting Azure subscription..."
27+
if ! az account set --subscription "$SUBSCRIPTION_ID"; then
28+
echo "❌ ERROR: Invalid subscription ID or insufficient permissions."
29+
exit 1
30+
fi
31+
echo "✅ Azure subscription set successfully."
32+
33+
# Define models and their minimum required capacities
34+
declare -A MIN_CAPACITY=(
35+
["OpenAI.Standard.gpt-4o"]=$GPT_MIN_CAPACITY
36+
["OpenAI.Standard.text-embedding-ada-002"]=$TEXT_EMBEDDING_MIN_CAPACITY
37+
)
38+
39+
VALID_REGION=""
40+
for REGION in "${REGIONS[@]}"; do
41+
echo "----------------------------------------"
42+
echo "🔍 Checking region: $REGION"
43+
44+
QUOTA_INFO=$(az cognitiveservices usage list --location "$REGION" --output json)
45+
if [ -z "$QUOTA_INFO" ]; then
46+
echo "⚠️ WARNING: Failed to retrieve quota for region $REGION. Skipping."
47+
continue
48+
fi
49+
50+
INSUFFICIENT_QUOTA=false
51+
for MODEL in "${!MIN_CAPACITY[@]}"; do
52+
MODEL_INFO=$(echo "$QUOTA_INFO" | awk -v model="\"value\": \"$MODEL\"" '
53+
BEGIN { RS="},"; FS="," }
54+
$0 ~ model { print $0 }
55+
')
56+
57+
if [ -z "$MODEL_INFO" ]; then
58+
echo "⚠️ WARNING: No quota information found for model: $MODEL in $REGION. Skipping."
59+
continue
60+
fi
61+
62+
CURRENT_VALUE=$(echo "$MODEL_INFO" | awk -F': ' '/"currentValue"/ {print $2}' | tr -d ',' | tr -d ' ')
63+
LIMIT=$(echo "$MODEL_INFO" | awk -F': ' '/"limit"/ {print $2}' | tr -d ',' | tr -d ' ')
64+
65+
CURRENT_VALUE=${CURRENT_VALUE:-0}
66+
LIMIT=${LIMIT:-0}
67+
68+
CURRENT_VALUE=$(echo "$CURRENT_VALUE" | cut -d'.' -f1)
69+
LIMIT=$(echo "$LIMIT" | cut -d'.' -f1)
70+
71+
AVAILABLE=$((LIMIT - CURRENT_VALUE))
72+
73+
echo "✅ Model: $MODEL | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE"
74+
75+
if [ "$AVAILABLE" -lt "${MIN_CAPACITY[$MODEL]}" ]; then
76+
echo "❌ ERROR: $MODEL in $REGION has insufficient quota."
77+
INSUFFICIENT_QUOTA=true
78+
break
79+
fi
80+
done
81+
82+
if [ "$INSUFFICIENT_QUOTA" = false ]; then
83+
VALID_REGION="$REGION"
84+
break
85+
fi
86+
87+
done
88+
89+
if [ -z "$VALID_REGION" ]; then
90+
echo "❌ No region with sufficient quota found. Blocking deployment."
91+
echo "QUOTA_FAILED=true" >> "$GITHUB_ENV"
92+
exit 0
93+
else
94+
echo "✅ Suggested Region: $VALID_REGION"
95+
echo "VALID_REGION=$VALID_REGION" >> "$GITHUB_ENV"
96+
exit 0
97+
fi

0 commit comments

Comments
 (0)