Skip to content

Commit 5207ebf

Browse files
Roopan-MicrosoftRoopan-Microsoft
authored
Merge pull request #201 from microsoft/task/ci-pipeline-quota-check-dkm
ci: implement quota check in CI Pipeline to ensure resource availability in supported region
2 parents 3f67a8d + c6115c4 commit 5207ebf

File tree

2 files changed

+168
-3
lines changed

2 files changed

+168
-3
lines changed

‎.github/workflows/CI.yml‎

Lines changed: 64 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,14 +56,75 @@ jobs:
5656
run: |
5757
$PSVersionTable.PSVersion
5858
59+
# Run Quota Check Script
60+
- name: Run Quota Check
61+
id: quota-check
62+
shell: pwsh
63+
run: |
64+
$ErrorActionPreference = "Stop" # Ensure that any error stops the pipeline
65+
66+
# Path to the PowerShell script for quota check
67+
$quotaCheckScript = "Deployment/checkquota.ps1"
68+
69+
# Check if the script exists and is executable (not needed for PowerShell like chmod)
70+
if (-not (Test-Path $quotaCheckScript)) {
71+
Write-Host "❌ Error: Quota check script not found."
72+
exit 1
73+
}
74+
75+
# Run the script
76+
.\Deployment\checkquota.ps1
77+
78+
# If the script fails, check for the failure message
79+
$quotaFailedMessage = "No region with sufficient quota found"
80+
$output = Get-Content "Deployment/checkquota.ps1"
81+
82+
if ($output -contains $quotaFailedMessage) {
83+
echo "QUOTA_FAILED=true" >> $GITHUB_ENV
84+
}
85+
env:
86+
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
87+
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
88+
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
89+
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
90+
GPT_MIN_CAPACITY: '130'
91+
TEXT_EMBEDDING_MIN_CAPACITY: '80'
92+
93+
94+
# Send Notification on Quota Failure
95+
- name: Send Notification on Quota Failure
96+
if: env.QUOTA_FAILED == 'true'
97+
shell: pwsh
98+
run: |
99+
$RUN_URL = "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
100+
101+
# Construct the email body
102+
$EMAIL_BODY = @"
103+
{
104+
"body": "<p>Dear Team,</p><p>The quota check has failed, and the pipeline cannot proceed.</p><p><strong>Build URL:</strong> $RUN_URL</p><p>Please take necessary action.</p><p>Best regards,<br>Your Automation Team</p>"
105+
}
106+
"@
107+
108+
# Send the notification
109+
try {
110+
$response = Invoke-RestMethod -Uri "${{ secrets.LOGIC_APP_URL }}" -Method Post -ContentType "application/json" -Body $EMAIL_BODY
111+
Write-Host "Notification sent successfully."
112+
} catch {
113+
Write-Host "❌ Failed to send notification."
114+
}
115+
116+
- name: Fail Pipeline if Quota Check Fails
117+
if: env.QUOTA_FAILED == 'true'
118+
run: exit 1
119+
59120
- name: Run Deployment Script with Input
60121
shell: pwsh
61122
run: |
62123
cd Deployment
63124
$input = @"
64125
${{ secrets.AZURE_SUBSCRIPTION_ID }}
65126
CanadaCentral
66-
WestUS3
127+
${{ env.VALID_REGION }}
67128
${{ secrets.EMAIL }}
68129
yes
69130
"@
@@ -136,8 +197,8 @@ jobs:
136197
Write-Host "Purging CognitiveService Account: $cognitiveservice_name"
137198
138199
# Construct resource IDs
139-
$openaiResourceId = "/subscriptions/$subscriptionId/providers/Microsoft.CognitiveServices/locations/westus3/resourceGroups/$resourceGroupName/deletedAccounts/$openai_name"
140-
$cognitiveResourceId = "/subscriptions/$subscriptionId/providers/Microsoft.CognitiveServices/locations/eastus/resourceGroups/$resourceGroupName/deletedAccounts/$cognitiveservice_name"
200+
$openaiResourceId = "/subscriptions/$subscriptionId/providers/Microsoft.CognitiveServices/locations/${{ env.VALID_REGION }}/resourceGroups/$resourceGroupName/deletedAccounts/$openai_name"
201+
$cognitiveResourceId = "/subscriptions/$subscriptionId/providers/Microsoft.CognitiveServices/locations/${{ env.VALID_REGION }}/resourceGroups/$resourceGroupName/deletedAccounts/$cognitiveservice_name"
141202
142203
# Debug: Print constructed resource IDs
143204
Write-Host "Command to purge OpenAI resource: az resource delete --ids `"$openaiResourceId`" --verbose"

‎Deployment/checkquota.ps1‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
# List of Azure regions to check for quota (update as needed)
2+
$REGIONS = @("WestUS3", "CanadaCentral", "eastus")
3+
4+
$SUBSCRIPTION_ID = $env:AZURE_SUBSCRIPTION_ID
5+
$GPT_MIN_CAPACITY = $env:GPT_MIN_CAPACITY
6+
$TEXT_EMBEDDING_MIN_CAPACITY = $env:TEXT_EMBEDDING_MIN_CAPACITY
7+
$AZURE_CLIENT_ID = $env:AZURE_CLIENT_ID
8+
$AZURE_TENANT_ID = $env:AZURE_TENANT_ID
9+
$AZURE_CLIENT_SECRET = $env:AZURE_CLIENT_SECRET
10+
11+
# Authenticate using Service Principal
12+
Write-Host "Authentication using Service Principal..."
13+
# Ensure Azure PowerShell module is installed and imported
14+
Install-Module -Name Az -AllowClobber -Force -Scope CurrentUser
15+
Import-Module Az
16+
17+
# Create a PSCredential object for authentication
18+
$creds = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $AZURE_CLIENT_ID, (ConvertTo-SecureString $AZURE_CLIENT_SECRET -AsPlainText -Force)
19+
20+
# Attempt to connect using Service Principal
21+
try {
22+
Connect-AzAccount -ServicePrincipal -TenantId $AZURE_TENANT_ID -Credential $creds
23+
} catch {
24+
Write-Host "❌ Error: Failed to authenticate using Service Principal. $_"
25+
exit 1
26+
}
27+
28+
Write-Host "🔄 Validating required environment variables..."
29+
if (-not $SUBSCRIPTION_ID -or -not $GPT_MIN_CAPACITY -or -not $TEXT_EMBEDDING_MIN_CAPACITY) {
30+
Write-Host "❌ ERROR: Missing required environment variables."
31+
exit 1
32+
}
33+
34+
Write-Host "🔄 Setting Azure subscription..."
35+
$setSubscriptionResult = Set-AzContext -SubscriptionId $SUBSCRIPTION_ID
36+
if ($setSubscriptionResult -eq $null) {
37+
Write-Host "❌ ERROR: Invalid subscription ID or insufficient permissions."
38+
exit 1
39+
}
40+
Write-Host "✅ Azure subscription set successfully."
41+
42+
# Define models and their minimum required capacities
43+
$MIN_CAPACITY = @{
44+
"OpenAI.Standard.gpt-4o-mini" = $GPT_MIN_CAPACITY
45+
"OpenAI.Standard.text-embedding-3-large" = $TEXT_EMBEDDING_MIN_CAPACITY
46+
}
47+
48+
$VALID_REGION = ""
49+
50+
foreach ($REGION in $REGIONS) {
51+
Write-Host "----------------------------------------"
52+
Write-Host "🔍 Checking region: $REGION"
53+
54+
# Get the Cognitive Services usage information for the region
55+
$QUOTA_INFO = Get-AzCognitiveServicesUsage -Location $REGION
56+
if (-not $QUOTA_INFO) {
57+
Write-Host "⚠️ WARNING: Failed to retrieve quota for region $REGION. Skipping."
58+
continue
59+
}
60+
61+
$INSUFFICIENT_QUOTA = $false
62+
63+
foreach ($MODEL in $MIN_CAPACITY.Keys) {
64+
65+
$MODEL_INFO = $QUOTA_INFO | Where-Object { $_.Name -eq $MODEL }
66+
67+
Write-Host "MODEL_INFO ID: $MODEL_INFO"
68+
Write-Host "QUOTA_INFO: $QUOTA_INFO"
69+
70+
if (-not $MODEL_INFO) {
71+
Write-Host "⚠️ WARNING: No quota information found for model: $MODEL in $REGION. Skipping."
72+
continue
73+
}
74+
75+
$CURRENT_VALUE = [int]$MODEL_INFO.CurrentValue
76+
$LIMIT = [int]$MODEL_INFO.Limit
77+
78+
$AVAILABLE = $LIMIT - $CURRENT_VALUE
79+
80+
Write-Host "✅ Model: $MODEL | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE"
81+
82+
if ($AVAILABLE -lt $MIN_CAPACITY[$MODEL]) {
83+
Write-Host "❌ ERROR: $MODEL in $REGION has insufficient quota."
84+
$INSUFFICIENT_QUOTA = $true
85+
break
86+
}
87+
}
88+
89+
if ($INSUFFICIENT_QUOTA -eq $false) {
90+
$VALID_REGION = $REGION
91+
break
92+
}
93+
94+
}
95+
96+
if (-not $VALID_REGION) {
97+
Write-Host "❌ No region with sufficient quota found. Blocking deployment."
98+
echo "QUOTA_FAILED=true" >> $env:GITHUB_ENV # Set QUOTA_FAILED for subsequent steps
99+
exit 0
100+
} else {
101+
Write-Host "✅ Suggested Region: $VALID_REGION"
102+
echo "VALID_REGION=$VALID_REGION" >> $env:GITHUB_ENV # Set VALID_REGION for subsequent steps
103+
exit 0
104+
}

0 commit comments

Comments
 (0)