Skip to content

Commit 3fb530a

Browse files
Merge pull request #250 from microsoft/azd-deploy-auto-quota-validation
feat: quota auto validation before deployment
2 parents 60d4b7f + ea8101d commit 3fb530a

File tree

7 files changed

+464
-2
lines changed

7 files changed

+464
-2
lines changed

azure.yaml

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,20 @@
11
# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json
22
name: multi-agent-custom-automation-engine-solution-accelerator
33
metadata:
4-
4+
5+
hooks:
6+
preprovision:
7+
posix:
8+
shell: sh
9+
run: >
10+
./infra/scripts/validate_model_deployment_quota.sh --subscription "$AZURE_SUBSCRIPTION_ID" --location "${AZURE_ENV_OPENAI_LOCATION:-swedencentral}" --models-parameter "aiModelDeployments"
11+
interactive: false
12+
continueOnError: false
13+
14+
windows:
15+
shell: pwsh
16+
run: >
17+
$location = if ($env:AZURE_ENV_OPENAI_LOCATION) { $env:AZURE_ENV_OPENAI_LOCATION } else { "swedencentral" };
18+
./infra/scripts/validate_model_deployment_quotas.ps1 -SubscriptionId $env:AZURE_SUBSCRIPTION_ID -Location $location -ModelsParameter "aiModelDeployments"
19+
interactive: false
20+
continueOnError: false

infra/main.parameters.json

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
{
2+
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#",
3+
"contentVersion": "1.0.0.0",
4+
"parameters": {
5+
"aiModelDeployments": {
6+
"value": [
7+
{
8+
"name": "gpt",
9+
"model": {
10+
"name": "gpt-4o",
11+
"version": "2024-08-06",
12+
"format": "OpenAI"
13+
},
14+
"sku": {
15+
"name": "GlobalStandard",
16+
"capacity": 140
17+
}
18+
}
19+
]
20+
},
21+
"environmentName": {
22+
"value": "${AZURE_ENV_NAME}"
23+
},
24+
"location": {
25+
"value": "${AZURE_LOCATION}"
26+
},
27+
"backendExists": {
28+
"value": "${SERVICE_BACKEND_RESOURCE_EXISTS=false}"
29+
},
30+
"backendDefinition": {
31+
"value": {
32+
"settings": [
33+
{
34+
"name": "",
35+
"value": "${VAR}",
36+
"_comment_name": "The name of the environment variable when running in Azure. If empty, ignored.",
37+
"_comment_value": "The value to provide. This can be a fixed literal, or an expression like ${VAR} to use the value of 'VAR' from the current environment."
38+
},
39+
{
40+
"name": "",
41+
"value": "${VAR_S}",
42+
"secret": true,
43+
"_comment_name": "The name of the environment variable when running in Azure. If empty, ignored.",
44+
"_comment_value": "The value to provide. This can be a fixed literal, or an expression like ${VAR_S} to use the value of 'VAR_S' from the current environment."
45+
}
46+
]
47+
}
48+
},
49+
"frontendExists": {
50+
"value": "${SERVICE_FRONTEND_RESOURCE_EXISTS=false}"
51+
},
52+
"frontendDefinition": {
53+
"value": {
54+
"settings": [
55+
{
56+
"name": "",
57+
"value": "${VAR}",
58+
"_comment_name": "The name of the environment variable when running in Azure. If empty, ignored.",
59+
"_comment_value": "The value to provide. This can be a fixed literal, or an expression like ${VAR} to use the value of 'VAR' from the current environment."
60+
},
61+
{
62+
"name": "",
63+
"value": "${VAR_S}",
64+
"secret": true,
65+
"_comment_name": "The name of the environment variable when running in Azure. If empty, ignored.",
66+
"_comment_value": "The value to provide. This can be a fixed literal, or an expression like ${VAR_S} to use the value of 'VAR_S' from the current environment."
67+
}
68+
]
69+
}
70+
},
71+
"principalId": {
72+
"value": "${AZURE_PRINCIPAL_ID}"
73+
}
74+
}
75+
}

infra/scripts/quota_check_params.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ az account set --subscription "$AZURE_SUBSCRIPTION_ID"
9292
echo "🎯 Active Subscription: $(az account show --query '[name, id]' --output tsv)"
9393

9494
# Default Regions to check (Comma-separated, now configurable)
95-
DEFAULT_REGIONS="eastus,uksouth,eastus2,northcentralus,swedencentral,westus,westus2,southcentralus,canadacentral"
95+
DEFAULT_REGIONS="australiaeast,eastus2,francecentral,japaneast,norwayeast,swedencentral,uksouth,westus"
9696
IFS=',' read -r -a DEFAULT_REGION_ARRAY <<< "$DEFAULT_REGIONS"
9797

9898
# Read parameters (if any)
Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#!/bin/bash
2+
3+
SUBSCRIPTION_ID=""
4+
LOCATION=""
5+
MODELS_PARAMETER=""
6+
7+
while [[ $# -gt 0 ]]; do
8+
case "$1" in
9+
--subscription)
10+
SUBSCRIPTION_ID="$2"
11+
shift 2
12+
;;
13+
--location)
14+
LOCATION="$2"
15+
shift 2
16+
;;
17+
--models-parameter)
18+
MODELS_PARAMETER="$2"
19+
shift 2
20+
;;
21+
*)
22+
echo "Unknown option: $1"
23+
exit 1
24+
;;
25+
esac
26+
done
27+
28+
# Verify all required parameters are provided and echo missing ones
29+
MISSING_PARAMS=()
30+
31+
if [[ -z "$SUBSCRIPTION_ID" ]]; then
32+
MISSING_PARAMS+=("subscription")
33+
fi
34+
35+
if [[ -z "$LOCATION" ]]; then
36+
MISSING_PARAMS+=("location")
37+
fi
38+
39+
if [[ -z "$MODELS_PARAMETER" ]]; then
40+
MISSING_PARAMS+=("models-parameter")
41+
fi
42+
43+
if [[ ${#MISSING_PARAMS[@]} -ne 0 ]]; then
44+
echo "❌ ERROR: Missing required parameters: ${MISSING_PARAMS[*]}"
45+
echo "Usage: $0 --subscription <SUBSCRIPTION_ID> --location <LOCATION> --models-parameter <MODELS_PARAMETER>"
46+
exit 1
47+
fi
48+
49+
aiModelDeployments=$(jq -c ".parameters.$MODELS_PARAMETER.value[]" ./infra/main.parameters.json)
50+
51+
if [ $? -ne 0 ]; then
52+
echo "Error: Failed to parse main.parameters.json. Ensure jq is installed and the JSON file is valid."
53+
exit 1
54+
fi
55+
56+
az account set --subscription "$SUBSCRIPTION_ID"
57+
echo "🎯 Active Subscription: $(az account show --query '[name, id]' --output tsv)"
58+
59+
quotaAvailable=true
60+
61+
while IFS= read -r deployment; do
62+
name=$(echo "$deployment" | jq -r '.name')
63+
model=$(echo "$deployment" | jq -r '.model.name')
64+
type=$(echo "$deployment" | jq -r '.sku.name')
65+
capacity=$(echo "$deployment" | jq -r '.sku.capacity')
66+
67+
echo "🔍 Validating model deployment: $name ..."
68+
./infra/scripts/validate_model_quota.sh --location "$LOCATION" --model "$model" --capacity $capacity --deployment-type $type
69+
70+
# Check if the script failed
71+
exit_code=$?
72+
if [ $exit_code -ne 0 ]; then
73+
if [ $exit_code -eq 2 ]; then
74+
# Skip printing any quota validation error — already handled inside the validation script
75+
exit 1
76+
fi
77+
echo "❌ ERROR: Quota validation failed for model deployment: $name"
78+
quotaAvailable=false
79+
fi
80+
done <<< "$(echo "$aiModelDeployments")"
81+
82+
if [ "$quotaAvailable" = false ]; then
83+
echo "❌ ERROR: One or more model deployments failed validation."
84+
exit 1
85+
else
86+
echo "✅ All model deployments passed quota validation successfully."
87+
exit 0
88+
fi
Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
param (
2+
[string]$SubscriptionId,
3+
[string]$Location,
4+
[string]$ModelsParameter
5+
)
6+
7+
# Verify all required parameters are provided
8+
$MissingParams = @()
9+
10+
if (-not $SubscriptionId) {
11+
$MissingParams += "subscription"
12+
}
13+
14+
if (-not $Location) {
15+
$MissingParams += "location"
16+
}
17+
18+
if (-not $ModelsParameter) {
19+
$MissingParams += "models-parameter"
20+
}
21+
22+
if ($MissingParams.Count -gt 0) {
23+
Write-Error "❌ ERROR: Missing required parameters: $($MissingParams -join ', ')"
24+
Write-Host "Usage: .\validate_model_deployment_quotas.ps1 -SubscriptionId <SUBSCRIPTION_ID> -Location <LOCATION> -ModelsParameter <MODELS_PARAMETER>"
25+
exit 1
26+
}
27+
28+
$JsonContent = Get-Content -Path "./infra/main.parameters.json" -Raw | ConvertFrom-Json
29+
30+
if (-not $JsonContent) {
31+
Write-Error "❌ ERROR: Failed to parse main.parameters.json. Ensure the JSON file is valid."
32+
exit 1
33+
}
34+
35+
$aiModelDeployments = $JsonContent.parameters.$ModelsParameter.value
36+
37+
if (-not $aiModelDeployments -or -not ($aiModelDeployments -is [System.Collections.IEnumerable])) {
38+
Write-Error "❌ ERROR: The specified property $ModelsParameter does not exist or is not an array."
39+
exit 1
40+
}
41+
42+
az account set --subscription $SubscriptionId
43+
Write-Host "🎯 Active Subscription: $(az account show --query '[name, id]' --output tsv)"
44+
45+
$QuotaAvailable = $true
46+
47+
foreach ($deployment in $aiModelDeployments) {
48+
$name = $deployment.name
49+
$model = $deployment.model.name
50+
$type = $deployment.sku.name
51+
$capacity = $deployment.sku.capacity
52+
53+
Write-Host "🔍 Validating model deployment: $name ..."
54+
& .\infra\scripts\validate_model_quota.ps1 -Location $Location -Model $model -Capacity $capacity -DeploymentType $type
55+
56+
# Check if the script failed
57+
$exitCode = $LASTEXITCODE
58+
59+
if ($exitCode -ne 0) {
60+
if ($exitCode -eq 2) {
61+
# Quota error already printed inside the script, exit gracefully without reprinting
62+
exit 1
63+
}
64+
Write-Error "❌ ERROR: Quota validation failed for model deployment: $name"
65+
$QuotaAvailable = $false
66+
}
67+
}
68+
69+
if (-not $QuotaAvailable) {
70+
Write-Error "❌ ERROR: One or more model deployments failed validation."
71+
exit 1
72+
} else {
73+
Write-Host "✅ All model deployments passed quota validation successfully."
74+
exit 0
75+
}
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
param (
2+
[string]$Location,
3+
[string]$Model,
4+
[string]$DeploymentType = "Standard",
5+
[int]$Capacity
6+
)
7+
8+
# Verify required parameters
9+
$MissingParams = @()
10+
if (-not $Location) { $MissingParams += "location" }
11+
if (-not $Model) { $MissingParams += "model" }
12+
if (-not $Capacity) { $MissingParams += "capacity" }
13+
if (-not $DeploymentType) { $MissingParams += "deployment-type" }
14+
15+
if ($MissingParams.Count -gt 0) {
16+
Write-Error "❌ ERROR: Missing required parameters: $($MissingParams -join ', ')"
17+
Write-Host "Usage: .\validate_model_quota.ps1 -Location <LOCATION> -Model <MODEL> -Capacity <CAPACITY> [-DeploymentType <DEPLOYMENT_TYPE>]"
18+
exit 1
19+
}
20+
21+
if ($DeploymentType -ne "Standard" -and $DeploymentType -ne "GlobalStandard") {
22+
Write-Error "❌ ERROR: Invalid deployment type: $DeploymentType. Allowed values are 'Standard' or 'GlobalStandard'."
23+
exit 1
24+
}
25+
26+
$ModelType = "OpenAI.$DeploymentType.$Model"
27+
28+
$PreferredRegions = @('australiaeast', 'eastus2', 'francecentral', 'japaneast', 'norwayeast', 'swedencentral', 'uksouth', 'westus')
29+
$AllResults = @()
30+
31+
function Check-Quota {
32+
param (
33+
[string]$Region
34+
)
35+
36+
$ModelInfoRaw = az cognitiveservices usage list --location $Region --query "[?name.value=='$ModelType']" --output json
37+
$ModelInfo = $null
38+
39+
try {
40+
$ModelInfo = $ModelInfoRaw | ConvertFrom-Json
41+
} catch {
42+
return
43+
}
44+
45+
if (-not $ModelInfo) {
46+
return
47+
}
48+
49+
$CurrentValue = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).currentValue
50+
$Limit = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).limit
51+
52+
$CurrentValue = [int]($CurrentValue -replace '\.0+$', '')
53+
$Limit = [int]($Limit -replace '\.0+$', '')
54+
$Available = $Limit - $CurrentValue
55+
56+
$script:AllResults += [PSCustomObject]@{
57+
Region = $Region
58+
Model = $ModelType
59+
Limit = $Limit
60+
Used = $CurrentValue
61+
Available = $Available
62+
}
63+
}
64+
65+
foreach ($region in $PreferredRegions) {
66+
Check-Quota -Region $region
67+
}
68+
69+
# Display Results Table
70+
Write-Host "\n-------------------------------------------------------------------------------------------------------------"
71+
Write-Host "| No. | Region | Model Name | Limit | Used | Available |"
72+
Write-Host "-------------------------------------------------------------------------------------------------------------"
73+
74+
$count = 1
75+
foreach ($entry in $AllResults) {
76+
$index = $PreferredRegions.IndexOf($entry.Region) + 1
77+
$modelShort = $entry.Model.Substring($entry.Model.LastIndexOf(".") + 1)
78+
Write-Host ("| {0,-4} | {1,-16} | {2,-35} | {3,-7} | {4,-7} | {5,-9} |" -f $index, $entry.Region, $entry.Model, $entry.Limit, $entry.Used, $entry.Available)
79+
$count++
80+
}
81+
Write-Host "-------------------------------------------------------------------------------------------------------------"
82+
83+
$EligibleRegion = $AllResults | Where-Object { $_.Region -eq $Location -and $_.Available -ge $Capacity }
84+
if ($EligibleRegion) {
85+
Write-Host "\n✅ Sufficient quota found in original region '$Location'."
86+
exit 0
87+
}
88+
89+
$FallbackRegions = $AllResults | Where-Object { $_.Region -ne $Location -and $_.Available -ge $Capacity }
90+
91+
if ($FallbackRegions.Count -gt 0) {
92+
Write-Host "`n❌ Deployment cannot proceed because the original region '$Location' lacks sufficient quota."
93+
Write-Host "➡️ You can retry using one of the following regions with sufficient quota:`n"
94+
95+
foreach ($region in $FallbackRegions) {
96+
Write-Host "$($region.Region) (Available: $($region.Available))"
97+
}
98+
99+
Write-Host "`n🔧 To proceed, run:"
100+
Write-Host " azd env set AZURE_ENV_OPENAI_LOCATION '<region>'"
101+
Write-Host "📌 To confirm it's set correctly, run:"
102+
Write-Host " azd env get-value AZURE_ENV_OPENAI_LOCATION"
103+
Write-Host "▶️ Once confirmed, re-run azd up to deploy the model in the new region."
104+
exit 2
105+
}
106+
107+
Write-Error "❌ ERROR: No available quota found in any region."
108+
exit 1

0 commit comments

Comments
 (0)