From 449e6b21c52831c4d01289a7c24978169a4bfaca Mon Sep 17 00:00:00 2001 From: Priyanka-Microsoft Date: Wed, 4 Jun 2025 18:33:38 +0530 Subject: [PATCH 1/8] quota auto validation --- azure.yaml | 20 +++- .../validate_model_deployment_quota.sh | 83 +++++++++++++++++ .../validate_model_deployment_quotas.ps1 | 69 ++++++++++++++ infra/scripts/validate_model_quota.ps1 | 66 +++++++++++++ infra/scripts/validate_model_quota.sh | 93 +++++++++++++++++++ 5 files changed, 329 insertions(+), 2 deletions(-) create mode 100644 infra/scripts/validate_model_deployment_quota.sh create mode 100644 infra/scripts/validate_model_deployment_quotas.ps1 create mode 100644 infra/scripts/validate_model_quota.ps1 create mode 100644 infra/scripts/validate_model_quota.sh diff --git a/azure.yaml b/azure.yaml index ee4810b1..b8195cf2 100644 --- a/azure.yaml +++ b/azure.yaml @@ -1,4 +1,20 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json name: multi-agent-custom-automation-engine-solution-accelerator metadata: - template: multi-agent-custom-automation-engine-solution-accelerator@1.0 \ No newline at end of file + template: multi-agent-custom-automation-engine-solution-accelerator@1.0 + +hooks: + preprovision: + posix: + shell: sh + run: > + ./infra/scripts/validate_model_deployment_quota.sh --subscription "$AZURE_SUBSCRIPTION_ID" --location "${AZURE_OPENAI_LOCATION:-swedencentral}" --models-parameter "aiModelDeployments" + interactive: false + continueOnError: false + + windows: + shell: pwsh + run: > + $location = if ($env:AZURE_OPENAI_LOCATION) { $env:AZURE_OPENAI_LOCATION } else { "swedencentral" }; + ./infra/scripts/validate_model_deployment_quotas.ps1 -SubscriptionId $env:AZURE_SUBSCRIPTION_ID -Location $location -ModelsParameter "aiModelDeployments" + interactive: false + continueOnError: false diff --git a/infra/scripts/validate_model_deployment_quota.sh b/infra/scripts/validate_model_deployment_quota.sh new file mode 100644 index 00000000..4d4277c7 --- /dev/null +++ b/infra/scripts/validate_model_deployment_quota.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +SUBSCRIPTION_ID="" +LOCATION="" +MODELS_PARAMETER="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --subscription) + SUBSCRIPTION_ID="$2" + shift 2 + ;; + --location) + LOCATION="$2" + shift 2 + ;; + --models-parameter) + MODELS_PARAMETER="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +# Verify all required parameters are provided and echo missing ones +MISSING_PARAMS=() + +if [[ -z "$SUBSCRIPTION_ID" ]]; then + MISSING_PARAMS+=("subscription") +fi + +if [[ -z "$LOCATION" ]]; then + MISSING_PARAMS+=("location") +fi + +if [[ -z "$MODELS_PARAMETER" ]]; then + MISSING_PARAMS+=("models-parameter") +fi + +if [[ ${#MISSING_PARAMS[@]} -ne 0 ]]; then + echo "āŒ ERROR: Missing required parameters: ${MISSING_PARAMS[*]}" + echo "Usage: $0 --subscription --location --models-parameter " + exit 1 +fi + +aiModelDeployments=$(jq -c ".parameters.$MODELS_PARAMETER.value[]" ./infra/main.parameters.json) + +if [ $? -ne 0 ]; then + echo "Error: Failed to parse main.parameters.json. Ensure jq is installed and the JSON file is valid." + exit 1 +fi + +az account set --subscription "$SUBSCRIPTION_ID" +echo "šŸŽÆ Active Subscription: $(az account show --query '[name, id]' --output tsv)" + +quotaAvailable=true + +while IFS= read -r deployment; do + name=$(echo "$deployment" | jq -r '.name') + model=$(echo "$deployment" | jq -r '.model.name') + type=$(echo "$deployment" | jq -r '.sku.name') + capacity=$(echo "$deployment" | jq -r '.sku.capacity') + + echo "šŸ” Validating model deployment: $name ..." + ./infra/scripts/validate_model_quota.sh --location "$LOCATION" --model "$model" --capacity $capacity --deployment-type $type + + # Check if the script failed + if [ $? -ne 0 ]; then + echo "āŒ ERROR: Quota validation failed for model deployment: $name" + quotaAvailable=false + fi +done <<< "$(echo "$aiModelDeployments")" + +if [ "$quotaAvailable" = false ]; then + echo "āŒ ERROR: One or more model deployments failed validation." + exit 1 +else + echo "āœ… All model deployments passed quota validation successfully." + exit 0 +fi \ No newline at end of file diff --git a/infra/scripts/validate_model_deployment_quotas.ps1 b/infra/scripts/validate_model_deployment_quotas.ps1 new file mode 100644 index 00000000..dfe5f7ef --- /dev/null +++ b/infra/scripts/validate_model_deployment_quotas.ps1 @@ -0,0 +1,69 @@ +param ( + [string]$SubscriptionId, + [string]$Location, + [string]$ModelsParameter +) + +# Verify all required parameters are provided +$MissingParams = @() + +if (-not $SubscriptionId) { + $MissingParams += "subscription" +} + +if (-not $Location) { + $MissingParams += "location" +} + +if (-not $ModelsParameter) { + $MissingParams += "models-parameter" +} + +if ($MissingParams.Count -gt 0) { + Write-Error "āŒ ERROR: Missing required parameters: $($MissingParams -join ', ')" + Write-Host "Usage: .\validate_model_deployment_quotas.ps1 -SubscriptionId -Location -ModelsParameter " + exit 1 +} + +$JsonContent = Get-Content -Path "./infra/main.parameters.json" -Raw | ConvertFrom-Json + +if (-not $JsonContent) { + Write-Error "āŒ ERROR: Failed to parse main.parameters.json. Ensure the JSON file is valid." + exit 1 +} + +$aiModelDeployments = $JsonContent.parameters.$ModelsParameter.value + +if (-not $aiModelDeployments -or -not ($aiModelDeployments -is [System.Collections.IEnumerable])) { + Write-Error "āŒ ERROR: The specified property $ModelsParameter does not exist or is not an array." + exit 1 +} + +az account set --subscription $SubscriptionId +Write-Host "šŸŽÆ Active Subscription: $(az account show --query '[name, id]' --output tsv)" + +$QuotaAvailable = $true + +foreach ($deployment in $aiModelDeployments) { + $name = $deployment.name + $model = $deployment.model.name + $type = $deployment.sku.name + $capacity = $deployment.sku.capacity + + Write-Host "šŸ” Validating model deployment: $name ..." + & .\infra\scripts\validate_model_quota.ps1 -Location $Location -Model $model -Capacity $capacity -DeploymentType $type + + # Check if the script failed + if ($LASTEXITCODE -ne 0) { + Write-Error "āŒ ERROR: Quota validation failed for model deployment: $name" + $QuotaAvailable = $false + } +} + +if (-not $QuotaAvailable) { + Write-Error "āŒ ERROR: One or more model deployments failed validation." + exit 1 +} else { + Write-Host "āœ… All model deployments passed quota validation successfully." + exit 0 +} \ No newline at end of file diff --git a/infra/scripts/validate_model_quota.ps1 b/infra/scripts/validate_model_quota.ps1 new file mode 100644 index 00000000..eda43ae7 --- /dev/null +++ b/infra/scripts/validate_model_quota.ps1 @@ -0,0 +1,66 @@ +param ( + [string]$Location, + [string]$Model, + [string]$DeploymentType = "Standard", + [int]$Capacity +) + +# Verify all required parameters are provided +$MissingParams = @() + +if (-not $Location) { + $MissingParams += "location" +} + +if (-not $Model) { + $MissingParams += "model" +} + +if (-not $Capacity) { + $MissingParams += "capacity" +} + +if (-not $DeploymentType) { + $MissingParams += "deployment-type" +} + +if ($MissingParams.Count -gt 0) { + Write-Error "āŒ ERROR: Missing required parameters: $($MissingParams -join ', ')" + Write-Host "Usage: .\validate_model_quota.ps1 -Location -Model -Capacity [-DeploymentType ]" + exit 1 +} + +if ($DeploymentType -ne "Standard" -and $DeploymentType -ne "GlobalStandard") { + Write-Error "āŒ ERROR: Invalid deployment type: $DeploymentType. Allowed values are 'Standard' or 'GlobalStandard'." + exit 1 +} + +$ModelType = "OpenAI.$DeploymentType.$Model" + +Write-Host "šŸ” Checking quota for $ModelType in $Location ..." + +# Get model quota information +$ModelInfo = az cognitiveservices usage list --location $Location --query "[?name.value=='$ModelType']" --output json | ConvertFrom-Json + +if (-not $ModelInfo) { + Write-Error "āŒ ERROR: No quota information found for model: $Model in location: $Location for model type: $ModelType." + exit 1 +} + +if ($ModelInfo) { + $CurrentValue = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).currentValue + $Limit = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).limit + + $CurrentValue = [int]($CurrentValue -replace '\.0+$', '') # Remove decimals + $Limit = [int]($Limit -replace '\.0+$', '') # Remove decimals + + $Available = $Limit - $CurrentValue + Write-Host "āœ… Model available - Model: $ModelType | Used: $CurrentValue | Limit: $Limit | Available: $Available" + + if ($Available -lt $Capacity) { + Write-Error "āŒ ERROR: Insufficient quota for model: $Model in location: $Location. Available: $Available, Requested: $Capacity." + exit 1 + } else { + Write-Host "āœ… Sufficient quota for model: $Model in location: $Location. Available: $Available, Requested: $Capacity." + } +} \ No newline at end of file diff --git a/infra/scripts/validate_model_quota.sh b/infra/scripts/validate_model_quota.sh new file mode 100644 index 00000000..31090709 --- /dev/null +++ b/infra/scripts/validate_model_quota.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +LOCATION="" +MODEL="" +DEPLOYMENT_TYPE="Standard" +CAPACITY=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --model) + MODEL="$2" + shift 2 + ;; + --capacity) + CAPACITY="$2" + shift 2 + ;; + --deployment-type) + DEPLOYMENT_TYPE="$2" + shift 2 + ;; + --location) + LOCATION="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +# Verify all required parameters are provided and echo missing ones +MISSING_PARAMS=() + +if [[ -z "$LOCATION" ]]; then + MISSING_PARAMS+=("location") +fi + +if [[ -z "$MODEL" ]]; then + MISSING_PARAMS+=("model") +fi + +if [[ -z "$CAPACITY" ]]; then + MISSING_PARAMS+=("capacity") +fi + +if [[ -z "$DEPLOYMENT_TYPE" ]]; then + MISSING_PARAMS+=("deployment-type") +fi + +if [[ ${#MISSING_PARAMS[@]} -ne 0 ]]; then + echo "āŒ ERROR: Missing required parameters: ${MISSING_PARAMS[*]}" + echo "Usage: $0 --location --model --capacity [--deployment-type ]" + exit 1 +fi + +if [[ "$DEPLOYMENT_TYPE" != "Standard" && "$DEPLOYMENT_TYPE" != "GlobalStandard" ]]; then + echo "āŒ ERROR: Invalid deployment type: $DEPLOYMENT_TYPE. Allowed values are 'Standard' or 'GlobalStandard'." + exit 1 +fi + +MODEL_TYPE="OpenAI.$DEPLOYMENT_TYPE.$MODEL" + +echo "šŸ” Checking quota for $MODEL_TYPE in $LOCATION ..." + +MODEL_INFO=$(az cognitiveservices usage list --location "$LOCATION" --query "[?name.value=='$MODEL_TYPE']" --output json | tr '[:upper:]' '[:lower:]') + +if [ -z "$MODEL_INFO" ]; then + echo "āŒ ERROR: No quota information found for model: $MODEL in location: $LOCATION for model type: $MODEL_TYPE." + exit 1 +fi + +if [ -n "$MODEL_INFO" ]; then + CURRENT_VALUE=$(echo "$MODEL_INFO" | awk -F': ' '/"currentvalue"/ {print $2}' | tr -d ',' | tr -d ' ') + LIMIT=$(echo "$MODEL_INFO" | awk -F': ' '/"limit"/ {print $2}' | tr -d ',' | tr -d ' ') + + CURRENT_VALUE=${CURRENT_VALUE:-0} + LIMIT=${LIMIT:-0} + + CURRENT_VALUE=$(echo "$CURRENT_VALUE" | cut -d'.' -f1) + LIMIT=$(echo "$LIMIT" | cut -d'.' -f1) + + AVAILABLE=$((LIMIT - CURRENT_VALUE)) + echo "āœ… Model available - Model: $MODEL_TYPE | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE" + + if [ "$AVAILABLE" -lt "$CAPACITY" ]; then + echo "āŒ ERROR: Insufficient quota for model: $MODEL in location: $LOCATION. Available: $AVAILABLE, Requested: $CAPACITY." + exit 1 + else + echo "āœ… Sufficient quota for model: $MODEL in location: $LOCATION. Available: $AVAILABLE, Requested: $CAPACITY." + fi +fi \ No newline at end of file From 32aa1f0ad7a5b3faeb1154481a4ce6e8b3a788d2 Mon Sep 17 00:00:00 2001 From: Priyanka-Microsoft Date: Wed, 4 Jun 2025 19:28:19 +0530 Subject: [PATCH 2/8] quota auto validation --- infra/main.parameters.json | 16 +++++ infra/scripts/validate_model_quota.ps1 | 90 +++++++++++++++-------- infra/scripts/validate_model_quota.sh | 99 +++++++++++++++----------- 3 files changed, 131 insertions(+), 74 deletions(-) diff --git a/infra/main.parameters.json b/infra/main.parameters.json index c7fc26a4..e57dcbfb 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -2,6 +2,22 @@ "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", "contentVersion": "1.0.0.0", "parameters": { + "aiModelDeployments": { + "value": [ + { + "name": "gpt", + "model": { + "name": "gpt-4o", + "version": "2024-08-06", + "format": "OpenAI" + }, + "sku": { + "name": "GlobalStandard", + "capacity": 140 + } + } + ] + }, "environmentName": { "value": "${AZURE_ENV_NAME}" }, diff --git a/infra/scripts/validate_model_quota.ps1 b/infra/scripts/validate_model_quota.ps1 index eda43ae7..f96bb29b 100644 --- a/infra/scripts/validate_model_quota.ps1 +++ b/infra/scripts/validate_model_quota.ps1 @@ -5,24 +5,12 @@ param ( [int]$Capacity ) -# Verify all required parameters are provided +# Verify required parameters $MissingParams = @() - -if (-not $Location) { - $MissingParams += "location" -} - -if (-not $Model) { - $MissingParams += "model" -} - -if (-not $Capacity) { - $MissingParams += "capacity" -} - -if (-not $DeploymentType) { - $MissingParams += "deployment-type" -} +if (-not $Location) { $MissingParams += "location" } +if (-not $Model) { $MissingParams += "model" } +if (-not $Capacity) { $MissingParams += "capacity" } +if (-not $DeploymentType) { $MissingParams += "deployment-type" } if ($MissingParams.Count -gt 0) { Write-Error "āŒ ERROR: Missing required parameters: $($MissingParams -join ', ')" @@ -37,17 +25,28 @@ if ($DeploymentType -ne "Standard" -and $DeploymentType -ne "GlobalStandard") { $ModelType = "OpenAI.$DeploymentType.$Model" -Write-Host "šŸ” Checking quota for $ModelType in $Location ..." +function Check-Quota { + param ( + [string]$Region + ) -# Get model quota information -$ModelInfo = az cognitiveservices usage list --location $Location --query "[?name.value=='$ModelType']" --output json | ConvertFrom-Json + Write-Host "`nšŸ” Checking quota for $ModelType in $Region ..." -if (-not $ModelInfo) { - Write-Error "āŒ ERROR: No quota information found for model: $Model in location: $Location for model type: $ModelType." - exit 1 -} + $ModelInfoRaw = az cognitiveservices usage list --location $Region --query "[?name.value=='$ModelType']" --output json + $ModelInfo = $null + + try { + $ModelInfo = $ModelInfoRaw | ConvertFrom-Json + } catch { + Write-Warning "āš ļø Failed to parse quota info for region: $Region" + return $false + } + + if (-not $ModelInfo) { + Write-Host "āš ļø No quota information found for $ModelType in $Region" + return $false + } -if ($ModelInfo) { $CurrentValue = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).currentValue $Limit = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).limit @@ -55,12 +54,41 @@ if ($ModelInfo) { $Limit = [int]($Limit -replace '\.0+$', '') # Remove decimals $Available = $Limit - $CurrentValue - Write-Host "āœ… Model available - Model: $ModelType | Used: $CurrentValue | Limit: $Limit | Available: $Available" + Write-Host "šŸ”Ž Model: $ModelType | Used: $CurrentValue | Limit: $Limit | Available: $Available" - if ($Available -lt $Capacity) { - Write-Error "āŒ ERROR: Insufficient quota for model: $Model in location: $Location. Available: $Available, Requested: $Capacity." - exit 1 + if ($Available -ge $Capacity) { + Write-Host "āœ… Sufficient quota in $Region" + return $true } else { - Write-Host "āœ… Sufficient quota for model: $Model in location: $Location. Available: $Available, Requested: $Capacity." + Write-Host "āŒ Insufficient quota in $Region (Available: $Available, Required: $Capacity)" + return $false + } +} + +# List of fallback regions (excluding the one already tried) +$PreferredRegions = @('australiaeast', 'eastus2', 'francecentral', 'japaneast', 'norwayeast', 'swedencentral', 'uksouth', 'westus') | Where-Object { $_ -ne $Location } + +# Try original location first +if (Check-Quota -Region $Location) { + exit 0 +} + +Write-Host "`nšŸ” Trying fallback regions for available quota..." + +foreach ($region in $PreferredRegions) { + if (Check-Quota -Region $region) { + Write-Host "šŸš« Deployment cannot proceed because the original region '$Location' lacks sufficient quota." + Write-Host "āž”ļø You can retry using the available region: '$region'" + Write-Host "šŸ”§ To proceed, update the 'AZURE_OPENAI_LOCATION' value in the 'main.bicepparam' file, then run the following command:" + Write-Host " azd env set AZURE_OPENAI_LOCATION '$region'" + + # Optional: update `.azure/env` (uncomment if needed) + # Add-Content ".azure/env" "`nAZURE_OPENAI_LOCATION=$region" + + # Exit with non-zero code to halt deployment + exit 2 } -} \ No newline at end of file +} + +Write-Error "āŒ ERROR: No available quota found in any region." +exit 1 diff --git a/infra/scripts/validate_model_quota.sh b/infra/scripts/validate_model_quota.sh index 31090709..90fa3590 100644 --- a/infra/scripts/validate_model_quota.sh +++ b/infra/scripts/validate_model_quota.sh @@ -5,6 +5,8 @@ MODEL="" DEPLOYMENT_TYPE="Standard" CAPACITY=0 +ALL_REGIONS=('australiaeast' 'eastus2' 'francecentral' 'japaneast' 'norwayeast' 'swedencentral' 'uksouth' 'westus') + while [[ $# -gt 0 ]]; do case "$1" in --model) @@ -30,64 +32,75 @@ while [[ $# -gt 0 ]]; do esac done -# Verify all required parameters are provided and echo missing ones +# Validate required params MISSING_PARAMS=() +[[ -z "$LOCATION" ]] && MISSING_PARAMS+=("location") +[[ -z "$MODEL" ]] && MISSING_PARAMS+=("model") +[[ -z "$CAPACITY" ]] && MISSING_PARAMS+=("capacity") -if [[ -z "$LOCATION" ]]; then - MISSING_PARAMS+=("location") +if [[ ${#MISSING_PARAMS[@]} -ne 0 ]]; then + echo "āŒ ERROR: Missing required parameters: ${MISSING_PARAMS[*]}" + echo "Usage: $0 --location --model --capacity [--deployment-type ]" + exit 1 fi -if [[ -z "$MODEL" ]]; then - MISSING_PARAMS+=("model") +if [[ "$DEPLOYMENT_TYPE" != "Standard" && "$DEPLOYMENT_TYPE" != "GlobalStandard" ]]; then + echo "āŒ ERROR: Invalid deployment type: $DEPLOYMENT_TYPE. Allowed values are 'Standard' or 'GlobalStandard'." + exit 1 fi -if [[ -z "$CAPACITY" ]]; then - MISSING_PARAMS+=("capacity") -fi +MODEL_TYPE="OpenAI.$DEPLOYMENT_TYPE.$MODEL" -if [[ -z "$DEPLOYMENT_TYPE" ]]; then - MISSING_PARAMS+=("deployment-type") -fi +check_quota() { + local region="$1" + echo "šŸ” Checking quota for $MODEL_TYPE in $region ..." -if [[ ${#MISSING_PARAMS[@]} -ne 0 ]]; then - echo "āŒ ERROR: Missing required parameters: ${MISSING_PARAMS[*]}" - echo "Usage: $0 --location --model --capacity [--deployment-type ]" - exit 1 -fi + MODEL_INFO=$(az cognitiveservices usage list --location "$region" --query "[?name.value=='$MODEL_TYPE']" --output json 2>/dev/null) -if [[ "$DEPLOYMENT_TYPE" != "Standard" && "$DEPLOYMENT_TYPE" != "GlobalStandard" ]]; then - echo "āŒ ERROR: Invalid deployment type: $DEPLOYMENT_TYPE. Allowed values are 'Standard' or 'GlobalStandard'." - exit 1 -fi + if [[ -z "$MODEL_INFO" || "$MODEL_INFO" == "[]" ]]; then + echo "āš ļø No quota info found for $MODEL_TYPE in $region" + return 1 + fi -MODEL_TYPE="OpenAI.$DEPLOYMENT_TYPE.$MODEL" + CURRENT_VALUE=$(echo "$MODEL_INFO" | jq -r '.[0].currentValue // 0' | cut -d'.' -f1) + LIMIT=$(echo "$MODEL_INFO" | jq -r '.[0].limit // 0' | cut -d'.' -f1) + AVAILABLE=$((LIMIT - CURRENT_VALUE)) -echo "šŸ” Checking quota for $MODEL_TYPE in $LOCATION ..." + echo "šŸ”Ž Model: $MODEL_TYPE | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE" -MODEL_INFO=$(az cognitiveservices usage list --location "$LOCATION" --query "[?name.value=='$MODEL_TYPE']" --output json | tr '[:upper:]' '[:lower:]') + if (( AVAILABLE >= CAPACITY )); then + echo "āœ… Sufficient quota in $region" + return 0 + else + echo "āŒ Insufficient quota in $region (Available: $AVAILABLE, Required: $CAPACITY)" + return 1 + fi +} -if [ -z "$MODEL_INFO" ]; then - echo "āŒ ERROR: No quota information found for model: $MODEL in location: $LOCATION for model type: $MODEL_TYPE." - exit 1 +# Try user-provided region +if check_quota "$LOCATION"; then + exit 0 fi -if [ -n "$MODEL_INFO" ]; then - CURRENT_VALUE=$(echo "$MODEL_INFO" | awk -F': ' '/"currentvalue"/ {print $2}' | tr -d ',' | tr -d ' ') - LIMIT=$(echo "$MODEL_INFO" | awk -F': ' '/"limit"/ {print $2}' | tr -d ',' | tr -d ' ') - - CURRENT_VALUE=${CURRENT_VALUE:-0} - LIMIT=${LIMIT:-0} +# Try fallback regions +REMAINING_REGIONS=() +for region in "${ALL_REGIONS[@]}"; do + if [[ "$region" != "$LOCATION" ]]; then + REMAINING_REGIONS+=("$region") + fi +done - CURRENT_VALUE=$(echo "$CURRENT_VALUE" | cut -d'.' -f1) - LIMIT=$(echo "$LIMIT" | cut -d'.' -f1) +echo "šŸ” Trying fallback regions for available quota..." - AVAILABLE=$((LIMIT - CURRENT_VALUE)) - echo "āœ… Model available - Model: $MODEL_TYPE | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE" +for region in "${REMAINING_REGIONS[@]}"; do + if check_quota "$region"; then + echo "šŸš« Deployment cannot proceed because the original region '$LOCATION' lacks sufficient quota." + echo "āž”ļø You can retry using the available region: '$region'" + echo "šŸ”§ To proceed, update the 'AZURE_OPENAI_LOCATION' value in the 'main.bicepparam' file, then run:" + echo " azd env set AZURE_OPENAI_LOCATION '$region'" + exit 1 + fi +done - if [ "$AVAILABLE" -lt "$CAPACITY" ]; then - echo "āŒ ERROR: Insufficient quota for model: $MODEL in location: $LOCATION. Available: $AVAILABLE, Requested: $CAPACITY." - exit 1 - else - echo "āœ… Sufficient quota for model: $MODEL in location: $LOCATION. Available: $AVAILABLE, Requested: $CAPACITY." - fi -fi \ No newline at end of file +echo "āŒ ERROR: No available quota found in any of the fallback regions." +exit 1 From c3600a92ff9dc01cc594b213e1ccc365a04b7bf7 Mon Sep 17 00:00:00 2001 From: Priyanka-Microsoft Date: Thu, 5 Jun 2025 10:09:57 +0530 Subject: [PATCH 3/8] quota check region parameter updated --- infra/scripts/quota_check_params.sh | 498 ++++++++++++++-------------- 1 file changed, 249 insertions(+), 249 deletions(-) diff --git a/infra/scripts/quota_check_params.sh b/infra/scripts/quota_check_params.sh index add6ac47..1af5876e 100644 --- a/infra/scripts/quota_check_params.sh +++ b/infra/scripts/quota_check_params.sh @@ -1,249 +1,249 @@ -#!/bin/bash -# VERBOSE=false - -MODELS="" -REGIONS="" -VERBOSE=false - -while [[ $# -gt 0 ]]; do - case "$1" in - --models) - MODELS="$2" - shift 2 - ;; - --regions) - REGIONS="$2" - shift 2 - ;; - --verbose) - VERBOSE=true - shift - ;; - *) - echo "Unknown option: $1" - exit 1 - ;; - esac -done - -# Fallback to defaults if not provided -[[ -z "$MODELS" ]] -[[ -z "$REGIONS" ]] - -echo "Models: $MODELS" -echo "Regions: $REGIONS" -echo "Verbose: $VERBOSE" - -for arg in "$@"; do - if [ "$arg" = "--verbose" ]; then - VERBOSE=true - fi -done - -log_verbose() { - if [ "$VERBOSE" = true ]; then - echo "$1" - fi -} - -# Default Models and Capacities (Comma-separated in "model:capacity" format) -DEFAULT_MODEL_CAPACITY="gpt-4o:50" -# Convert the comma-separated string into an array -IFS=',' read -r -a MODEL_CAPACITY_PAIRS <<< "$DEFAULT_MODEL_CAPACITY" - -echo "šŸ”„ Fetching available Azure subscriptions..." -SUBSCRIPTIONS=$(az account list --query "[?state=='Enabled'].{Name:name, ID:id}" --output tsv) -SUB_COUNT=$(echo "$SUBSCRIPTIONS" | wc -l) - -if [ "$SUB_COUNT" -eq 0 ]; then - echo "āŒ ERROR: No active Azure subscriptions found. Please log in using 'az login' and ensure you have an active subscription." - exit 1 -elif [ "$SUB_COUNT" -eq 1 ]; then - # If only one subscription, automatically select it - AZURE_SUBSCRIPTION_ID=$(echo "$SUBSCRIPTIONS" | awk '{print $2}') - if [ -z "$AZURE_SUBSCRIPTION_ID" ]; then - echo "āŒ ERROR: No active Azure subscriptions found. Please log in using 'az login' and ensure you have an active subscription." - exit 1 - fi - echo "āœ… Using the only available subscription: $AZURE_SUBSCRIPTION_ID" -else - # If multiple subscriptions exist, prompt the user to choose one - echo "Multiple subscriptions found:" - echo "$SUBSCRIPTIONS" | awk '{print NR")", $1, "-", $2}' - - while true; do - echo "Enter the number of the subscription to use:" - read SUB_INDEX - - # Validate user input - if [[ "$SUB_INDEX" =~ ^[0-9]+$ ]] && [ "$SUB_INDEX" -ge 1 ] && [ "$SUB_INDEX" -le "$SUB_COUNT" ]; then - AZURE_SUBSCRIPTION_ID=$(echo "$SUBSCRIPTIONS" | awk -v idx="$SUB_INDEX" 'NR==idx {print $2}') - echo "āœ… Selected Subscription: $AZURE_SUBSCRIPTION_ID" - break - else - echo "āŒ Invalid selection. Please enter a valid number from the list." - fi - done -fi - - -# Set the selected subscription -az account set --subscription "$AZURE_SUBSCRIPTION_ID" -echo "šŸŽÆ Active Subscription: $(az account show --query '[name, id]' --output tsv)" - -# Default Regions to check (Comma-separated, now configurable) -DEFAULT_REGIONS="eastus,uksouth,eastus2,northcentralus,swedencentral,westus,westus2,southcentralus,canadacentral" -IFS=',' read -r -a DEFAULT_REGION_ARRAY <<< "$DEFAULT_REGIONS" - -# Read parameters (if any) -IFS=',' read -r -a USER_PROVIDED_PAIRS <<< "$MODELS" -USER_REGION="$REGIONS" - -IS_USER_PROVIDED_PAIRS=false - -if [ ${#USER_PROVIDED_PAIRS[@]} -lt 1 ]; then - echo "No parameters provided, using default model-capacity pairs: ${MODEL_CAPACITY_PAIRS[*]}" -else - echo "Using provided model and capacity pairs: ${USER_PROVIDED_PAIRS[*]}" - IS_USER_PROVIDED_PAIRS=true - MODEL_CAPACITY_PAIRS=("${USER_PROVIDED_PAIRS[@]}") -fi - -declare -a FINAL_MODEL_NAMES -declare -a FINAL_CAPACITIES -declare -a TABLE_ROWS - -for PAIR in "${MODEL_CAPACITY_PAIRS[@]}"; do - MODEL_NAME=$(echo "$PAIR" | cut -d':' -f1 | tr '[:upper:]' '[:lower:]') - CAPACITY=$(echo "$PAIR" | cut -d':' -f2) - - if [ -z "$MODEL_NAME" ] || [ -z "$CAPACITY" ]; then - echo "āŒ ERROR: Invalid model and capacity pair '$PAIR'. Both model and capacity must be specified." - exit 1 - fi - - FINAL_MODEL_NAMES+=("$MODEL_NAME") - FINAL_CAPACITIES+=("$CAPACITY") - -done - -echo "šŸ”„ Using Models: ${FINAL_MODEL_NAMES[*]} with respective Capacities: ${FINAL_CAPACITIES[*]}" -echo "----------------------------------------" - -# Check if the user provided a region, if not, use the default regions -if [ -n "$USER_REGION" ]; then - echo "šŸ” User provided region: $USER_REGION" - IFS=',' read -r -a REGIONS <<< "$USER_REGION" -else - echo "No region specified, using default regions: ${DEFAULT_REGION_ARRAY[*]}" - REGIONS=("${DEFAULT_REGION_ARRAY[@]}") - APPLY_OR_CONDITION=true -fi - -echo "āœ… Retrieved Azure regions. Checking availability..." -INDEX=1 - -VALID_REGIONS=() -for REGION in "${REGIONS[@]}"; do - log_verbose "----------------------------------------" - log_verbose "šŸ” Checking region: $REGION" - - QUOTA_INFO=$(az cognitiveservices usage list --location "$REGION" --output json | tr '[:upper:]' '[:lower:]') - if [ -z "$QUOTA_INFO" ]; then - log_verbose "āš ļø WARNING: Failed to retrieve quota for region $REGION. Skipping." - continue - fi - - TEXT_EMBEDDING_AVAILABLE=false - AT_LEAST_ONE_MODEL_AVAILABLE=false - TEMP_TABLE_ROWS=() - - for index in "${!FINAL_MODEL_NAMES[@]}"; do - MODEL_NAME="${FINAL_MODEL_NAMES[$index]}" - REQUIRED_CAPACITY="${FINAL_CAPACITIES[$index]}" - FOUND=false - INSUFFICIENT_QUOTA=false - - if [ "$MODEL_NAME" = "text-embedding-ada-002" ]; then - MODEL_TYPES=("openai.standard.$MODEL_NAME") - else - MODEL_TYPES=("openai.standard.$MODEL_NAME" "openai.globalstandard.$MODEL_NAME") - fi - - for MODEL_TYPE in "${MODEL_TYPES[@]}"; do - FOUND=false - INSUFFICIENT_QUOTA=false - log_verbose "šŸ” Checking model: $MODEL_NAME with required capacity: $REQUIRED_CAPACITY ($MODEL_TYPE)" - - MODEL_INFO=$(echo "$QUOTA_INFO" | awk -v model="\"value\": \"$MODEL_TYPE\"" ' - BEGIN { RS="},"; FS="," } - $0 ~ model { print $0 } - ') - - if [ -z "$MODEL_INFO" ]; then - FOUND=false - log_verbose "āš ļø WARNING: No quota information found for model: $MODEL_NAME in region: $REGION for model type: $MODEL_TYPE." - continue - fi - - if [ -n "$MODEL_INFO" ]; then - FOUND=true - CURRENT_VALUE=$(echo "$MODEL_INFO" | awk -F': ' '/"currentvalue"/ {print $2}' | tr -d ',' | tr -d ' ') - LIMIT=$(echo "$MODEL_INFO" | awk -F': ' '/"limit"/ {print $2}' | tr -d ',' | tr -d ' ') - - CURRENT_VALUE=${CURRENT_VALUE:-0} - LIMIT=${LIMIT:-0} - - CURRENT_VALUE=$(echo "$CURRENT_VALUE" | cut -d'.' -f1) - LIMIT=$(echo "$LIMIT" | cut -d'.' -f1) - - AVAILABLE=$((LIMIT - CURRENT_VALUE)) - log_verbose "āœ… Model: $MODEL_TYPE | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE" - - if [ "$AVAILABLE" -ge "$REQUIRED_CAPACITY" ]; then - FOUND=true - if [ "$MODEL_NAME" = "text-embedding-ada-002" ]; then - TEXT_EMBEDDING_AVAILABLE=true - fi - AT_LEAST_ONE_MODEL_AVAILABLE=true - TEMP_TABLE_ROWS+=("$(printf "| %-4s | %-20s | %-43s | %-10s | %-10s | %-10s |" "$INDEX" "$REGION" "$MODEL_TYPE" "$LIMIT" "$CURRENT_VALUE" "$AVAILABLE")") - else - INSUFFICIENT_QUOTA=true - fi - fi - - if [ "$FOUND" = false ]; then - log_verbose "āŒ No models found for model: $MODEL_NAME in region: $REGION (${MODEL_TYPES[*]})" - - elif [ "$INSUFFICIENT_QUOTA" = true ]; then - log_verbose "āš ļø Model $MODEL_NAME in region: $REGION has insufficient quota (${MODEL_TYPES[*]})." - fi - done - done - -if { [ "$IS_USER_PROVIDED_PAIRS" = true ] && [ "$INSUFFICIENT_QUOTA" = false ] && [ "$FOUND" = true ]; } || { [ "$APPLY_OR_CONDITION" != true ] || [ "$AT_LEAST_ONE_MODEL_AVAILABLE" = true ]; }; then - VALID_REGIONS+=("$REGION") - TABLE_ROWS+=("${TEMP_TABLE_ROWS[@]}") - INDEX=$((INDEX + 1)) - elif [ ${#USER_PROVIDED_PAIRS[@]} -eq 0 ]; then - echo "šŸš« Skipping $REGION as it does not meet quota requirements." - fi - -done - -if [ ${#TABLE_ROWS[@]} -eq 0 ]; then - echo "--------------------------------------------------------------------------------------------------------------------" - - echo "āŒ No regions have sufficient quota for all required models. Please request a quota increase: https://aka.ms/oai/stuquotarequest" -else - echo "---------------------------------------------------------------------------------------------------------------------" - printf "| %-4s | %-20s | %-43s | %-10s | %-10s | %-10s |\n" "No." "Region" "Model Name" "Limit" "Used" "Available" - echo "---------------------------------------------------------------------------------------------------------------------" - for ROW in "${TABLE_ROWS[@]}"; do - echo "$ROW" - done - echo "---------------------------------------------------------------------------------------------------------------------" - echo "āž”ļø To request a quota increase, visit: https://aka.ms/oai/stuquotarequest" -fi - -echo "āœ… Script completed." +#!/bin/bash +# VERBOSE=false + +MODELS="" +REGIONS="" +VERBOSE=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --models) + MODELS="$2" + shift 2 + ;; + --regions) + REGIONS="$2" + shift 2 + ;; + --verbose) + VERBOSE=true + shift + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +# Fallback to defaults if not provided +[[ -z "$MODELS" ]] +[[ -z "$REGIONS" ]] + +echo "Models: $MODELS" +echo "Regions: $REGIONS" +echo "Verbose: $VERBOSE" + +for arg in "$@"; do + if [ "$arg" = "--verbose" ]; then + VERBOSE=true + fi +done + +log_verbose() { + if [ "$VERBOSE" = true ]; then + echo "$1" + fi +} + +# Default Models and Capacities (Comma-separated in "model:capacity" format) +DEFAULT_MODEL_CAPACITY="gpt-4o:50" +# Convert the comma-separated string into an array +IFS=',' read -r -a MODEL_CAPACITY_PAIRS <<< "$DEFAULT_MODEL_CAPACITY" + +echo "šŸ”„ Fetching available Azure subscriptions..." +SUBSCRIPTIONS=$(az account list --query "[?state=='Enabled'].{Name:name, ID:id}" --output tsv) +SUB_COUNT=$(echo "$SUBSCRIPTIONS" | wc -l) + +if [ "$SUB_COUNT" -eq 0 ]; then + echo "āŒ ERROR: No active Azure subscriptions found. Please log in using 'az login' and ensure you have an active subscription." + exit 1 +elif [ "$SUB_COUNT" -eq 1 ]; then + # If only one subscription, automatically select it + AZURE_SUBSCRIPTION_ID=$(echo "$SUBSCRIPTIONS" | awk '{print $2}') + if [ -z "$AZURE_SUBSCRIPTION_ID" ]; then + echo "āŒ ERROR: No active Azure subscriptions found. Please log in using 'az login' and ensure you have an active subscription." + exit 1 + fi + echo "āœ… Using the only available subscription: $AZURE_SUBSCRIPTION_ID" +else + # If multiple subscriptions exist, prompt the user to choose one + echo "Multiple subscriptions found:" + echo "$SUBSCRIPTIONS" | awk '{print NR")", $1, "-", $2}' + + while true; do + echo "Enter the number of the subscription to use:" + read SUB_INDEX + + # Validate user input + if [[ "$SUB_INDEX" =~ ^[0-9]+$ ]] && [ "$SUB_INDEX" -ge 1 ] && [ "$SUB_INDEX" -le "$SUB_COUNT" ]; then + AZURE_SUBSCRIPTION_ID=$(echo "$SUBSCRIPTIONS" | awk -v idx="$SUB_INDEX" 'NR==idx {print $2}') + echo "āœ… Selected Subscription: $AZURE_SUBSCRIPTION_ID" + break + else + echo "āŒ Invalid selection. Please enter a valid number from the list." + fi + done +fi + + +# Set the selected subscription +az account set --subscription "$AZURE_SUBSCRIPTION_ID" +echo "šŸŽÆ Active Subscription: $(az account show --query '[name, id]' --output tsv)" + +# Default Regions to check (Comma-separated, now configurable) +DEFAULT_REGIONS="australiaeast,eastus2,francecentral,japaneast,norwayeast,swedencentral,uksouth,westus" +IFS=',' read -r -a DEFAULT_REGION_ARRAY <<< "$DEFAULT_REGIONS" + +# Read parameters (if any) +IFS=',' read -r -a USER_PROVIDED_PAIRS <<< "$MODELS" +USER_REGION="$REGIONS" + +IS_USER_PROVIDED_PAIRS=false + +if [ ${#USER_PROVIDED_PAIRS[@]} -lt 1 ]; then + echo "No parameters provided, using default model-capacity pairs: ${MODEL_CAPACITY_PAIRS[*]}" +else + echo "Using provided model and capacity pairs: ${USER_PROVIDED_PAIRS[*]}" + IS_USER_PROVIDED_PAIRS=true + MODEL_CAPACITY_PAIRS=("${USER_PROVIDED_PAIRS[@]}") +fi + +declare -a FINAL_MODEL_NAMES +declare -a FINAL_CAPACITIES +declare -a TABLE_ROWS + +for PAIR in "${MODEL_CAPACITY_PAIRS[@]}"; do + MODEL_NAME=$(echo "$PAIR" | cut -d':' -f1 | tr '[:upper:]' '[:lower:]') + CAPACITY=$(echo "$PAIR" | cut -d':' -f2) + + if [ -z "$MODEL_NAME" ] || [ -z "$CAPACITY" ]; then + echo "āŒ ERROR: Invalid model and capacity pair '$PAIR'. Both model and capacity must be specified." + exit 1 + fi + + FINAL_MODEL_NAMES+=("$MODEL_NAME") + FINAL_CAPACITIES+=("$CAPACITY") + +done + +echo "šŸ”„ Using Models: ${FINAL_MODEL_NAMES[*]} with respective Capacities: ${FINAL_CAPACITIES[*]}" +echo "----------------------------------------" + +# Check if the user provided a region, if not, use the default regions +if [ -n "$USER_REGION" ]; then + echo "šŸ” User provided region: $USER_REGION" + IFS=',' read -r -a REGIONS <<< "$USER_REGION" +else + echo "No region specified, using default regions: ${DEFAULT_REGION_ARRAY[*]}" + REGIONS=("${DEFAULT_REGION_ARRAY[@]}") + APPLY_OR_CONDITION=true +fi + +echo "āœ… Retrieved Azure regions. Checking availability..." +INDEX=1 + +VALID_REGIONS=() +for REGION in "${REGIONS[@]}"; do + log_verbose "----------------------------------------" + log_verbose "šŸ” Checking region: $REGION" + + QUOTA_INFO=$(az cognitiveservices usage list --location "$REGION" --output json | tr '[:upper:]' '[:lower:]') + if [ -z "$QUOTA_INFO" ]; then + log_verbose "āš ļø WARNING: Failed to retrieve quota for region $REGION. Skipping." + continue + fi + + TEXT_EMBEDDING_AVAILABLE=false + AT_LEAST_ONE_MODEL_AVAILABLE=false + TEMP_TABLE_ROWS=() + + for index in "${!FINAL_MODEL_NAMES[@]}"; do + MODEL_NAME="${FINAL_MODEL_NAMES[$index]}" + REQUIRED_CAPACITY="${FINAL_CAPACITIES[$index]}" + FOUND=false + INSUFFICIENT_QUOTA=false + + if [ "$MODEL_NAME" = "text-embedding-ada-002" ]; then + MODEL_TYPES=("openai.standard.$MODEL_NAME") + else + MODEL_TYPES=("openai.standard.$MODEL_NAME" "openai.globalstandard.$MODEL_NAME") + fi + + for MODEL_TYPE in "${MODEL_TYPES[@]}"; do + FOUND=false + INSUFFICIENT_QUOTA=false + log_verbose "šŸ” Checking model: $MODEL_NAME with required capacity: $REQUIRED_CAPACITY ($MODEL_TYPE)" + + MODEL_INFO=$(echo "$QUOTA_INFO" | awk -v model="\"value\": \"$MODEL_TYPE\"" ' + BEGIN { RS="},"; FS="," } + $0 ~ model { print $0 } + ') + + if [ -z "$MODEL_INFO" ]; then + FOUND=false + log_verbose "āš ļø WARNING: No quota information found for model: $MODEL_NAME in region: $REGION for model type: $MODEL_TYPE." + continue + fi + + if [ -n "$MODEL_INFO" ]; then + FOUND=true + CURRENT_VALUE=$(echo "$MODEL_INFO" | awk -F': ' '/"currentvalue"/ {print $2}' | tr -d ',' | tr -d ' ') + LIMIT=$(echo "$MODEL_INFO" | awk -F': ' '/"limit"/ {print $2}' | tr -d ',' | tr -d ' ') + + CURRENT_VALUE=${CURRENT_VALUE:-0} + LIMIT=${LIMIT:-0} + + CURRENT_VALUE=$(echo "$CURRENT_VALUE" | cut -d'.' -f1) + LIMIT=$(echo "$LIMIT" | cut -d'.' -f1) + + AVAILABLE=$((LIMIT - CURRENT_VALUE)) + log_verbose "āœ… Model: $MODEL_TYPE | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE" + + if [ "$AVAILABLE" -ge "$REQUIRED_CAPACITY" ]; then + FOUND=true + if [ "$MODEL_NAME" = "text-embedding-ada-002" ]; then + TEXT_EMBEDDING_AVAILABLE=true + fi + AT_LEAST_ONE_MODEL_AVAILABLE=true + TEMP_TABLE_ROWS+=("$(printf "| %-4s | %-20s | %-43s | %-10s | %-10s | %-10s |" "$INDEX" "$REGION" "$MODEL_TYPE" "$LIMIT" "$CURRENT_VALUE" "$AVAILABLE")") + else + INSUFFICIENT_QUOTA=true + fi + fi + + if [ "$FOUND" = false ]; then + log_verbose "āŒ No models found for model: $MODEL_NAME in region: $REGION (${MODEL_TYPES[*]})" + + elif [ "$INSUFFICIENT_QUOTA" = true ]; then + log_verbose "āš ļø Model $MODEL_NAME in region: $REGION has insufficient quota (${MODEL_TYPES[*]})." + fi + done + done + +if { [ "$IS_USER_PROVIDED_PAIRS" = true ] && [ "$INSUFFICIENT_QUOTA" = false ] && [ "$FOUND" = true ]; } || { [ "$APPLY_OR_CONDITION" != true ] || [ "$AT_LEAST_ONE_MODEL_AVAILABLE" = true ]; }; then + VALID_REGIONS+=("$REGION") + TABLE_ROWS+=("${TEMP_TABLE_ROWS[@]}") + INDEX=$((INDEX + 1)) + elif [ ${#USER_PROVIDED_PAIRS[@]} -eq 0 ]; then + echo "šŸš« Skipping $REGION as it does not meet quota requirements." + fi + +done + +if [ ${#TABLE_ROWS[@]} -eq 0 ]; then + echo "--------------------------------------------------------------------------------------------------------------------" + + echo "āŒ No regions have sufficient quota for all required models. Please request a quota increase: https://aka.ms/oai/stuquotarequest" +else + echo "---------------------------------------------------------------------------------------------------------------------" + printf "| %-4s | %-20s | %-43s | %-10s | %-10s | %-10s |\n" "No." "Region" "Model Name" "Limit" "Used" "Available" + echo "---------------------------------------------------------------------------------------------------------------------" + for ROW in "${TABLE_ROWS[@]}"; do + echo "$ROW" + done + echo "---------------------------------------------------------------------------------------------------------------------" + echo "āž”ļø To request a quota increase, visit: https://aka.ms/oai/stuquotarequest" +fi + +echo "āœ… Script completed." From 91d67a6df88cadc6434405c6d63227931d95d7e5 Mon Sep 17 00:00:00 2001 From: Priyanka-Microsoft Date: Thu, 5 Jun 2025 10:14:45 +0530 Subject: [PATCH 4/8] quota check region parameter updated --- infra/scripts/quota_check_params.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/scripts/quota_check_params.sh b/infra/scripts/quota_check_params.sh index 1af5876e..db378ef9 100644 --- a/infra/scripts/quota_check_params.sh +++ b/infra/scripts/quota_check_params.sh @@ -246,4 +246,4 @@ else echo "āž”ļø To request a quota increase, visit: https://aka.ms/oai/stuquotarequest" fi -echo "āœ… Script completed." +echo "āœ… Script completed." \ No newline at end of file From c9e541bd77339f5387d900b0a353f6553ba26f7b Mon Sep 17 00:00:00 2001 From: Priyanka-Microsoft Date: Thu, 5 Jun 2025 12:35:17 +0530 Subject: [PATCH 5/8] updated the output --- infra/scripts/validate_model_quota.ps1 | 77 +++++++++++++++----------- infra/scripts/validate_model_quota.sh | 71 +++++++++++------------- 2 files changed, 75 insertions(+), 73 deletions(-) diff --git a/infra/scripts/validate_model_quota.ps1 b/infra/scripts/validate_model_quota.ps1 index f96bb29b..7b4451c1 100644 --- a/infra/scripts/validate_model_quota.ps1 +++ b/infra/scripts/validate_model_quota.ps1 @@ -25,69 +25,80 @@ if ($DeploymentType -ne "Standard" -and $DeploymentType -ne "GlobalStandard") { $ModelType = "OpenAI.$DeploymentType.$Model" +$PreferredRegions = @('australiaeast', 'eastus2', 'francecentral', 'japaneast', 'norwayeast', 'swedencentral', 'uksouth', 'westus') +$AllResults = @() + function Check-Quota { param ( [string]$Region ) - Write-Host "`nšŸ” Checking quota for $ModelType in $Region ..." - $ModelInfoRaw = az cognitiveservices usage list --location $Region --query "[?name.value=='$ModelType']" --output json $ModelInfo = $null try { $ModelInfo = $ModelInfoRaw | ConvertFrom-Json } catch { - Write-Warning "āš ļø Failed to parse quota info for region: $Region" - return $false + return } if (-not $ModelInfo) { - Write-Host "āš ļø No quota information found for $ModelType in $Region" - return $false + return } $CurrentValue = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).currentValue $Limit = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).limit - $CurrentValue = [int]($CurrentValue -replace '\.0+$', '') # Remove decimals - $Limit = [int]($Limit -replace '\.0+$', '') # Remove decimals - + $CurrentValue = [int]($CurrentValue -replace '\.0+$', '') + $Limit = [int]($Limit -replace '\.0+$', '') $Available = $Limit - $CurrentValue - Write-Host "šŸ”Ž Model: $ModelType | Used: $CurrentValue | Limit: $Limit | Available: $Available" - - if ($Available -ge $Capacity) { - Write-Host "āœ… Sufficient quota in $Region" - return $true - } else { - Write-Host "āŒ Insufficient quota in $Region (Available: $Available, Required: $Capacity)" - return $false + + $script:AllResults += [PSCustomObject]@{ + Region = $Region + Model = $ModelType + Limit = $Limit + Used = $CurrentValue + Available = $Available } } -# List of fallback regions (excluding the one already tried) -$PreferredRegions = @('australiaeast', 'eastus2', 'francecentral', 'japaneast', 'norwayeast', 'swedencentral', 'uksouth', 'westus') | Where-Object { $_ -ne $Location } +foreach ($region in $PreferredRegions) { + Check-Quota -Region $region +} -# Try original location first -if (Check-Quota -Region $Location) { +# Display Results Table +Write-Host "\n-------------------------------------------------------------------------------------------------------------" +Write-Host "| No. | Region | Model Name | Limit | Used | Available |" +Write-Host "-------------------------------------------------------------------------------------------------------------" + +$count = 1 +foreach ($entry in $AllResults) { + $index = $PreferredRegions.IndexOf($entry.Region) + 1 + $modelShort = $entry.Model.Substring($entry.Model.LastIndexOf(".") + 1) + Write-Host ("| {0,-4} | {1,-16} | {2,-35} | {3,-7} | {4,-7} | {5,-9} |" -f $index, $entry.Region, $entry.Model, $entry.Limit, $entry.Used, $entry.Available) + $count++ +} +Write-Host "-------------------------------------------------------------------------------------------------------------" + +$EligibleRegion = $AllResults | Where-Object { $_.Region -eq $Location -and $_.Available -ge $Capacity } +if ($EligibleRegion) { + Write-Host "\nāœ… Sufficient quota found in original region '$Location'." exit 0 } -Write-Host "`nšŸ” Trying fallback regions for available quota..." +$FallbackRegions = $AllResults | Where-Object { $_.Region -ne $Location -and $_.Available -ge $Capacity } -foreach ($region in $PreferredRegions) { - if (Check-Quota -Region $region) { - Write-Host "šŸš« Deployment cannot proceed because the original region '$Location' lacks sufficient quota." - Write-Host "āž”ļø You can retry using the available region: '$region'" - Write-Host "šŸ”§ To proceed, update the 'AZURE_OPENAI_LOCATION' value in the 'main.bicepparam' file, then run the following command:" - Write-Host " azd env set AZURE_OPENAI_LOCATION '$region'" +if ($FallbackRegions.Count -gt 0) { + Write-Host "`nāŒ Deployment cannot proceed because the original region '$Location' lacks sufficient quota." + Write-Host "āž”ļø You can retry using one of the following regions with sufficient quota:`n" - # Optional: update `.azure/env` (uncomment if needed) - # Add-Content ".azure/env" "`nAZURE_OPENAI_LOCATION=$region" - - # Exit with non-zero code to halt deployment - exit 2 + foreach ($region in $FallbackRegions) { + Write-Host " ā€¢ $($region.Region) (Available: $($region.Available))" } + + Write-Host "`nšŸ”§ To proceed, update the 'AZURE_OPENAI_LOCATION' value in the 'main.bicepparam' file, then run:" + Write-Host " azd env set AZURE_OPENAI_LOCATION ''" + exit 2 } Write-Error "āŒ ERROR: No available quota found in any region." diff --git a/infra/scripts/validate_model_quota.sh b/infra/scripts/validate_model_quota.sh index 90fa3590..fad1f6e6 100644 --- a/infra/scripts/validate_model_quota.sh +++ b/infra/scripts/validate_model_quota.sh @@ -51,56 +51,47 @@ fi MODEL_TYPE="OpenAI.$DEPLOYMENT_TYPE.$MODEL" -check_quota() { - local region="$1" - echo "šŸ” Checking quota for $MODEL_TYPE in $region ..." +declare -a FALLBACK_REGIONS=() +ROW_NO=1 +printf "\n%-5s | %-20s | %-40s | %-10s | %-10s | %-10s\n" "No." "Region" "Model Name" "Limit" "Used" "Available" +printf -- "---------------------------------------------------------------------------------------------------------------------\n" + +for region in "${ALL_REGIONS[@]}"; do MODEL_INFO=$(az cognitiveservices usage list --location "$region" --query "[?name.value=='$MODEL_TYPE']" --output json 2>/dev/null) - if [[ -z "$MODEL_INFO" || "$MODEL_INFO" == "[]" ]]; then - echo "āš ļø No quota info found for $MODEL_TYPE in $region" - return 1 - fi + if [[ -n "$MODEL_INFO" && "$MODEL_INFO" != "[]" ]]; then + CURRENT_VALUE=$(echo "$MODEL_INFO" | jq -r '.[0].currentValue // 0' | cut -d'.' -f1) + LIMIT=$(echo "$MODEL_INFO" | jq -r '.[0].limit // 0' | cut -d'.' -f1) + AVAILABLE=$((LIMIT - CURRENT_VALUE)) - CURRENT_VALUE=$(echo "$MODEL_INFO" | jq -r '.[0].currentValue // 0' | cut -d'.' -f1) - LIMIT=$(echo "$MODEL_INFO" | jq -r '.[0].limit // 0' | cut -d'.' -f1) - AVAILABLE=$((LIMIT - CURRENT_VALUE)) + printf "%-5s | %-20s | %-40s | %-10s | %-10s | %-10s\n" "$ROW_NO" "$region" "$MODEL_TYPE" "$LIMIT" "$CURRENT_VALUE" "$AVAILABLE" - echo "šŸ”Ž Model: $MODEL_TYPE | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE" + if [[ "$region" == "$LOCATION" && "$AVAILABLE" -ge "$CAPACITY" ]]; then + echo -e "\nāœ… Sufficient quota available in user-specified region: $LOCATION" + exit 0 + fi - if (( AVAILABLE >= CAPACITY )); then - echo "āœ… Sufficient quota in $region" - return 0 - else - echo "āŒ Insufficient quota in $region (Available: $AVAILABLE, Required: $CAPACITY)" - return 1 + if [[ "$region" != "$LOCATION" && "$AVAILABLE" -ge "$CAPACITY" ]]; then + FALLBACK_REGIONS+=("$region ($AVAILABLE)") + fi fi -} -# Try user-provided region -if check_quota "$LOCATION"; then - exit 0 -fi - -# Try fallback regions -REMAINING_REGIONS=() -for region in "${ALL_REGIONS[@]}"; do - if [[ "$region" != "$LOCATION" ]]; then - REMAINING_REGIONS+=("$region") - fi + ((ROW_NO++)) done -echo "šŸ” Trying fallback regions for available quota..." - -for region in "${REMAINING_REGIONS[@]}"; do - if check_quota "$region"; then - echo "šŸš« Deployment cannot proceed because the original region '$LOCATION' lacks sufficient quota." - echo "āž”ļø You can retry using the available region: '$region'" - echo "šŸ”§ To proceed, update the 'AZURE_OPENAI_LOCATION' value in the 'main.bicepparam' file, then run:" - echo " azd env set AZURE_OPENAI_LOCATION '$region'" - exit 1 - fi -done +printf -- "---------------------------------------------------------------------------------------------------------------------\n" + +if [[ "${#FALLBACK_REGIONS[@]}" -gt 0 ]]; then + echo -e "\nāŒ Deployment cannot proceed because the original region '$LOCATION' lacks sufficient quota." + echo "āž”ļø You can retry using one of the following regions with sufficient quota:" + for fallback in "${FALLBACK_REGIONS[@]}"; do + echo " ā€¢ $fallback" + done + echo -e "\nšŸ”§ To proceed, update the 'AZURE_OPENAI_LOCATION' value in the 'main.bicepparam' file, then run:" + echo " azd env set AZURE_OPENAI_LOCATION ''" + exit 2 +fi echo "āŒ ERROR: No available quota found in any of the fallback regions." exit 1 From dfa9b3665eb509b13d470d8e9762c19cd318be82 Mon Sep 17 00:00:00 2001 From: Priyanka-Microsoft Date: Thu, 5 Jun 2025 13:03:23 +0530 Subject: [PATCH 6/8] updated the message --- infra/scripts/validate_model_quota.ps1 | 4 +++- infra/scripts/validate_model_quota.sh | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/infra/scripts/validate_model_quota.ps1 b/infra/scripts/validate_model_quota.ps1 index 7b4451c1..9e5918a4 100644 --- a/infra/scripts/validate_model_quota.ps1 +++ b/infra/scripts/validate_model_quota.ps1 @@ -96,8 +96,10 @@ if ($FallbackRegions.Count -gt 0) { Write-Host " ā€¢ $($region.Region) (Available: $($region.Available))" } - Write-Host "`nšŸ”§ To proceed, update the 'AZURE_OPENAI_LOCATION' value in the 'main.bicepparam' file, then run:" + Write-Host "`nšŸ”§ To proceed, run:" Write-Host " azd env set AZURE_OPENAI_LOCATION ''" + Write-Host "šŸ“Œ To confirm it's set correctly, run:" + Write-Host " azd env get-value AZURE_OPENAI_LOCATION" exit 2 } diff --git a/infra/scripts/validate_model_quota.sh b/infra/scripts/validate_model_quota.sh index fad1f6e6..23261177 100644 --- a/infra/scripts/validate_model_quota.sh +++ b/infra/scripts/validate_model_quota.sh @@ -88,8 +88,10 @@ if [[ "${#FALLBACK_REGIONS[@]}" -gt 0 ]]; then for fallback in "${FALLBACK_REGIONS[@]}"; do echo " ā€¢ $fallback" done - echo -e "\nšŸ”§ To proceed, update the 'AZURE_OPENAI_LOCATION' value in the 'main.bicepparam' file, then run:" + echo -e "\nšŸ”§ To proceed, run:" echo " azd env set AZURE_OPENAI_LOCATION ''" + echo "šŸ“Œ To confirm it's set correctly, run:" + echo " azd env get-value AZURE_OPENAI_LOCATION" exit 2 fi From bd9d92bbf8a5376a9400b39a15f8a47be5fba7b7 Mon Sep 17 00:00:00 2001 From: Priyanka-Microsoft Date: Thu, 5 Jun 2025 13:14:30 +0530 Subject: [PATCH 7/8] exit code --- infra/scripts/validate_model_deployment_quota.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/infra/scripts/validate_model_deployment_quota.sh b/infra/scripts/validate_model_deployment_quota.sh index 4d4277c7..1f890b0e 100644 --- a/infra/scripts/validate_model_deployment_quota.sh +++ b/infra/scripts/validate_model_deployment_quota.sh @@ -68,9 +68,14 @@ while IFS= read -r deployment; do ./infra/scripts/validate_model_quota.sh --location "$LOCATION" --model "$model" --capacity $capacity --deployment-type $type # Check if the script failed - if [ $? -ne 0 ]; then - echo "āŒ ERROR: Quota validation failed for model deployment: $name" - quotaAvailable=false + exit_code=$? + if [ $exit_code -ne 0 ]; then + if [ $exit_code -eq 2 ]; then + # Skip printing any quota validation error ā€” already handled inside the validation script + exit 1 + fi + echo "āŒ ERROR: Quota validation failed for model deployment: $name" + quotaAvailable=false fi done <<< "$(echo "$aiModelDeployments")" From 64df1f5ee9c0d3378aa0a4e5cf9ef8b6d63ff10a Mon Sep 17 00:00:00 2001 From: Priyanka-Microsoft Date: Thu, 5 Jun 2025 13:24:31 +0530 Subject: [PATCH 8/8] refined the output message --- infra/scripts/validate_model_quota.ps1 | 1 + infra/scripts/validate_model_quota.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/infra/scripts/validate_model_quota.ps1 b/infra/scripts/validate_model_quota.ps1 index 9e5918a4..fb3bbbb1 100644 --- a/infra/scripts/validate_model_quota.ps1 +++ b/infra/scripts/validate_model_quota.ps1 @@ -100,6 +100,7 @@ if ($FallbackRegions.Count -gt 0) { Write-Host " azd env set AZURE_OPENAI_LOCATION ''" Write-Host "šŸ“Œ To confirm it's set correctly, run:" Write-Host " azd env get-value AZURE_OPENAI_LOCATION" + Write-Host "ā–¶ļø Once confirmed, re-run azd up to deploy the model in the new region." exit 2 } diff --git a/infra/scripts/validate_model_quota.sh b/infra/scripts/validate_model_quota.sh index 23261177..eb8d6b25 100644 --- a/infra/scripts/validate_model_quota.sh +++ b/infra/scripts/validate_model_quota.sh @@ -92,6 +92,7 @@ if [[ "${#FALLBACK_REGIONS[@]}" -gt 0 ]]; then echo " azd env set AZURE_OPENAI_LOCATION ''" echo "šŸ“Œ To confirm it's set correctly, run:" echo " azd env get-value AZURE_OPENAI_LOCATION" + echo "ā–¶ļø Once confirmed, re-run azd up to deploy the model in the new region." exit 2 fi