diff --git a/azure.yaml b/azure.yaml index ee4810b1..b8195cf2 100644 --- a/azure.yaml +++ b/azure.yaml @@ -1,4 +1,20 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json name: multi-agent-custom-automation-engine-solution-accelerator metadata: - template: multi-agent-custom-automation-engine-solution-accelerator@1.0 \ No newline at end of file + template: multi-agent-custom-automation-engine-solution-accelerator@1.0 + +hooks: + preprovision: + posix: + shell: sh + run: > + ./infra/scripts/validate_model_deployment_quota.sh --subscription "$AZURE_SUBSCRIPTION_ID" --location "${AZURE_OPENAI_LOCATION:-swedencentral}" --models-parameter "aiModelDeployments" + interactive: false + continueOnError: false + + windows: + shell: pwsh + run: > + $location = if ($env:AZURE_OPENAI_LOCATION) { $env:AZURE_OPENAI_LOCATION } else { "swedencentral" }; + ./infra/scripts/validate_model_deployment_quotas.ps1 -SubscriptionId $env:AZURE_SUBSCRIPTION_ID -Location $location -ModelsParameter "aiModelDeployments" + interactive: false + continueOnError: false diff --git a/infra/main.parameters.json b/infra/main.parameters.json index c7fc26a4..e57dcbfb 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -2,6 +2,22 @@ "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", "contentVersion": "1.0.0.0", "parameters": { + "aiModelDeployments": { + "value": [ + { + "name": "gpt", + "model": { + "name": "gpt-4o", + "version": "2024-08-06", + "format": "OpenAI" + }, + "sku": { + "name": "GlobalStandard", + "capacity": 140 + } + } + ] + }, "environmentName": { "value": "${AZURE_ENV_NAME}" }, diff --git a/infra/scripts/quota_check_params.sh b/infra/scripts/quota_check_params.sh index add6ac47..db378ef9 100644 --- a/infra/scripts/quota_check_params.sh +++ b/infra/scripts/quota_check_params.sh @@ -1,249 +1,249 @@ -#!/bin/bash -# VERBOSE=false - -MODELS="" -REGIONS="" -VERBOSE=false - -while [[ $# -gt 0 ]]; do - case "$1" in - --models) - MODELS="$2" - shift 2 - ;; - --regions) - REGIONS="$2" - shift 2 - ;; - --verbose) - VERBOSE=true - shift - ;; - *) - echo "Unknown option: $1" - exit 1 - ;; - esac -done - -# Fallback to defaults if not provided -[[ -z "$MODELS" ]] -[[ -z "$REGIONS" ]] - -echo "Models: $MODELS" -echo "Regions: $REGIONS" -echo "Verbose: $VERBOSE" - -for arg in "$@"; do - if [ "$arg" = "--verbose" ]; then - VERBOSE=true - fi -done - -log_verbose() { - if [ "$VERBOSE" = true ]; then - echo "$1" - fi -} - -# Default Models and Capacities (Comma-separated in "model:capacity" format) -DEFAULT_MODEL_CAPACITY="gpt-4o:50" -# Convert the comma-separated string into an array -IFS=',' read -r -a MODEL_CAPACITY_PAIRS <<< "$DEFAULT_MODEL_CAPACITY" - -echo "šŸ”„ Fetching available Azure subscriptions..." -SUBSCRIPTIONS=$(az account list --query "[?state=='Enabled'].{Name:name, ID:id}" --output tsv) -SUB_COUNT=$(echo "$SUBSCRIPTIONS" | wc -l) - -if [ "$SUB_COUNT" -eq 0 ]; then - echo "āŒ ERROR: No active Azure subscriptions found. Please log in using 'az login' and ensure you have an active subscription." - exit 1 -elif [ "$SUB_COUNT" -eq 1 ]; then - # If only one subscription, automatically select it - AZURE_SUBSCRIPTION_ID=$(echo "$SUBSCRIPTIONS" | awk '{print $2}') - if [ -z "$AZURE_SUBSCRIPTION_ID" ]; then - echo "āŒ ERROR: No active Azure subscriptions found. Please log in using 'az login' and ensure you have an active subscription." - exit 1 - fi - echo "āœ… Using the only available subscription: $AZURE_SUBSCRIPTION_ID" -else - # If multiple subscriptions exist, prompt the user to choose one - echo "Multiple subscriptions found:" - echo "$SUBSCRIPTIONS" | awk '{print NR")", $1, "-", $2}' - - while true; do - echo "Enter the number of the subscription to use:" - read SUB_INDEX - - # Validate user input - if [[ "$SUB_INDEX" =~ ^[0-9]+$ ]] && [ "$SUB_INDEX" -ge 1 ] && [ "$SUB_INDEX" -le "$SUB_COUNT" ]; then - AZURE_SUBSCRIPTION_ID=$(echo "$SUBSCRIPTIONS" | awk -v idx="$SUB_INDEX" 'NR==idx {print $2}') - echo "āœ… Selected Subscription: $AZURE_SUBSCRIPTION_ID" - break - else - echo "āŒ Invalid selection. Please enter a valid number from the list." - fi - done -fi - - -# Set the selected subscription -az account set --subscription "$AZURE_SUBSCRIPTION_ID" -echo "šŸŽÆ Active Subscription: $(az account show --query '[name, id]' --output tsv)" - -# Default Regions to check (Comma-separated, now configurable) -DEFAULT_REGIONS="eastus,uksouth,eastus2,northcentralus,swedencentral,westus,westus2,southcentralus,canadacentral" -IFS=',' read -r -a DEFAULT_REGION_ARRAY <<< "$DEFAULT_REGIONS" - -# Read parameters (if any) -IFS=',' read -r -a USER_PROVIDED_PAIRS <<< "$MODELS" -USER_REGION="$REGIONS" - -IS_USER_PROVIDED_PAIRS=false - -if [ ${#USER_PROVIDED_PAIRS[@]} -lt 1 ]; then - echo "No parameters provided, using default model-capacity pairs: ${MODEL_CAPACITY_PAIRS[*]}" -else - echo "Using provided model and capacity pairs: ${USER_PROVIDED_PAIRS[*]}" - IS_USER_PROVIDED_PAIRS=true - MODEL_CAPACITY_PAIRS=("${USER_PROVIDED_PAIRS[@]}") -fi - -declare -a FINAL_MODEL_NAMES -declare -a FINAL_CAPACITIES -declare -a TABLE_ROWS - -for PAIR in "${MODEL_CAPACITY_PAIRS[@]}"; do - MODEL_NAME=$(echo "$PAIR" | cut -d':' -f1 | tr '[:upper:]' '[:lower:]') - CAPACITY=$(echo "$PAIR" | cut -d':' -f2) - - if [ -z "$MODEL_NAME" ] || [ -z "$CAPACITY" ]; then - echo "āŒ ERROR: Invalid model and capacity pair '$PAIR'. Both model and capacity must be specified." - exit 1 - fi - - FINAL_MODEL_NAMES+=("$MODEL_NAME") - FINAL_CAPACITIES+=("$CAPACITY") - -done - -echo "šŸ”„ Using Models: ${FINAL_MODEL_NAMES[*]} with respective Capacities: ${FINAL_CAPACITIES[*]}" -echo "----------------------------------------" - -# Check if the user provided a region, if not, use the default regions -if [ -n "$USER_REGION" ]; then - echo "šŸ” User provided region: $USER_REGION" - IFS=',' read -r -a REGIONS <<< "$USER_REGION" -else - echo "No region specified, using default regions: ${DEFAULT_REGION_ARRAY[*]}" - REGIONS=("${DEFAULT_REGION_ARRAY[@]}") - APPLY_OR_CONDITION=true -fi - -echo "āœ… Retrieved Azure regions. Checking availability..." -INDEX=1 - -VALID_REGIONS=() -for REGION in "${REGIONS[@]}"; do - log_verbose "----------------------------------------" - log_verbose "šŸ” Checking region: $REGION" - - QUOTA_INFO=$(az cognitiveservices usage list --location "$REGION" --output json | tr '[:upper:]' '[:lower:]') - if [ -z "$QUOTA_INFO" ]; then - log_verbose "āš ļø WARNING: Failed to retrieve quota for region $REGION. Skipping." - continue - fi - - TEXT_EMBEDDING_AVAILABLE=false - AT_LEAST_ONE_MODEL_AVAILABLE=false - TEMP_TABLE_ROWS=() - - for index in "${!FINAL_MODEL_NAMES[@]}"; do - MODEL_NAME="${FINAL_MODEL_NAMES[$index]}" - REQUIRED_CAPACITY="${FINAL_CAPACITIES[$index]}" - FOUND=false - INSUFFICIENT_QUOTA=false - - if [ "$MODEL_NAME" = "text-embedding-ada-002" ]; then - MODEL_TYPES=("openai.standard.$MODEL_NAME") - else - MODEL_TYPES=("openai.standard.$MODEL_NAME" "openai.globalstandard.$MODEL_NAME") - fi - - for MODEL_TYPE in "${MODEL_TYPES[@]}"; do - FOUND=false - INSUFFICIENT_QUOTA=false - log_verbose "šŸ” Checking model: $MODEL_NAME with required capacity: $REQUIRED_CAPACITY ($MODEL_TYPE)" - - MODEL_INFO=$(echo "$QUOTA_INFO" | awk -v model="\"value\": \"$MODEL_TYPE\"" ' - BEGIN { RS="},"; FS="," } - $0 ~ model { print $0 } - ') - - if [ -z "$MODEL_INFO" ]; then - FOUND=false - log_verbose "āš ļø WARNING: No quota information found for model: $MODEL_NAME in region: $REGION for model type: $MODEL_TYPE." - continue - fi - - if [ -n "$MODEL_INFO" ]; then - FOUND=true - CURRENT_VALUE=$(echo "$MODEL_INFO" | awk -F': ' '/"currentvalue"/ {print $2}' | tr -d ',' | tr -d ' ') - LIMIT=$(echo "$MODEL_INFO" | awk -F': ' '/"limit"/ {print $2}' | tr -d ',' | tr -d ' ') - - CURRENT_VALUE=${CURRENT_VALUE:-0} - LIMIT=${LIMIT:-0} - - CURRENT_VALUE=$(echo "$CURRENT_VALUE" | cut -d'.' -f1) - LIMIT=$(echo "$LIMIT" | cut -d'.' -f1) - - AVAILABLE=$((LIMIT - CURRENT_VALUE)) - log_verbose "āœ… Model: $MODEL_TYPE | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE" - - if [ "$AVAILABLE" -ge "$REQUIRED_CAPACITY" ]; then - FOUND=true - if [ "$MODEL_NAME" = "text-embedding-ada-002" ]; then - TEXT_EMBEDDING_AVAILABLE=true - fi - AT_LEAST_ONE_MODEL_AVAILABLE=true - TEMP_TABLE_ROWS+=("$(printf "| %-4s | %-20s | %-43s | %-10s | %-10s | %-10s |" "$INDEX" "$REGION" "$MODEL_TYPE" "$LIMIT" "$CURRENT_VALUE" "$AVAILABLE")") - else - INSUFFICIENT_QUOTA=true - fi - fi - - if [ "$FOUND" = false ]; then - log_verbose "āŒ No models found for model: $MODEL_NAME in region: $REGION (${MODEL_TYPES[*]})" - - elif [ "$INSUFFICIENT_QUOTA" = true ]; then - log_verbose "āš ļø Model $MODEL_NAME in region: $REGION has insufficient quota (${MODEL_TYPES[*]})." - fi - done - done - -if { [ "$IS_USER_PROVIDED_PAIRS" = true ] && [ "$INSUFFICIENT_QUOTA" = false ] && [ "$FOUND" = true ]; } || { [ "$APPLY_OR_CONDITION" != true ] || [ "$AT_LEAST_ONE_MODEL_AVAILABLE" = true ]; }; then - VALID_REGIONS+=("$REGION") - TABLE_ROWS+=("${TEMP_TABLE_ROWS[@]}") - INDEX=$((INDEX + 1)) - elif [ ${#USER_PROVIDED_PAIRS[@]} -eq 0 ]; then - echo "🚫 Skipping $REGION as it does not meet quota requirements." - fi - -done - -if [ ${#TABLE_ROWS[@]} -eq 0 ]; then - echo "--------------------------------------------------------------------------------------------------------------------" - - echo "āŒ No regions have sufficient quota for all required models. Please request a quota increase: https://aka.ms/oai/stuquotarequest" -else - echo "---------------------------------------------------------------------------------------------------------------------" - printf "| %-4s | %-20s | %-43s | %-10s | %-10s | %-10s |\n" "No." "Region" "Model Name" "Limit" "Used" "Available" - echo "---------------------------------------------------------------------------------------------------------------------" - for ROW in "${TABLE_ROWS[@]}"; do - echo "$ROW" - done - echo "---------------------------------------------------------------------------------------------------------------------" - echo "āž”ļø To request a quota increase, visit: https://aka.ms/oai/stuquotarequest" -fi - -echo "āœ… Script completed." +#!/bin/bash +# VERBOSE=false + +MODELS="" +REGIONS="" +VERBOSE=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --models) + MODELS="$2" + shift 2 + ;; + --regions) + REGIONS="$2" + shift 2 + ;; + --verbose) + VERBOSE=true + shift + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +# Fallback to defaults if not provided +[[ -z "$MODELS" ]] +[[ -z "$REGIONS" ]] + +echo "Models: $MODELS" +echo "Regions: $REGIONS" +echo "Verbose: $VERBOSE" + +for arg in "$@"; do + if [ "$arg" = "--verbose" ]; then + VERBOSE=true + fi +done + +log_verbose() { + if [ "$VERBOSE" = true ]; then + echo "$1" + fi +} + +# Default Models and Capacities (Comma-separated in "model:capacity" format) +DEFAULT_MODEL_CAPACITY="gpt-4o:50" +# Convert the comma-separated string into an array +IFS=',' read -r -a MODEL_CAPACITY_PAIRS <<< "$DEFAULT_MODEL_CAPACITY" + +echo "šŸ”„ Fetching available Azure subscriptions..." +SUBSCRIPTIONS=$(az account list --query "[?state=='Enabled'].{Name:name, ID:id}" --output tsv) +SUB_COUNT=$(echo "$SUBSCRIPTIONS" | wc -l) + +if [ "$SUB_COUNT" -eq 0 ]; then + echo "āŒ ERROR: No active Azure subscriptions found. Please log in using 'az login' and ensure you have an active subscription." + exit 1 +elif [ "$SUB_COUNT" -eq 1 ]; then + # If only one subscription, automatically select it + AZURE_SUBSCRIPTION_ID=$(echo "$SUBSCRIPTIONS" | awk '{print $2}') + if [ -z "$AZURE_SUBSCRIPTION_ID" ]; then + echo "āŒ ERROR: No active Azure subscriptions found. Please log in using 'az login' and ensure you have an active subscription." + exit 1 + fi + echo "āœ… Using the only available subscription: $AZURE_SUBSCRIPTION_ID" +else + # If multiple subscriptions exist, prompt the user to choose one + echo "Multiple subscriptions found:" + echo "$SUBSCRIPTIONS" | awk '{print NR")", $1, "-", $2}' + + while true; do + echo "Enter the number of the subscription to use:" + read SUB_INDEX + + # Validate user input + if [[ "$SUB_INDEX" =~ ^[0-9]+$ ]] && [ "$SUB_INDEX" -ge 1 ] && [ "$SUB_INDEX" -le "$SUB_COUNT" ]; then + AZURE_SUBSCRIPTION_ID=$(echo "$SUBSCRIPTIONS" | awk -v idx="$SUB_INDEX" 'NR==idx {print $2}') + echo "āœ… Selected Subscription: $AZURE_SUBSCRIPTION_ID" + break + else + echo "āŒ Invalid selection. Please enter a valid number from the list." + fi + done +fi + + +# Set the selected subscription +az account set --subscription "$AZURE_SUBSCRIPTION_ID" +echo "šŸŽÆ Active Subscription: $(az account show --query '[name, id]' --output tsv)" + +# Default Regions to check (Comma-separated, now configurable) +DEFAULT_REGIONS="australiaeast,eastus2,francecentral,japaneast,norwayeast,swedencentral,uksouth,westus" +IFS=',' read -r -a DEFAULT_REGION_ARRAY <<< "$DEFAULT_REGIONS" + +# Read parameters (if any) +IFS=',' read -r -a USER_PROVIDED_PAIRS <<< "$MODELS" +USER_REGION="$REGIONS" + +IS_USER_PROVIDED_PAIRS=false + +if [ ${#USER_PROVIDED_PAIRS[@]} -lt 1 ]; then + echo "No parameters provided, using default model-capacity pairs: ${MODEL_CAPACITY_PAIRS[*]}" +else + echo "Using provided model and capacity pairs: ${USER_PROVIDED_PAIRS[*]}" + IS_USER_PROVIDED_PAIRS=true + MODEL_CAPACITY_PAIRS=("${USER_PROVIDED_PAIRS[@]}") +fi + +declare -a FINAL_MODEL_NAMES +declare -a FINAL_CAPACITIES +declare -a TABLE_ROWS + +for PAIR in "${MODEL_CAPACITY_PAIRS[@]}"; do + MODEL_NAME=$(echo "$PAIR" | cut -d':' -f1 | tr '[:upper:]' '[:lower:]') + CAPACITY=$(echo "$PAIR" | cut -d':' -f2) + + if [ -z "$MODEL_NAME" ] || [ -z "$CAPACITY" ]; then + echo "āŒ ERROR: Invalid model and capacity pair '$PAIR'. Both model and capacity must be specified." + exit 1 + fi + + FINAL_MODEL_NAMES+=("$MODEL_NAME") + FINAL_CAPACITIES+=("$CAPACITY") + +done + +echo "šŸ”„ Using Models: ${FINAL_MODEL_NAMES[*]} with respective Capacities: ${FINAL_CAPACITIES[*]}" +echo "----------------------------------------" + +# Check if the user provided a region, if not, use the default regions +if [ -n "$USER_REGION" ]; then + echo "šŸ” User provided region: $USER_REGION" + IFS=',' read -r -a REGIONS <<< "$USER_REGION" +else + echo "No region specified, using default regions: ${DEFAULT_REGION_ARRAY[*]}" + REGIONS=("${DEFAULT_REGION_ARRAY[@]}") + APPLY_OR_CONDITION=true +fi + +echo "āœ… Retrieved Azure regions. Checking availability..." +INDEX=1 + +VALID_REGIONS=() +for REGION in "${REGIONS[@]}"; do + log_verbose "----------------------------------------" + log_verbose "šŸ” Checking region: $REGION" + + QUOTA_INFO=$(az cognitiveservices usage list --location "$REGION" --output json | tr '[:upper:]' '[:lower:]') + if [ -z "$QUOTA_INFO" ]; then + log_verbose "āš ļø WARNING: Failed to retrieve quota for region $REGION. Skipping." + continue + fi + + TEXT_EMBEDDING_AVAILABLE=false + AT_LEAST_ONE_MODEL_AVAILABLE=false + TEMP_TABLE_ROWS=() + + for index in "${!FINAL_MODEL_NAMES[@]}"; do + MODEL_NAME="${FINAL_MODEL_NAMES[$index]}" + REQUIRED_CAPACITY="${FINAL_CAPACITIES[$index]}" + FOUND=false + INSUFFICIENT_QUOTA=false + + if [ "$MODEL_NAME" = "text-embedding-ada-002" ]; then + MODEL_TYPES=("openai.standard.$MODEL_NAME") + else + MODEL_TYPES=("openai.standard.$MODEL_NAME" "openai.globalstandard.$MODEL_NAME") + fi + + for MODEL_TYPE in "${MODEL_TYPES[@]}"; do + FOUND=false + INSUFFICIENT_QUOTA=false + log_verbose "šŸ” Checking model: $MODEL_NAME with required capacity: $REQUIRED_CAPACITY ($MODEL_TYPE)" + + MODEL_INFO=$(echo "$QUOTA_INFO" | awk -v model="\"value\": \"$MODEL_TYPE\"" ' + BEGIN { RS="},"; FS="," } + $0 ~ model { print $0 } + ') + + if [ -z "$MODEL_INFO" ]; then + FOUND=false + log_verbose "āš ļø WARNING: No quota information found for model: $MODEL_NAME in region: $REGION for model type: $MODEL_TYPE." + continue + fi + + if [ -n "$MODEL_INFO" ]; then + FOUND=true + CURRENT_VALUE=$(echo "$MODEL_INFO" | awk -F': ' '/"currentvalue"/ {print $2}' | tr -d ',' | tr -d ' ') + LIMIT=$(echo "$MODEL_INFO" | awk -F': ' '/"limit"/ {print $2}' | tr -d ',' | tr -d ' ') + + CURRENT_VALUE=${CURRENT_VALUE:-0} + LIMIT=${LIMIT:-0} + + CURRENT_VALUE=$(echo "$CURRENT_VALUE" | cut -d'.' -f1) + LIMIT=$(echo "$LIMIT" | cut -d'.' -f1) + + AVAILABLE=$((LIMIT - CURRENT_VALUE)) + log_verbose "āœ… Model: $MODEL_TYPE | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE" + + if [ "$AVAILABLE" -ge "$REQUIRED_CAPACITY" ]; then + FOUND=true + if [ "$MODEL_NAME" = "text-embedding-ada-002" ]; then + TEXT_EMBEDDING_AVAILABLE=true + fi + AT_LEAST_ONE_MODEL_AVAILABLE=true + TEMP_TABLE_ROWS+=("$(printf "| %-4s | %-20s | %-43s | %-10s | %-10s | %-10s |" "$INDEX" "$REGION" "$MODEL_TYPE" "$LIMIT" "$CURRENT_VALUE" "$AVAILABLE")") + else + INSUFFICIENT_QUOTA=true + fi + fi + + if [ "$FOUND" = false ]; then + log_verbose "āŒ No models found for model: $MODEL_NAME in region: $REGION (${MODEL_TYPES[*]})" + + elif [ "$INSUFFICIENT_QUOTA" = true ]; then + log_verbose "āš ļø Model $MODEL_NAME in region: $REGION has insufficient quota (${MODEL_TYPES[*]})." + fi + done + done + +if { [ "$IS_USER_PROVIDED_PAIRS" = true ] && [ "$INSUFFICIENT_QUOTA" = false ] && [ "$FOUND" = true ]; } || { [ "$APPLY_OR_CONDITION" != true ] || [ "$AT_LEAST_ONE_MODEL_AVAILABLE" = true ]; }; then + VALID_REGIONS+=("$REGION") + TABLE_ROWS+=("${TEMP_TABLE_ROWS[@]}") + INDEX=$((INDEX + 1)) + elif [ ${#USER_PROVIDED_PAIRS[@]} -eq 0 ]; then + echo "🚫 Skipping $REGION as it does not meet quota requirements." + fi + +done + +if [ ${#TABLE_ROWS[@]} -eq 0 ]; then + echo "--------------------------------------------------------------------------------------------------------------------" + + echo "āŒ No regions have sufficient quota for all required models. Please request a quota increase: https://aka.ms/oai/stuquotarequest" +else + echo "---------------------------------------------------------------------------------------------------------------------" + printf "| %-4s | %-20s | %-43s | %-10s | %-10s | %-10s |\n" "No." "Region" "Model Name" "Limit" "Used" "Available" + echo "---------------------------------------------------------------------------------------------------------------------" + for ROW in "${TABLE_ROWS[@]}"; do + echo "$ROW" + done + echo "---------------------------------------------------------------------------------------------------------------------" + echo "āž”ļø To request a quota increase, visit: https://aka.ms/oai/stuquotarequest" +fi + +echo "āœ… Script completed." \ No newline at end of file diff --git a/infra/scripts/validate_model_deployment_quota.sh b/infra/scripts/validate_model_deployment_quota.sh new file mode 100644 index 00000000..1f890b0e --- /dev/null +++ b/infra/scripts/validate_model_deployment_quota.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +SUBSCRIPTION_ID="" +LOCATION="" +MODELS_PARAMETER="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --subscription) + SUBSCRIPTION_ID="$2" + shift 2 + ;; + --location) + LOCATION="$2" + shift 2 + ;; + --models-parameter) + MODELS_PARAMETER="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +# Verify all required parameters are provided and echo missing ones +MISSING_PARAMS=() + +if [[ -z "$SUBSCRIPTION_ID" ]]; then + MISSING_PARAMS+=("subscription") +fi + +if [[ -z "$LOCATION" ]]; then + MISSING_PARAMS+=("location") +fi + +if [[ -z "$MODELS_PARAMETER" ]]; then + MISSING_PARAMS+=("models-parameter") +fi + +if [[ ${#MISSING_PARAMS[@]} -ne 0 ]]; then + echo "āŒ ERROR: Missing required parameters: ${MISSING_PARAMS[*]}" + echo "Usage: $0 --subscription --location --models-parameter " + exit 1 +fi + +aiModelDeployments=$(jq -c ".parameters.$MODELS_PARAMETER.value[]" ./infra/main.parameters.json) + +if [ $? -ne 0 ]; then + echo "Error: Failed to parse main.parameters.json. Ensure jq is installed and the JSON file is valid." + exit 1 +fi + +az account set --subscription "$SUBSCRIPTION_ID" +echo "šŸŽÆ Active Subscription: $(az account show --query '[name, id]' --output tsv)" + +quotaAvailable=true + +while IFS= read -r deployment; do + name=$(echo "$deployment" | jq -r '.name') + model=$(echo "$deployment" | jq -r '.model.name') + type=$(echo "$deployment" | jq -r '.sku.name') + capacity=$(echo "$deployment" | jq -r '.sku.capacity') + + echo "šŸ” Validating model deployment: $name ..." + ./infra/scripts/validate_model_quota.sh --location "$LOCATION" --model "$model" --capacity $capacity --deployment-type $type + + # Check if the script failed + exit_code=$? + if [ $exit_code -ne 0 ]; then + if [ $exit_code -eq 2 ]; then + # Skip printing any quota validation error — already handled inside the validation script + exit 1 + fi + echo "āŒ ERROR: Quota validation failed for model deployment: $name" + quotaAvailable=false + fi +done <<< "$(echo "$aiModelDeployments")" + +if [ "$quotaAvailable" = false ]; then + echo "āŒ ERROR: One or more model deployments failed validation." + exit 1 +else + echo "āœ… All model deployments passed quota validation successfully." + exit 0 +fi \ No newline at end of file diff --git a/infra/scripts/validate_model_deployment_quotas.ps1 b/infra/scripts/validate_model_deployment_quotas.ps1 new file mode 100644 index 00000000..dfe5f7ef --- /dev/null +++ b/infra/scripts/validate_model_deployment_quotas.ps1 @@ -0,0 +1,69 @@ +param ( + [string]$SubscriptionId, + [string]$Location, + [string]$ModelsParameter +) + +# Verify all required parameters are provided +$MissingParams = @() + +if (-not $SubscriptionId) { + $MissingParams += "subscription" +} + +if (-not $Location) { + $MissingParams += "location" +} + +if (-not $ModelsParameter) { + $MissingParams += "models-parameter" +} + +if ($MissingParams.Count -gt 0) { + Write-Error "āŒ ERROR: Missing required parameters: $($MissingParams -join ', ')" + Write-Host "Usage: .\validate_model_deployment_quotas.ps1 -SubscriptionId -Location -ModelsParameter " + exit 1 +} + +$JsonContent = Get-Content -Path "./infra/main.parameters.json" -Raw | ConvertFrom-Json + +if (-not $JsonContent) { + Write-Error "āŒ ERROR: Failed to parse main.parameters.json. Ensure the JSON file is valid." + exit 1 +} + +$aiModelDeployments = $JsonContent.parameters.$ModelsParameter.value + +if (-not $aiModelDeployments -or -not ($aiModelDeployments -is [System.Collections.IEnumerable])) { + Write-Error "āŒ ERROR: The specified property $ModelsParameter does not exist or is not an array." + exit 1 +} + +az account set --subscription $SubscriptionId +Write-Host "šŸŽÆ Active Subscription: $(az account show --query '[name, id]' --output tsv)" + +$QuotaAvailable = $true + +foreach ($deployment in $aiModelDeployments) { + $name = $deployment.name + $model = $deployment.model.name + $type = $deployment.sku.name + $capacity = $deployment.sku.capacity + + Write-Host "šŸ” Validating model deployment: $name ..." + & .\infra\scripts\validate_model_quota.ps1 -Location $Location -Model $model -Capacity $capacity -DeploymentType $type + + # Check if the script failed + if ($LASTEXITCODE -ne 0) { + Write-Error "āŒ ERROR: Quota validation failed for model deployment: $name" + $QuotaAvailable = $false + } +} + +if (-not $QuotaAvailable) { + Write-Error "āŒ ERROR: One or more model deployments failed validation." + exit 1 +} else { + Write-Host "āœ… All model deployments passed quota validation successfully." + exit 0 +} \ No newline at end of file diff --git a/infra/scripts/validate_model_quota.ps1 b/infra/scripts/validate_model_quota.ps1 new file mode 100644 index 00000000..fb3bbbb1 --- /dev/null +++ b/infra/scripts/validate_model_quota.ps1 @@ -0,0 +1,108 @@ +param ( + [string]$Location, + [string]$Model, + [string]$DeploymentType = "Standard", + [int]$Capacity +) + +# Verify required parameters +$MissingParams = @() +if (-not $Location) { $MissingParams += "location" } +if (-not $Model) { $MissingParams += "model" } +if (-not $Capacity) { $MissingParams += "capacity" } +if (-not $DeploymentType) { $MissingParams += "deployment-type" } + +if ($MissingParams.Count -gt 0) { + Write-Error "āŒ ERROR: Missing required parameters: $($MissingParams -join ', ')" + Write-Host "Usage: .\validate_model_quota.ps1 -Location -Model -Capacity [-DeploymentType ]" + exit 1 +} + +if ($DeploymentType -ne "Standard" -and $DeploymentType -ne "GlobalStandard") { + Write-Error "āŒ ERROR: Invalid deployment type: $DeploymentType. Allowed values are 'Standard' or 'GlobalStandard'." + exit 1 +} + +$ModelType = "OpenAI.$DeploymentType.$Model" + +$PreferredRegions = @('australiaeast', 'eastus2', 'francecentral', 'japaneast', 'norwayeast', 'swedencentral', 'uksouth', 'westus') +$AllResults = @() + +function Check-Quota { + param ( + [string]$Region + ) + + $ModelInfoRaw = az cognitiveservices usage list --location $Region --query "[?name.value=='$ModelType']" --output json + $ModelInfo = $null + + try { + $ModelInfo = $ModelInfoRaw | ConvertFrom-Json + } catch { + return + } + + if (-not $ModelInfo) { + return + } + + $CurrentValue = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).currentValue + $Limit = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).limit + + $CurrentValue = [int]($CurrentValue -replace '\.0+$', '') + $Limit = [int]($Limit -replace '\.0+$', '') + $Available = $Limit - $CurrentValue + + $script:AllResults += [PSCustomObject]@{ + Region = $Region + Model = $ModelType + Limit = $Limit + Used = $CurrentValue + Available = $Available + } +} + +foreach ($region in $PreferredRegions) { + Check-Quota -Region $region +} + +# Display Results Table +Write-Host "\n-------------------------------------------------------------------------------------------------------------" +Write-Host "| No. | Region | Model Name | Limit | Used | Available |" +Write-Host "-------------------------------------------------------------------------------------------------------------" + +$count = 1 +foreach ($entry in $AllResults) { + $index = $PreferredRegions.IndexOf($entry.Region) + 1 + $modelShort = $entry.Model.Substring($entry.Model.LastIndexOf(".") + 1) + Write-Host ("| {0,-4} | {1,-16} | {2,-35} | {3,-7} | {4,-7} | {5,-9} |" -f $index, $entry.Region, $entry.Model, $entry.Limit, $entry.Used, $entry.Available) + $count++ +} +Write-Host "-------------------------------------------------------------------------------------------------------------" + +$EligibleRegion = $AllResults | Where-Object { $_.Region -eq $Location -and $_.Available -ge $Capacity } +if ($EligibleRegion) { + Write-Host "\nāœ… Sufficient quota found in original region '$Location'." + exit 0 +} + +$FallbackRegions = $AllResults | Where-Object { $_.Region -ne $Location -and $_.Available -ge $Capacity } + +if ($FallbackRegions.Count -gt 0) { + Write-Host "`nāŒ Deployment cannot proceed because the original region '$Location' lacks sufficient quota." + Write-Host "āž”ļø You can retry using one of the following regions with sufficient quota:`n" + + foreach ($region in $FallbackRegions) { + Write-Host " • $($region.Region) (Available: $($region.Available))" + } + + Write-Host "`nšŸ”§ To proceed, run:" + Write-Host " azd env set AZURE_OPENAI_LOCATION ''" + Write-Host "šŸ“Œ To confirm it's set correctly, run:" + Write-Host " azd env get-value AZURE_OPENAI_LOCATION" + Write-Host "ā–¶ļø Once confirmed, re-run azd up to deploy the model in the new region." + exit 2 +} + +Write-Error "āŒ ERROR: No available quota found in any region." +exit 1 diff --git a/infra/scripts/validate_model_quota.sh b/infra/scripts/validate_model_quota.sh new file mode 100644 index 00000000..eb8d6b25 --- /dev/null +++ b/infra/scripts/validate_model_quota.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +LOCATION="" +MODEL="" +DEPLOYMENT_TYPE="Standard" +CAPACITY=0 + +ALL_REGIONS=('australiaeast' 'eastus2' 'francecentral' 'japaneast' 'norwayeast' 'swedencentral' 'uksouth' 'westus') + +while [[ $# -gt 0 ]]; do + case "$1" in + --model) + MODEL="$2" + shift 2 + ;; + --capacity) + CAPACITY="$2" + shift 2 + ;; + --deployment-type) + DEPLOYMENT_TYPE="$2" + shift 2 + ;; + --location) + LOCATION="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +# Validate required params +MISSING_PARAMS=() +[[ -z "$LOCATION" ]] && MISSING_PARAMS+=("location") +[[ -z "$MODEL" ]] && MISSING_PARAMS+=("model") +[[ -z "$CAPACITY" ]] && MISSING_PARAMS+=("capacity") + +if [[ ${#MISSING_PARAMS[@]} -ne 0 ]]; then + echo "āŒ ERROR: Missing required parameters: ${MISSING_PARAMS[*]}" + echo "Usage: $0 --location --model --capacity [--deployment-type ]" + exit 1 +fi + +if [[ "$DEPLOYMENT_TYPE" != "Standard" && "$DEPLOYMENT_TYPE" != "GlobalStandard" ]]; then + echo "āŒ ERROR: Invalid deployment type: $DEPLOYMENT_TYPE. Allowed values are 'Standard' or 'GlobalStandard'." + exit 1 +fi + +MODEL_TYPE="OpenAI.$DEPLOYMENT_TYPE.$MODEL" + +declare -a FALLBACK_REGIONS=() +ROW_NO=1 + +printf "\n%-5s | %-20s | %-40s | %-10s | %-10s | %-10s\n" "No." "Region" "Model Name" "Limit" "Used" "Available" +printf -- "---------------------------------------------------------------------------------------------------------------------\n" + +for region in "${ALL_REGIONS[@]}"; do + MODEL_INFO=$(az cognitiveservices usage list --location "$region" --query "[?name.value=='$MODEL_TYPE']" --output json 2>/dev/null) + + if [[ -n "$MODEL_INFO" && "$MODEL_INFO" != "[]" ]]; then + CURRENT_VALUE=$(echo "$MODEL_INFO" | jq -r '.[0].currentValue // 0' | cut -d'.' -f1) + LIMIT=$(echo "$MODEL_INFO" | jq -r '.[0].limit // 0' | cut -d'.' -f1) + AVAILABLE=$((LIMIT - CURRENT_VALUE)) + + printf "%-5s | %-20s | %-40s | %-10s | %-10s | %-10s\n" "$ROW_NO" "$region" "$MODEL_TYPE" "$LIMIT" "$CURRENT_VALUE" "$AVAILABLE" + + if [[ "$region" == "$LOCATION" && "$AVAILABLE" -ge "$CAPACITY" ]]; then + echo -e "\nāœ… Sufficient quota available in user-specified region: $LOCATION" + exit 0 + fi + + if [[ "$region" != "$LOCATION" && "$AVAILABLE" -ge "$CAPACITY" ]]; then + FALLBACK_REGIONS+=("$region ($AVAILABLE)") + fi + fi + + ((ROW_NO++)) +done + +printf -- "---------------------------------------------------------------------------------------------------------------------\n" + +if [[ "${#FALLBACK_REGIONS[@]}" -gt 0 ]]; then + echo -e "\nāŒ Deployment cannot proceed because the original region '$LOCATION' lacks sufficient quota." + echo "āž”ļø You can retry using one of the following regions with sufficient quota:" + for fallback in "${FALLBACK_REGIONS[@]}"; do + echo " • $fallback" + done + echo -e "\nšŸ”§ To proceed, run:" + echo " azd env set AZURE_OPENAI_LOCATION ''" + echo "šŸ“Œ To confirm it's set correctly, run:" + echo " azd env get-value AZURE_OPENAI_LOCATION" + echo "ā–¶ļø Once confirmed, re-run azd up to deploy the model in the new region." + exit 2 +fi + +echo "āŒ ERROR: No available quota found in any of the fallback regions." +exit 1