fix: Fixed deployment script issue for AVM module, fixed relative path issues & refactored bicep

infra/modules/ai-services.bicep

@@ -190,7 +190,7 @@ resource cognitiveServiceNew 'Microsoft.CognitiveServices/accounts@2025-06-01' =
       ? {
           defaultAction: networkAcls.?defaultAction
           virtualNetworkRules: networkAcls.?virtualNetworkRules ?? []
-          ipRules: networkAcls.?ipRules ?? []      
+          ipRules: networkAcls.?ipRules ?? []
           bypass: networkAcls.?bypass ?? 'None'
         }
       : null
@@ -236,7 +236,7 @@ resource cognitiveServiceNew 'Microsoft.CognitiveServices/accounts@2025-06-01' =
 
 var existingCognitiveServiceDetails = split(existingFoundryProjectResourceId, '/')
 
-resource cognitiveServiceExisting 'Microsoft.CognitiveServices/accounts@2025-04-01-preview' existing = if(useExistingService) {
+resource cognitiveServiceExisting 'Microsoft.CognitiveServices/accounts@2025-09-01' existing = if(useExistingService) {
   name: existingCognitiveServiceDetails[8]
   scope: resourceGroup(existingCognitiveServiceDetails[2], existingCognitiveServiceDetails[4])
 }
@@ -245,7 +245,7 @@ module cognitive_service_dependencies './dependencies.bicep' = if(!useExistingSe
   params: {
     projectName: projectName
     projectDescription: projectDescription
-    name:  cognitiveServiceNew.name 
+    name:  cognitiveServiceNew.name
     location: location
     deployments: deployments
     diagnosticSettings: diagnosticSettings
@@ -259,7 +259,7 @@ module cognitive_service_dependencies './dependencies.bicep' = if(!useExistingSe
 
 module existing_cognitive_service_dependencies './dependencies.bicep' = if(useExistingService) {
   params: {
-    name:  cognitiveServiceExisting.name 
+    name:  cognitiveServiceExisting.name
     projectName: projectName
     projectDescription: projectDescription
     existingFoundryProjectResourceId: existingFoundryProjectResourceId

infra/modules/dependencies.bicep

@@ -248,7 +248,7 @@ resource cognitiveService_diagnosticSettings 'Microsoft.Insights/diagnosticSetti
   }
 ]
 
-module cognitiveService_privateEndpoints 'br/public:avm/res/network/private-endpoint:0.11.0' = [
+module cognitiveService_privateEndpoints 'br/public:avm/res/network/private-endpoint:0.11.1' = [
   for (privateEndpoint, index) in (privateEndpoints ?? []): {
     name: '${uniqueString(deployment().name, location)}-cognitiveService-PrivateEndpoint-${index}'
     scope: resourceGroup(
@@ -322,7 +322,6 @@ resource cognitiveService_roleAssignments 'Microsoft.Authorization/roleAssignmen
   }
 ]
 
-
 module aiProject 'project.bicep' = if(!empty(projectName) || !empty(existingFoundryProjectResourceId)) {
   name: take('${name}-ai-project-${projectName}-deployment', 64)
   params: {

infra/modules/virtualNetwork.bicep

@@ -2,7 +2,7 @@
 // Networking - NSGs, VNET and Subnets. Each subnet has its own NSG
 /****************************************************************************************************************************/
 @description('Name of the virtual network.')
-param name string 
+param name string
 
 @description('Azure region to deploy resources.')
 param location string = resourceGroup().location
@@ -155,14 +155,14 @@ param subnets subnetType[] = [
     }
   }
   {
-        name: 'deployment-scripts'
-        addressPrefixes: ['10.0.4.0/24']
-        networkSecurityGroup: {
-          name: 'nsg-deployment-scripts'
-          securityRules: []
-        }
-        delegation: 'Microsoft.ContainerInstance/containerGroups'
-        serviceEndpoints: ['Microsoft.Storage']
+    name: 'deployment-scripts'
+    addressPrefixes: ['10.0.4.0/24']
+    networkSecurityGroup: {
+      name: 'nsg-deployment-scripts'
+      securityRules: []
+    }
+    delegation: 'Microsoft.ContainerInstance/containerGroups'
+    serviceEndpoints: ['Microsoft.Storage']
   }
 ]
 
@@ -185,7 +185,6 @@ param resourceSuffix string
 //     Standard_D2s_v3 (2 vCPU, 8 GiB RAM, Premium SSD) //  next most common
 //     Standard_D2s_v4 (2 vCPU, 8 GiB RAM, Premium SSD)  // Newest, so fewer regions availabl
 
-
 // Subnet Classless Inter-Doman Routing (CIDR)  Sizing Reference Table (Best Practices)
 // | CIDR      | # of Addresses | # of /24s | Notes                                 |
 // |-----------|---------------|-----------|----------------------------------------|
@@ -215,12 +214,12 @@ param resourceSuffix string
 // - Document subnet usage and purpose in code comments.
 // - For AVM modules, ensure only one delegation per subnet and leave delegations empty if not required.
 
-// 1. Create NSGs for subnets 
+// 1. Create NSGs for subnets
 // using AVM Network Security Group module
 // https://github.com/Azure/bicep-registry-modules/tree/main/avm/res/network/network-security-group
 
 @batchSize(1)
-module nsgs 'br/public:avm/res/network/network-security-group:0.5.1' = [
+module nsgs 'br/public:avm/res/network/network-security-group:0.5.2' = [
   for (subnet, i) in subnets: if (!empty(subnet.?networkSecurityGroup)) {
     name: take('avm.res.network.network-security-group.${subnet.?networkSecurityGroup.name}.${resourceSuffix}', 64)
     params: {
@@ -237,7 +236,7 @@ module nsgs 'br/public:avm/res/network/network-security-group:0.5.1' = [
 // using AVM Virtual Network module
 // https://github.com/Azure/bicep-registry-modules/tree/main/avm/res/network/virtual-network
 
-module virtualNetwork 'br/public:avm/res/network/virtual-network:0.7.0' = {
+module virtualNetwork 'br/public:avm/res/network/virtual-network:0.7.1' = {
   name: take('avm.res.network.virtual-network.${name}', 64)
   params: {
     name: name
@@ -290,11 +289,21 @@ output subnets subnetOutputType[] = [
 ]
 
 // Dynamic outputs for individual subnets for backward compatibility
-output webSubnetResourceId string = contains(map(subnets, subnet => subnet.name), 'web') ? virtualNetwork.outputs.subnetResourceIds[indexOf(map(subnets, subnet => subnet.name), 'web')] : ''
-output pepsSubnetResourceId string = contains(map(subnets, subnet => subnet.name), 'peps') ? virtualNetwork.outputs.subnetResourceIds[indexOf(map(subnets, subnet => subnet.name), 'peps')] : ''
-output bastionSubnetResourceId string = contains(map(subnets, subnet => subnet.name), 'AzureBastionSubnet') ? virtualNetwork.outputs.subnetResourceIds[indexOf(map(subnets, subnet => subnet.name), 'AzureBastionSubnet')] : ''
-output jumpboxSubnetResourceId string = contains(map(subnets, subnet => subnet.name), 'jumpbox') ? virtualNetwork.outputs.subnetResourceIds[indexOf(map(subnets, subnet => subnet.name), 'jumpbox')] : ''
-output deploymentScriptsSubnetResourceId string = contains(map(subnets, subnet => subnet.name), 'deployment-scripts') ? virtualNetwork.outputs.subnetResourceIds[indexOf(map(subnets, subnet => subnet.name), 'deployment-scripts')] : ''
+output webSubnetResourceId string = contains(map(subnets, subnet => subnet.name), 'web')
+  ? virtualNetwork.outputs.subnetResourceIds[indexOf(map(subnets, subnet => subnet.name), 'web')]
+  : ''
+output pepsSubnetResourceId string = contains(map(subnets, subnet => subnet.name), 'peps')
+  ? virtualNetwork.outputs.subnetResourceIds[indexOf(map(subnets, subnet => subnet.name), 'peps')]
+  : ''
+output bastionSubnetResourceId string = contains(map(subnets, subnet => subnet.name), 'AzureBastionSubnet')
+  ? virtualNetwork.outputs.subnetResourceIds[indexOf(map(subnets, subnet => subnet.name), 'AzureBastionSubnet')]
+  : ''
+output jumpboxSubnetResourceId string = contains(map(subnets, subnet => subnet.name), 'jumpbox')
+  ? virtualNetwork.outputs.subnetResourceIds[indexOf(map(subnets, subnet => subnet.name), 'jumpbox')]
+  : ''
+output deploymentScriptsSubnetResourceId string = contains(map(subnets, subnet => subnet.name), 'deployment-scripts')
+  ? virtualNetwork.outputs.subnetResourceIds[indexOf(map(subnets, subnet => subnet.name), 'deployment-scripts')]
+  : ''
 
 @export()
 @description('Custom type definition for subnet resource information as output')
@@ -318,8 +327,8 @@ type subnetType = {
   @description('Required. The Name of the subnet resource.')
   name: string
 
-  @description('Required. Prefixes for the subnet.')  // Required to ensure at least one prefix is provided
-  addressPrefixes: string[]   
+  @description('Required. Prefixes for the subnet.') // Required to ensure at least one prefix is provided
+  addressPrefixes: string[]
 
   @description('Optional. The delegation to enable on the subnet.')
   delegation: string?

infra/modules/web-sites.bicep

@@ -260,7 +260,7 @@ resource app_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0
   }
 ]
 
-module app_privateEndpoints 'br/public:avm/res/network/private-endpoint:0.11.0' = [
+module app_privateEndpoints 'br/public:avm/res/network/private-endpoint:0.11.1' = [
   for (privateEndpoint, index) in (privateEndpoints ?? []): {
     name: '${uniqueString(deployment().name, location)}-app-PrivateEndpoint-${index}'
     scope: resourceGroup(

infra/scripts/process_custom_data.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Get the directory where this script is located
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
 # Variables - Grouped by service for clarity
 # General Azure
 resourceGroupName="${1}"
@@ -345,6 +348,92 @@ get_values_from_azd_env() {
 	return 0
 }
 
+get_values_from_az_deployment() {
+	echo "Getting values from Azure deployment outputs..."
+
+    deploymentName=$(az group show --name "$resourceGroupName" --query "tags.DeploymentName" -o tsv)
+    echo "Deployment Name (from tag): $deploymentName"
+
+    echo "Fetching deployment outputs..."
+	# Get all outputs
+    deploymentOutputs=$(az deployment group show \
+        --name "$deploymentName" \
+        --resource-group "$resourceGroupName" \
+        --query "properties.outputs" -o json)
+
+	# Helper function to extract value from deployment outputs
+	# Usage: extract_value "primaryKey" "fallbackKey"
+	extract_value() {
+		local primary_key="$1"
+		local fallback_key="$2"
+		local value
+
+		value=$(echo "$deploymentOutputs" | grep -A 3 "\"$primary_key\"" | grep '"value"' | sed 's/.*"value": *"\([^"]*\)".*/\1/')
+		if [ -z "$value" ] && [ -n "$fallback_key" ]; then
+			value=$(echo "$deploymentOutputs" | grep -A 3 "\"$fallback_key\"" | grep '"value"' | sed 's/.*"value": *"\([^"]*\)".*/\1/')
+		fi
+		echo "$value"
+	}
+
+	# Extract each value using the helper function
+	storageAccountName=$(extract_value "storageAccountName" "storagE_ACCOUNT_NAME")
+	fileSystem=$(extract_value "storageContainerName" "storagE_CONTAINER_NAME")
+	sqlServerName=$(extract_value "sqlDBServer" "sqldB_SERVER")
+	SqlDatabaseName=$(extract_value "sqlDBDatabase" "sqldB_DATABASE")
+	backendUserMidClientId=$(extract_value "backendUserMid" "backenD_USER_MID")
+	backendUserMidDisplayName=$(extract_value "backendUserMidName" "backenD_USER_MID_NAME")
+	aiSearchName=$(extract_value "azureAISearchName" "azurE_AI_SEARCH_NAME")
+	searchEndpoint=$(extract_value "azureAISearchEndpoint" "azurE_AI_SEARCH_ENDPOINT")
+	aif_resource_id=$(extract_value "aiFoundryResourceId" "aI_FOUNDRY_RESOURCE_ID")
+	cu_foundry_resource_id=$(extract_value "cuFoundryResourceId" "cU_FOUNDRY_RESOURCE_ID")
+	openaiEndpoint=$(extract_value "azureOpenAIEndpoint" "azurE_OPENAI_ENDPOINT")
+	embeddingModel=$(extract_value "azureOpenAIEmbeddingModel" "azurE_OPENAI_EMBEDDING_MODEL")
+	cuEndpoint=$(extract_value "azureOpenAICuEndpoint" "azurE_OPENAI_CU_ENDPOINT")
+	aiAgentEndpoint=$(extract_value "azureAiAgentEndpoint" "azurE_AI_AGENT_ENDPOINT")
+	cuApiVersion=$(extract_value "azureContentUnderstandingApiVersion" "azurE_CONTENT_UNDERSTANDING_API_VERSION")
+	deploymentModel=$(extract_value "azureOpenAIDeploymentModel" "azurE_OPENAI_DEPLOYMENT_MODEL")
+	usecase=$(extract_value "useCase" "usE_CASE")
+
+	# Strip FQDN suffix from SQL server name if present (Azure CLI needs just the server name)
+	sqlServerName="${sqlServerName%.database.windows.net}"
+
+	# Define required values with their display names for error reporting
+	declare -A required_values=(
+		["storageAccountName"]="STORAGE_ACCOUNT_NAME"
+		["fileSystem"]="STORAGE_CONTAINER_NAME"
+		["sqlServerName"]="SQLDB_SERVER"
+		["SqlDatabaseName"]="SQLDB_DATABASE"
+		["backendUserMidClientId"]="BACKEND_USER_MID"
+		["backendUserMidDisplayName"]="BACKEND_USER_MID_NAME"
+		["aiSearchName"]="AZURE_AI_SEARCH_NAME"
+		["aif_resource_id"]="AI_FOUNDRY_RESOURCE_ID"
+		["cu_foundry_resource_id"]="CU_FOUNDRY_RESOURCE_ID"
+		["searchEndpoint"]="AZURE_AI_SEARCH_ENDPOINT"
+		["openaiEndpoint"]="AZURE_OPENAI_ENDPOINT"
+		["embeddingModel"]="AZURE_OPENAI_EMBEDDING_MODEL"
+		["cuEndpoint"]="AZURE_OPENAI_CU_ENDPOINT"
+		["aiAgentEndpoint"]="AZURE_AI_AGENT_ENDPOINT"
+		["cuApiVersion"]="AZURE_CONTENT_UNDERSTANDING_API_VERSION"
+		["deploymentModel"]="AZURE_OPENAI_DEPLOYMENT_MODEL"
+		["usecase"]="USE_CASE"
+	)
+
+	# Validate and collect missing values
+	missing_values=()
+	for var_name in "${!required_values[@]}"; do
+		if [ -z "${!var_name}" ]; then
+			missing_values+=("${required_values[$var_name]}")
+		fi
+	done
+
+	if [ ${#missing_values[@]} -gt 0 ]; then
+		echo "Error: The following required values could not be retrieved from Azure deployment outputs:"
+		printf '  - %s\n' "${missing_values[@]}" | sort
+		return 1
+	fi
+	return 0
+}
+
 # Check if user is logged in to Azure
 echo "Checking Azure authentication..."
 if az account show &> /dev/null; then
@@ -404,10 +493,39 @@ fi
 echo ""
 
 echo ""
-if ! get_values_from_azd_env; then
-    echo "Failed to get values from azd environment."
-    echo ""
-    exit 1
+
+if [ -z "$resourceGroupName" ]; then
+    # No resource group provided - use azd env
+    if ! get_values_from_azd_env; then
+        echo "Failed to get values from azd environment."
+		echo ""
+        echo "If you want to use deployment outputs instead, please provide the resource group name as an argument."
+        echo "Usage: $0 [ResourceGroupName]"
+		echo "Example: $0 my-resource-group"
+		echo ""
+        exit 1
+    fi
+else
+    # Resource group provided - use deployment outputs
+	echo ""
+    echo "Resource group provided: $resourceGroupName"
+
+    # Call deployment function
+    if ! get_values_from_az_deployment; then
+        echo "Failed to get values from deployment outputs."
+		echo ""
+		echo "Would you like to enter the values manually? (y/n): "
+		read -r manual_input_choice
+		if [[ "$manual_input_choice" == "y" || "$manual_input_choice" == "Y" ]]; then
+			if ! get_values_from_user; then
+				echo "Error: Manual input failed."
+				exit 1
+			fi
+		else
+			echo "Exiting script."
+			exit 1
+		fi
+	fi
 fi
 
 echo ""
@@ -441,7 +559,7 @@ if [ $? -ne 0 ]; then
 	exit 1
 fi
 
-pythonScriptPath="infra/scripts/index_scripts/"
+pythonScriptPath="$SCRIPT_DIR/index_scripts/"
 
 # Install the requirements
 pip install --quiet -r ${pythonScriptPath}requirements.txt
@@ -452,13 +570,13 @@ fi
 
 # Create Content Understanding analyzers
 echo "✓ Creating Content Understanding analyzer templates"
-python infra/scripts/index_scripts/02_create_cu_template_text.py --cu_endpoint="$cuEndpoint" --cu_api_version="$cuApiVersion"
+python "${pythonScriptPath}02_create_cu_template_text.py" --cu_endpoint="$cuEndpoint" --cu_api_version="$cuApiVersion"
 if [ $? -ne 0 ]; then
 	echo "Error: 02_create_cu_template_text.py failed."
 	exit 1
 fi
 
-python infra/scripts/index_scripts/02_create_cu_template_audio.py --cu_endpoint="$cuEndpoint" --cu_api_version="$cuApiVersion"
+python "${pythonScriptPath}02_create_cu_template_audio.py" --cu_endpoint="$cuEndpoint" --cu_api_version="$cuApiVersion"
 if [ $? -ne 0 ]; then
 	echo "Error: 02_create_cu_template_audio.py failed."
 	exit 1
@@ -467,7 +585,7 @@ fi
 # Run 04_cu_process_custom_data.py
 echo "✓ Processing custom data"
 sql_server_fqdn="$sqlServerName.database.windows.net"
-python infra/scripts/index_scripts/04_cu_process_custom_data.py \
+python "${pythonScriptPath}04_cu_process_custom_data.py" \
     --search_endpoint "$searchEndpoint" \
     --openai_endpoint "$openaiEndpoint" \
     --ai_project_endpoint "$aiAgentEndpoint" \