fix: WAF aligned changes merge from dev to main

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/devcontainers/python:3.11
+FROM mcr.microsoft.com/devcontainers/python:3.11-bookworm
 
 # install git
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \

.github/workflows/build-docker.yml

@@ -62,7 +62,7 @@ jobs:
         context: .
         file: ${{ inputs.dockerfile }}
         push: ${{ inputs.push }}
-        cache-from: type=registry,ref=${{ inputs.new_registry }}/${{ inputs.app_name }}:${{ github.ref_name == 'main' && 'latest' || github.ref_name == 'dev' && 'dev' || github.ref_name == 'demo' && 'demo'|| github.ref_name == 'dependabotchanges' && 'dependabotchanges' || github.head_ref || github.ref_name }}
+        cache-from: type=registry,ref=${{ inputs.new_registry }}/${{ inputs.app_name }}:${{ github.ref_name == 'main' && 'latest_waf' || github.ref_name == 'dev' && 'dev' || github.ref_name == 'demo' && 'demo'|| github.ref_name == 'dependabotchanges' && 'dependabotchanges' || github.head_ref || github.ref_name }}
         tags: |
-          ${{ inputs.new_registry }}/${{ inputs.app_name }}:${{ github.ref_name == 'main' && 'latest' || github.ref_name == 'dev' && 'dev' || github.ref_name == 'demo' && 'demo'|| github.ref_name == 'dependabotchanges' && 'dependabotchanges' || github.head_ref || 'default' }}
-          ${{ inputs.new_registry }}/${{ inputs.app_name }}:${{ steps.date.outputs.date }}_${{ github.run_number }}
+          ${{ inputs.new_registry }}/${{ inputs.app_name }}:${{ github.ref_name == 'main' && 'latest_waf' || github.ref_name == 'dev' && 'dev' || github.ref_name == 'demo' && 'demo'|| github.ref_name == 'dependabotchanges' && 'dependabotchanges' || github.head_ref || 'default' }}
+          ${{ inputs.new_registry }}/${{ inputs.app_name }}:${{ github.ref_name == 'main' && 'latest_waf' || github.ref_name == 'dev' && 'dev' || github.ref_name == 'demo' && 'demo'|| github.ref_name == 'dependabotchanges' && 'dependabotchanges' || github.head_ref || 'default' }}_${{ steps.date.outputs.date }}_${{ github.run_number }}

.github/workflows/ci.yml

@@ -36,6 +36,13 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Install AZD
+        run: |
+          set -e
+          echo "Fetching deployment output..."
+          # Install azd (Azure Developer CLI) - required by process_sample_data.sh
+          curl -fsSL https://aka.ms/install-azd.sh | bash
+
       - name: Run Quota Check
         id: quota-check
         run: |
@@ -79,6 +86,45 @@ jobs:
           echo "Selected Region: $VALID_REGION"
           echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV
 
+      - name: Generate Resource Group Name
+        id: generate_rg_name
+        run: |
+          echo "Generating a unique resource group name..."
+          ACCL_NAME="cwyd"  # Account name as specified
+          SHORT_UUID=$(uuidgen | cut -d'-' -f1)
+          UNIQUE_RG_NAME="arg-${ACCL_NAME}-${SHORT_UUID}"
+          echo "RESOURCE_GROUP_NAME=${UNIQUE_RG_NAME}" >> $GITHUB_ENV
+          echo "Generated RESOURCE_GROUP_NAME: ${UNIQUE_RG_NAME}"
+
+      - name: Check and Create Resource Group
+        id: check_create_rg
+        run: |
+          echo "RESOURCE_GROUP: ${{ env.RESOURCE_GROUP_NAME }}"
+          set -e
+          echo "Checking if resource group exists..."
+          rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
+          if [ "$rg_exists" = "false" ]; then
+            echo "Resource group does not exist. Creating..."
+            az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.AZURE_LOCATION }} --tags SecurityControl=Ignore || { echo "Error creating resource group"; exit 1; }
+          else
+            echo "Resource group already exists."
+          fi
+          # Set output for other jobs
+          echo "RESOURCE_GROUP_NAME=${{ env.RESOURCE_GROUP_NAME }}" >> $GITHUB_OUTPUT
+
+
+      - name: Generate Unique Solution Prefix
+        id: generate_solution_prefix
+        run: |
+          set -e
+          COMMON_PART="pslc"
+          TIMESTAMP=$(date +%s)
+          UPDATED_TIMESTAMP=$(echo $TIMESTAMP | tail -c 3)
+          UNIQUE_SOLUTION_SUFFIX="${COMMON_PART}${UPDATED_TIMESTAMP}"
+          echo "SOLUTION_SUFFIX=${UNIQUE_SOLUTION_SUFFIX}" >> $GITHUB_ENV
+          echo "SOLUTION_SUFFIX=${UNIQUE_SOLUTION_SUFFIX}" >> $GITHUB_OUTPUT
+          echo "Generated SOLUTION_SUFFIX: ${UNIQUE_SOLUTION_SUFFIX}"
+
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:
@@ -90,11 +136,11 @@ jobs:
         id: set-image-tag
         run: |
           if [[ "${{ github.event_name }}" == "schedule" ]]; then
-            echo "imageTag=latest" >> $GITHUB_ENV
-            echo "::set-output name=imageTag::latest"
+            echo "imageTag=latest_waf" >> $GITHUB_ENV
+            echo "::set-output name=imageTag::latest_waf"
           elif [[ "${{ github.ref_name }}" == "main" ]]; then
-            echo "imageTag=latest" >> $GITHUB_ENV
-            echo "::set-output name=imageTag::latest"
+            echo "imageTag=latest_waf" >> $GITHUB_ENV
+            echo "::set-output name=imageTag::latest_waf"
           else
             echo "imageTag=${{ github.ref_name }}" >> $GITHUB_ENV
             echo "::set-output name=imageTag::${{ github.ref_name }}"
@@ -103,8 +149,9 @@ jobs:
       - name: Pre-build image and deploy
         uses: devcontainers/[email protected]
         env:
-                      AZURE_ENV_NAME: ${{ github.run_id }}
+                      AZURE_ENV_NAME: ${{ env.SOLUTION_SUFFIX }}
                       AZURE_LOCATION: ${{ env.AZURE_LOCATION }}
+                      AZURE_RESOURCE_GROUP: ${{ env.RESOURCE_GROUP_NAME }}
         with:
                             push: never
                             imageName: ghcr.io/azure-samples/chat-with-your-data-solution-accelerator
@@ -141,6 +188,7 @@ jobs:
                               AZURE_SUBSCRIPTION_ID
                               AZURE_ENV_NAME
                               AZURE_LOCATION
+                              AZURE_RESOURCE_GROUP
                               AUTH_ENABLED=false
                               AZURE_USE_AUTHENTICATION=false
                               AZURE_ENABLE_AUTH=false

.github/workflows/group_dependabot_security_updates.yml

@@ -65,9 +65,10 @@ jobs:
           git config --global user.email "[email protected]"
 
       - name: Install required tools
-        uses: awalsh128/[email protected]
-        with:
-          packages: "jq gh"
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y jq gh
+        shell: bash
 
       - name: Enable strict error handling
         shell: bash

Makefile

@@ -65,7 +65,17 @@ azd-login: ## 🔑 Login to Azure with azd and a SPN
 # Fixed Makefile section for deploy target
 deploy: azd-login ## Deploy everything to Azure
 	@echo -e "\e[34m$@\e[0m" || true
-	@azd env new ${AZURE_ENV_NAME}
+	@echo "AZURE_ENV_NAME: '${AZURE_ENV_NAME}'"
+	@echo "AZURE_LOCATION: '${AZURE_LOCATION}'"
+	@echo "AZURE_RESOURCE_GROUP: '${AZURE_RESOURCE_GROUP}'"
+
+	# Validate required variables
+	@if [ -z "${AZURE_ENV_NAME}" ]; then echo "❌ AZURE_ENV_NAME not set"; exit 1; fi
+	@if [ -z "${AZURE_LOCATION}" ]; then echo "❌ AZURE_LOCATION not set"; exit 1; fi
+	@if [ -z "${AZURE_RESOURCE_GROUP}" ]; then echo "❌ AZURE_RESOURCE_GROUP not set"; exit 1; fi
+
+	@azd env new ${AZURE_ENV_NAME} --location ${AZURE_LOCATION}
+	@azd env set AZURE_RESOURCE_GROUP ${AZURE_RESOURCE_GROUP}
 
 	# Provision and deploy
 	@azd provision --no-prompt

azure.yaml

@@ -2,7 +2,7 @@
 
 name: chat-with-your-data-solution-accelerator
 metadata:
-  template: [email protected]
+   template: [email protected]
 hooks:
   postprovision:
     # run: ./infra/prompt-flow/create-prompt-flow.sh

code/backend/batch/local.settings.json.sample

@@ -2,7 +2,7 @@
     "IsEncrypted": false,
     "Values": {
       "FUNCTIONS_WORKER_RUNTIME": "python",
-      "AzureWebJobsStorage": "",
+      "AzureWebJobsStorage__accountName": "",
       "MyBindingConnection": "",
       "AzureWebJobs.HttpExample.Disabled": "true"
     },
@@ -11,4 +11,4 @@
       "CORS": "*",
       "CORSCredentials": false
     }
-  }
+  }

code/backend/batch/utilities/chat_history/database_factory.py

@@ -25,7 +25,7 @@ def get_conversation_client():
                 f"https://{env_helper.AZURE_COSMOSDB_ACCOUNT}.documents.azure.com:443/"
             )
             credential = (
-                get_azure_credential()
+                get_azure_credential(env_helper.MANAGED_IDENTITY_CLIENT_ID)
                 if not env_helper.AZURE_COSMOSDB_ACCOUNT_KEY
                 else env_helper.AZURE_COSMOSDB_ACCOUNT_KEY
             )

code/backend/batch/utilities/chat_history/postgresdbservice.py

@@ -2,6 +2,7 @@
 import asyncpg
 from datetime import datetime, timezone
 from ..helpers.azure_credential_utils import get_azure_credential
+from ..helpers.env_helper import EnvHelper
 
 from .database_client_base import DatabaseClientBase
 
@@ -13,6 +14,7 @@ class PostgresConversationClient(DatabaseClientBase):
     def __init__(
         self, user: str, host: str, database: str, enable_message_feedback: bool = False
     ):
+        self.env_helper = EnvHelper()
         self.user = user
         self.host = host
         self.database = database
@@ -21,7 +23,7 @@ def __init__(
 
     async def connect(self):
         try:
-            credential = get_azure_credential()
+            credential = get_azure_credential(self.env_helper.MANAGED_IDENTITY_CLIENT_ID)
             token = credential.get_token(
                 "https://ossrdbms-aad.database.windows.net/.default"
             ).token
@@ -31,7 +33,7 @@ async def connect(self):
                 database=self.database,
                 password=token,
                 port=5432,
-                ssl="require",
+                ssl=True,
             )
         except Exception as e:
             logger.error("Failed to connect to PostgreSQL: %s", e)

code/backend/batch/utilities/helpers/azure_blob_storage_client.py

@@ -25,7 +25,7 @@ def create_queue_client():
         return QueueClient(
             account_url=f"https://{env_helper.AZURE_BLOB_ACCOUNT_NAME}.queue.core.windows.net/",
             queue_name=env_helper.DOCUMENT_PROCESSING_QUEUE_NAME,
-            credential=get_azure_credential(),
+            credential=get_azure_credential(env_helper.MANAGED_IDENTITY_CLIENT_ID),
             message_encode_policy=BinaryBase64EncodePolicy(),
         )
 
@@ -56,7 +56,7 @@ def __init__(
         if self.auth_type == "rbac":
             self.account_key = None
             self.blob_service_client = BlobServiceClient(
-                account_url=self.endpoint, credential=get_azure_credential()
+                account_url=self.endpoint, credential=get_azure_credential(env_helper.MANAGED_IDENTITY_CLIENT_ID)
             )
             self.user_delegation_key = self.request_user_delegation_key(
                 blob_service_client=self.blob_service_client

code/backend/batch/utilities/helpers/azure_computer_vision_client.py

@@ -27,6 +27,7 @@ def __init__(self, env_helper: EnvHelper) -> None:
         self.model_version = (
             env_helper.AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION
         )
+        self.managed_identity_client_id = env_helper.MANAGED_IDENTITY_CLIENT_ID
 
     def vectorize_image(self, image_url: str) -> list[float]:
         logger.info(f"Making call to computer vision to vectorize image: {image_url}")
@@ -57,7 +58,7 @@ def __make_request(self, path: str, body) -> Response:
                 headers["Ocp-Apim-Subscription-Key"] = self.key
             else:
                 token_provider = get_bearer_token_provider(
-                    get_azure_credential(), self.__TOKEN_SCOPE
+                    get_azure_credential(self.managed_identity_client_id), self.__TOKEN_SCOPE
                 )
                 headers["Authorization"] = "Bearer " + token_provider()
 

code/backend/batch/utilities/helpers/azure_form_recognizer_helper.py

@@ -19,7 +19,7 @@ def __init__(self) -> None:
         if env_helper.AZURE_AUTH_TYPE == "rbac":
             self.document_analysis_client = DocumentAnalysisClient(
                 endpoint=self.AZURE_FORM_RECOGNIZER_ENDPOINT,
-                credential=get_azure_credential(),
+                credential=get_azure_credential(env_helper.MANAGED_IDENTITY_CLIENT_ID),
                 headers={
                     "x-ms-useragent": "chat-with-your-data-solution-accelerator/1.0.0"
                 },

code/backend/batch/utilities/helpers/azure_postgres_helper.py

@@ -24,14 +24,14 @@ def _create_search_client(self):
             dbname = self.env_helper.POSTGRESQL_DATABASE
 
             # Acquire the access token
-            credential = get_azure_credential()
+            credential = get_azure_credential(self.env_helper.MANAGED_IDENTITY_CLIENT_ID)
             access_token = credential.get_token(
                 "https://ossrdbms-aad.database.windows.net/.default"
             )
 
             # Use the token in the connection string
             conn_string = (
-                f"host={host} user={user} dbname={dbname} password={access_token.token}"
+                f"host={host} user={user} dbname={dbname} password={access_token.token} sslmode=require"
             )
             self.conn = psycopg2.connect(conn_string)
             logger.info("Connected to Azure PostgreSQL successfully.")

code/backend/batch/utilities/helpers/azure_search_helper.py

@@ -49,7 +49,7 @@ def _search_credential(self):
         if self.env_helper.is_auth_type_keys():
             return AzureKeyCredential(self.env_helper.AZURE_SEARCH_KEY)
         else:
-            return get_azure_credential()
+            return get_azure_credential(self.env_helper.MANAGED_IDENTITY_CLIENT_ID)
 
     def _create_search_client(
         self, search_credential: Union[AzureKeyCredential, get_azure_credential]
@@ -285,7 +285,7 @@ def get_conversation_logger(self):
         ]
 
         if self.env_helper.AZURE_AUTH_TYPE == "rbac":
-            credential = get_azure_credential()
+            credential = get_azure_credential(self.env_helper.MANAGED_IDENTITY_CLIENT_ID)
             return AzureSearch(
                 azure_search_endpoint=self.env_helper.AZURE_SEARCH_SERVICE,
                 azure_search_key=None,  # Remove API key

code/backend/batch/utilities/helpers/env_helper.py

@@ -35,10 +35,13 @@ def __load_config(self, **kwargs) -> None:
         self.secretHelper = SecretHelper()
 
         self.LOGLEVEL = os.environ.get("LOGLEVEL", "INFO").upper()
+        self.APP_ENV = os.getenv("APP_ENV", "Prod").lower()
 
         # Azure
         self.AZURE_SUBSCRIPTION_ID = os.getenv("AZURE_SUBSCRIPTION_ID", "")
         self.AZURE_RESOURCE_GROUP = os.getenv("AZURE_RESOURCE_GROUP", "")
+        self.MANAGED_IDENTITY_CLIENT_ID = os.getenv("MANAGED_IDENTITY_CLIENT_ID", "")
+        self.MANAGED_IDENTITY_RESOURCE_ID = os.getenv("MANAGED_IDENTITY_RESOURCE_ID", "")
 
         # Azure Search
         self.AZURE_SEARCH_SERVICE = os.getenv("AZURE_SEARCH_SERVICE", "")
@@ -217,7 +220,7 @@ def __load_config(self, **kwargs) -> None:
         )
 
         self.AZURE_TOKEN_PROVIDER = get_bearer_token_provider(
-            get_azure_credential(), "https://cognitiveservices.azure.com/.default"
+            get_azure_credential(self.MANAGED_IDENTITY_CLIENT_ID), "https://cognitiveservices.azure.com/.default"
         )
         self.ADVANCED_IMAGE_PROCESSING_MAX_IMAGES = self.get_env_var_int(
             "ADVANCED_IMAGE_PROCESSING_MAX_IMAGES", 1
@@ -234,7 +237,7 @@ def __load_config(self, **kwargs) -> None:
         self.AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION = os.getenv(
             "AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION", "2023-04-15"
         )
-        self.FUNCTION_KEY = os.getenv("FUNCTION_KEY", "")
+        self.FUNCTION_KEY = self.secretHelper.get_secret("FUNCTION_KEY")
 
         # Initialize Azure keys based on authentication type and environment settings.
         # When AZURE_AUTH_TYPE is "rbac", azure keys are None or an empty string.
@@ -243,7 +246,6 @@ def __load_config(self, **kwargs) -> None:
             self.AZURE_OPENAI_API_KEY = ""
             self.AZURE_SPEECH_KEY = None
             self.AZURE_COMPUTER_VISION_KEY = None
-            self.FUNCTION_KEY = self.secretHelper.get_secret("FUNCTION_KEY")
         else:
             self.AZURE_SEARCH_KEY = self.secretHelper.get_secret("AZURE_SEARCH_KEY")
             self.AZURE_OPENAI_API_KEY = self.secretHelper.get_secret(
@@ -429,8 +431,11 @@ def __init__(self) -> None:
         self.USE_KEY_VAULT = os.getenv("USE_KEY_VAULT", "").lower() == "true"
         self.secret_client = None
         if self.USE_KEY_VAULT:
+            vault_endpoint = os.environ.get("AZURE_KEY_VAULT_ENDPOINT")
+            if not vault_endpoint:
+                raise ValueError("AZURE_KEY_VAULT_ENDPOINT environment variable is required when USE_KEY_VAULT is true")
             self.secret_client = SecretClient(
-                os.environ.get("AZURE_KEY_VAULT_ENDPOINT"), get_azure_credential()
+                vault_endpoint, get_azure_credential(client_id=os.getenv("MANAGED_IDENTITY_CLIENT_ID", None))
             )
 
     def get_secret(self, secret_name: str) -> str:

code/backend/batch/utilities/helpers/llm_helper.py

@@ -166,7 +166,7 @@ def get_sk_service_settings(self, service: AzureChatCompletion):
     def get_ml_client(self):
         if not hasattr(self, "_ml_client"):
             self._ml_client = MLClient(
-                get_azure_credential(),
+                get_azure_credential(self.env_helper.MANAGED_IDENTITY_CLIENT_ID),
                 self.env_helper.AZURE_SUBSCRIPTION_ID,
                 self.env_helper.AZURE_RESOURCE_GROUP,
                 self.env_helper.AZURE_ML_WORKSPACE_NAME,

code/backend/batch/utilities/integrated_vectorization/azure_search_datasource.py

@@ -1,6 +1,7 @@
 from azure.search.documents.indexes.models import (
     SearchIndexerDataContainer,
     SearchIndexerDataSourceConnection,
+    SearchIndexerDataUserAssignedIdentity,
 )
 from azure.search.documents.indexes._generated.models import (
     NativeBlobSoftDeleteDeletionDetectionPolicy,
@@ -19,7 +20,7 @@ def __init__(self, env_helper: EnvHelper):
             (
                 AzureKeyCredential(self.env_helper.AZURE_SEARCH_KEY)
                 if self.env_helper.is_auth_type_keys()
-                else get_azure_credential()
+                else get_azure_credential(self.env_helper.MANAGED_IDENTITY_CLIENT_ID)
             ),
         )
 
@@ -35,6 +36,13 @@ def create_or_update_datasource(self):
             connection_string=connection_string,
             container=container,
             data_deletion_detection_policy=NativeBlobSoftDeleteDeletionDetectionPolicy(),
+            identity=(
+                None
+                if getattr(self.env_helper, "APP_ENV", "").lower() == "dev"
+                else SearchIndexerDataUserAssignedIdentity(
+                    user_assigned_identity=self.env_helper.MANAGED_IDENTITY_RESOURCE_ID
+                )
+            ),
         )
         self.indexer_client.create_or_update_data_source_connection(
             data_source_connection