Merge branch 'dev' into hb-bug-23482

Author: Harsh-Microsoft

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/devcontainers/python:3.11
+FROM mcr.microsoft.com/devcontainers/python:3.11-bookworm
 
 # install git
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \

.github/workflows/build-docker.yml

@@ -62,7 +62,7 @@ jobs:
         context: .
         file: ${{ inputs.dockerfile }}
         push: ${{ inputs.push }}
-        cache-from: type=registry,ref=${{ inputs.new_registry }}/${{ inputs.app_name }}:${{ github.ref_name == 'main' && 'latest' || github.ref_name == 'dev' && 'dev' || github.ref_name == 'demo' && 'demo'|| github.ref_name == 'dependabotchanges' && 'dependabotchanges' || github.head_ref || github.ref_name }}
+        cache-from: type=registry,ref=${{ inputs.new_registry }}/${{ inputs.app_name }}:${{ github.ref_name == 'main' && 'latest_waf' || github.ref_name == 'dev' && 'dev' || github.ref_name == 'demo' && 'demo'|| github.ref_name == 'dependabotchanges' && 'dependabotchanges' || github.head_ref || github.ref_name }}
         tags: |
-          ${{ inputs.new_registry }}/${{ inputs.app_name }}:${{ github.ref_name == 'main' && 'latest' || github.ref_name == 'dev' && 'dev' || github.ref_name == 'demo' && 'demo'|| github.ref_name == 'dependabotchanges' && 'dependabotchanges' || github.head_ref || 'default' }}
-          ${{ inputs.new_registry }}/${{ inputs.app_name }}:${{ steps.date.outputs.date }}_${{ github.run_number }}
+          ${{ inputs.new_registry }}/${{ inputs.app_name }}:${{ github.ref_name == 'main' && 'latest_waf' || github.ref_name == 'dev' && 'dev' || github.ref_name == 'demo' && 'demo'|| github.ref_name == 'dependabotchanges' && 'dependabotchanges' || github.head_ref || 'default' }}
+          ${{ inputs.new_registry }}/${{ inputs.app_name }}:${{ github.ref_name == 'main' && 'latest_waf' || github.ref_name == 'dev' && 'dev' || github.ref_name == 'demo' && 'demo'|| github.ref_name == 'dependabotchanges' && 'dependabotchanges' || github.head_ref || 'default' }}_${{ steps.date.outputs.date }}_${{ github.run_number }}

.github/workflows/ci.yml

@@ -36,6 +36,13 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Install AZD
+        run: |
+          set -e
+          echo "Fetching deployment output..."
+          # Install azd (Azure Developer CLI) - required by process_sample_data.sh
+          curl -fsSL https://aka.ms/install-azd.sh | bash
+
       - name: Run Quota Check
         id: quota-check
         run: |
@@ -79,6 +86,45 @@ jobs:
           echo "Selected Region: $VALID_REGION"
           echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV
 
+      - name: Generate Resource Group Name
+        id: generate_rg_name
+        run: |
+          echo "Generating a unique resource group name..."
+          ACCL_NAME="cwyd"  # Account name as specified
+          SHORT_UUID=$(uuidgen | cut -d'-' -f1)
+          UNIQUE_RG_NAME="arg-${ACCL_NAME}-${SHORT_UUID}"
+          echo "RESOURCE_GROUP_NAME=${UNIQUE_RG_NAME}" >> $GITHUB_ENV
+          echo "Generated RESOURCE_GROUP_NAME: ${UNIQUE_RG_NAME}"
+
+      - name: Check and Create Resource Group
+        id: check_create_rg
+        run: |
+          echo "RESOURCE_GROUP: ${{ env.RESOURCE_GROUP_NAME }}"
+          set -e
+          echo "Checking if resource group exists..."
+          rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
+          if [ "$rg_exists" = "false" ]; then
+            echo "Resource group does not exist. Creating..."
+            az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.AZURE_LOCATION }} --tags SecurityControl=Ignore || { echo "Error creating resource group"; exit 1; }
+          else
+            echo "Resource group already exists."
+          fi
+          # Set output for other jobs
+          echo "RESOURCE_GROUP_NAME=${{ env.RESOURCE_GROUP_NAME }}" >> $GITHUB_OUTPUT
+
+
+      - name: Generate Unique Solution Prefix
+        id: generate_solution_prefix
+        run: |
+          set -e
+          COMMON_PART="pslc"
+          TIMESTAMP=$(date +%s)
+          UPDATED_TIMESTAMP=$(echo $TIMESTAMP | tail -c 3)
+          UNIQUE_SOLUTION_SUFFIX="${COMMON_PART}${UPDATED_TIMESTAMP}"
+          echo "SOLUTION_SUFFIX=${UNIQUE_SOLUTION_SUFFIX}" >> $GITHUB_ENV
+          echo "SOLUTION_SUFFIX=${UNIQUE_SOLUTION_SUFFIX}" >> $GITHUB_OUTPUT
+          echo "Generated SOLUTION_SUFFIX: ${UNIQUE_SOLUTION_SUFFIX}"
+
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:
@@ -90,11 +136,11 @@ jobs:
         id: set-image-tag
         run: |
           if [[ "${{ github.event_name }}" == "schedule" ]]; then
-            echo "imageTag=latest" >> $GITHUB_ENV
-            echo "::set-output name=imageTag::latest"
+            echo "imageTag=latest_waf" >> $GITHUB_ENV
+            echo "::set-output name=imageTag::latest_waf"
           elif [[ "${{ github.ref_name }}" == "main" ]]; then
-            echo "imageTag=latest" >> $GITHUB_ENV
-            echo "::set-output name=imageTag::latest"
+            echo "imageTag=latest_waf" >> $GITHUB_ENV
+            echo "::set-output name=imageTag::latest_waf"
           else
             echo "imageTag=${{ github.ref_name }}" >> $GITHUB_ENV
             echo "::set-output name=imageTag::${{ github.ref_name }}"
@@ -103,8 +149,9 @@ jobs:
       - name: Pre-build image and deploy
         uses: devcontainers/[email protected]
         env:
-                      AZURE_ENV_NAME: ${{ github.run_id }}
+                      AZURE_ENV_NAME: ${{ env.SOLUTION_SUFFIX }}
                       AZURE_LOCATION: ${{ env.AZURE_LOCATION }}
+                      AZURE_RESOURCE_GROUP: ${{ env.RESOURCE_GROUP_NAME }}
         with:
                             push: never
                             imageName: ghcr.io/azure-samples/chat-with-your-data-solution-accelerator
@@ -141,6 +188,7 @@ jobs:
                               AZURE_SUBSCRIPTION_ID
                               AZURE_ENV_NAME
                               AZURE_LOCATION
+                              AZURE_RESOURCE_GROUP
                               AUTH_ENABLED=false
                               AZURE_USE_AUTHENTICATION=false
                               AZURE_ENABLE_AUTH=false

Makefile

@@ -65,7 +65,17 @@ azd-login: ## 🔑 Login to Azure with azd and a SPN
 # Fixed Makefile section for deploy target
 deploy: azd-login ## Deploy everything to Azure
 	@echo -e "\e[34m$@\e[0m" || true
-	@azd env new ${AZURE_ENV_NAME}
+	@echo "AZURE_ENV_NAME: '${AZURE_ENV_NAME}'"
+	@echo "AZURE_LOCATION: '${AZURE_LOCATION}'"
+	@echo "AZURE_RESOURCE_GROUP: '${AZURE_RESOURCE_GROUP}'"
+
+	# Validate required variables
+	@if [ -z "${AZURE_ENV_NAME}" ]; then echo "❌ AZURE_ENV_NAME not set"; exit 1; fi
+	@if [ -z "${AZURE_LOCATION}" ]; then echo "❌ AZURE_LOCATION not set"; exit 1; fi
+	@if [ -z "${AZURE_RESOURCE_GROUP}" ]; then echo "❌ AZURE_RESOURCE_GROUP not set"; exit 1; fi
+
+	@azd env new ${AZURE_ENV_NAME} --location ${AZURE_LOCATION}
+	@azd env set AZURE_RESOURCE_GROUP ${AZURE_RESOURCE_GROUP}
 
 	# Provision and deploy
 	@azd provision --no-prompt

azure.yaml

@@ -2,7 +2,7 @@
 
 name: chat-with-your-data-solution-accelerator
 metadata:
-  template: [email protected]
+   template: [email protected]
 hooks:
   postprovision:
     # run: ./infra/prompt-flow/create-prompt-flow.sh

code/backend/batch/local.settings.json.sample

@@ -2,7 +2,7 @@
     "IsEncrypted": false,
     "Values": {
       "FUNCTIONS_WORKER_RUNTIME": "python",
-      "AzureWebJobsStorage": "",
+      "AzureWebJobsStorage__accountName": "",
       "MyBindingConnection": "",
       "AzureWebJobs.HttpExample.Disabled": "true"
     },
@@ -11,4 +11,4 @@
       "CORS": "*",
       "CORSCredentials": false
     }
-  }
+  }

code/backend/batch/utilities/chat_history/database_factory.py

@@ -25,7 +25,7 @@ def get_conversation_client():
                 f"https://{env_helper.AZURE_COSMOSDB_ACCOUNT}.documents.azure.com:443/"
             )
             credential = (
-                get_azure_credential()
+                get_azure_credential(env_helper.MANAGED_IDENTITY_CLIENT_ID)
                 if not env_helper.AZURE_COSMOSDB_ACCOUNT_KEY
                 else env_helper.AZURE_COSMOSDB_ACCOUNT_KEY
             )

code/backend/batch/utilities/chat_history/postgresdbservice.py

@@ -2,6 +2,7 @@
 import asyncpg
 from datetime import datetime, timezone
 from ..helpers.azure_credential_utils import get_azure_credential
+from ..helpers.env_helper import EnvHelper
 
 from .database_client_base import DatabaseClientBase
 
@@ -13,6 +14,7 @@ class PostgresConversationClient(DatabaseClientBase):
     def __init__(
         self, user: str, host: str, database: str, enable_message_feedback: bool = False
     ):
+        self.env_helper = EnvHelper()
         self.user = user
         self.host = host
         self.database = database
@@ -21,7 +23,7 @@ def __init__(
 
     async def connect(self):
         try:
-            credential = get_azure_credential()
+            credential = get_azure_credential(self.env_helper.MANAGED_IDENTITY_CLIENT_ID)
             token = credential.get_token(
                 "https://ossrdbms-aad.database.windows.net/.default"
             ).token
@@ -31,7 +33,7 @@ async def connect(self):
                 database=self.database,
                 password=token,
                 port=5432,
-                ssl="require",
+                ssl=True,
             )
         except Exception as e:
             logger.error("Failed to connect to PostgreSQL: %s", e)

code/backend/batch/utilities/helpers/azure_blob_storage_client.py

@@ -25,7 +25,7 @@ def create_queue_client():
         return QueueClient(
             account_url=f"https://{env_helper.AZURE_BLOB_ACCOUNT_NAME}.queue.core.windows.net/",
             queue_name=env_helper.DOCUMENT_PROCESSING_QUEUE_NAME,
-            credential=get_azure_credential(),
+            credential=get_azure_credential(env_helper.MANAGED_IDENTITY_CLIENT_ID),
             message_encode_policy=BinaryBase64EncodePolicy(),
         )
 
@@ -56,7 +56,7 @@ def __init__(
         if self.auth_type == "rbac":
             self.account_key = None
             self.blob_service_client = BlobServiceClient(
-                account_url=self.endpoint, credential=get_azure_credential()
+                account_url=self.endpoint, credential=get_azure_credential(env_helper.MANAGED_IDENTITY_CLIENT_ID)
             )
             self.user_delegation_key = self.request_user_delegation_key(
                 blob_service_client=self.blob_service_client

code/backend/batch/utilities/helpers/azure_computer_vision_client.py

@@ -27,6 +27,7 @@ def __init__(self, env_helper: EnvHelper) -> None:
         self.model_version = (
             env_helper.AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION
         )
+        self.managed_identity_client_id = env_helper.MANAGED_IDENTITY_CLIENT_ID
 
     def vectorize_image(self, image_url: str) -> list[float]:
         logger.info(f"Making call to computer vision to vectorize image: {image_url}")
@@ -57,7 +58,7 @@ def __make_request(self, path: str, body) -> Response:
                 headers["Ocp-Apim-Subscription-Key"] = self.key
             else:
                 token_provider = get_bearer_token_provider(
-                    get_azure_credential(), self.__TOKEN_SCOPE
+                    get_azure_credential(self.managed_identity_client_id), self.__TOKEN_SCOPE
                 )
                 headers["Authorization"] = "Bearer " + token_provider()