Merge branch 'dev' of https://github.com/microsoft/Conversation-Knowledge-Mining-Solution-Accelerator into psl-pk-aifoundryresource

Author: Pavan-Microsoft

.github/workflows/test.yml

@@ -6,6 +6,7 @@ on:
       - main
       - dev
       - demo
+      - psl-backend-unittest
   pull_request:
     types:
       - opened
@@ -18,49 +19,48 @@ on:
       - demo
 
 jobs:
-#   frontend_tests:
-#     runs-on: ubuntu-latest
-
-#     steps:
-#       - name: Checkout code
-#         uses: actions/checkout@v3
-
-#       - name: Set up Node.js
-#         uses: actions/setup-node@v3
-#         with:
-#           node-version: '20'
-
-#       - name: Check if Frontend Test Files Exist
-#         id: check_frontend_tests
-#         run: |
-#           if [ -z "$(find App/frontend/src -type f -name '*.test.js' -o -name '*.test.ts' -o -name '*.test.tsx')" ]; then
-#             echo "No frontend test files found, skipping frontend tests."
-#             echo "skip_frontend_tests=true" >> $GITHUB_ENV
-#           else
-#             echo "Frontend test files found, running tests."
-#             echo "skip_frontend_tests=false" >> $GITHUB_ENV
-#           fi
-
-#       - name: Install Frontend Dependencies
-#         if: env.skip_frontend_tests == 'false'
-#         run: |
-#           cd App/frontend
-#           npm install
-
-#       - name: Run Frontend Tests with Coverage
-#         if: env.skip_frontend_tests == 'false'
-#         run: |
-#           cd App/frontend
-#           npm run test -- --coverage
-
-#       - name: Skip Frontend Tests
-#         if: env.skip_frontend_tests == 'true'
-#         run: |
-#           echo "Skipping frontend tests because no test files were found."
+  #   frontend_tests:
+  #     runs-on: ubuntu-latest
+
+  #     steps:
+  #       - name: Checkout code
+  #         uses: actions/checkout@v3
+
+  #       - name: Set up Node.js
+  #         uses: actions/setup-node@v3
+  #         with:
+  #           node-version: '20'
+
+  #       - name: Check if Frontend Test Files Exist
+  #         id: check_frontend_tests
+  #         run: |
+  #           if [ -z "$(find App/frontend/src -type f -name '*.test.js' -o -name '*.test.ts' -o -name '*.test.tsx')" ]; then
+  #             echo "No frontend test files found, skipping frontend tests."
+  #             echo "skip_frontend_tests=true" >> $GITHUB_ENV
+  #           else
+  #             echo "Frontend test files found, running tests."
+  #             echo "skip_frontend_tests=false" >> $GITHUB_ENV
+  #           fi
+
+  #       - name: Install Frontend Dependencies
+  #         if: env.skip_frontend_tests == 'false'
+  #         run: |
+  #           cd App/frontend
+  #           npm install
+
+  #       - name: Run Frontend Tests with Coverage
+  #         if: env.skip_frontend_tests == 'false'
+  #         run: |
+  #           cd App/frontend
+  #           npm run test -- --coverage
+
+  #       - name: Skip Frontend Tests
+  #         if: env.skip_frontend_tests == 'true'
+  #         run: |
+  #           echo "Skipping frontend tests because no test files were found."
 
   backend_tests:
     runs-on: ubuntu-latest
-    
 
     steps:
       - name: Checkout code
@@ -69,7 +69,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
+          python-version: "3.11"
 
       - name: Install Backend Dependencies
         run: |
@@ -81,7 +81,7 @@ jobs:
       - name: Check if Backend Test Files Exist
         id: check_backend_tests
         run: |
-          if [ -z "$(find src/api -type f -name 'test_*.py')" ]; then
+          if [ -z "$(find src/tests/api -type f -name 'test_*.py')" ]; then
             echo "No backend test files found, skipping backend tests."
             echo "skip_backend_tests=true" >> $GITHUB_ENV
           else
@@ -94,8 +94,6 @@ jobs:
         run: |
           pytest --cov=. --cov-report=term-missing --cov-report=xml
 
-
-
       - name: Skip Backend Tests
         if: env.skip_backend_tests == 'true'
         run: |

documents/CustomizingAzdParameters.md

@@ -45,4 +45,9 @@ Change the Embedding Deployment Capacity (choose a number based on available emb
 
 ```shell
 azd env set AZURE_OPENAI_EMBEDDING_MODEL_CAPACITY 80
+```
+
+Set the Log Analytics Workspace Id if you need to reuse the existing workspace which is already existing
+```shell
+azd env set AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID '/subscriptions/<subscription-id>/resourceGroups/<resource-group>/providers/Microsoft.OperationalInsights/workspaces/<workspace-name>'
 ```

documents/DeploymentGuide.md

@@ -116,6 +116,8 @@ When you start the deployment, most parameters will have **default values**, but
 | **GPT Model Deployment Capacity** | Configure capacity for **GPT models**. | 30k |
 | **Embedding Model** | Default: **text-embedding-ada-002**. | text-embedding-ada-002 |
 | **Embedding Model Capacity** | Set the capacity for **embedding models**. | 80k |
+| **Existing Log analytics workspace** | To reuse the existing Log analytics workspace Id. |  |
+
 
 </details>
 

documents/Fabric_deployment.md

@@ -3,11 +3,13 @@
 ### How to customize 
 
 If you'd like to customize the solution accelerator, here are some ways you might do that:
-- Ingest your own [audio conversation files](./ConversationalDataFormat.md) by uploading them into the `cu_audio_files_all` lakehouse folder and run the data pipeline
-- Deploy with Microsoft Fabric by following the steps in [Fabric_deployment.md](./Fabric_deployment.md)
+1. Ingest your own [audio conversation files](./ConversationalDataFormat.md) by uploading them into the `cu_audio_files_all` lakehouse folder and run the data pipeline
+2. Deploy with Microsoft Fabric by following the steps in [Fabric_deployment.md](./Fabric_deployment.md)
 
 
-3.  **Create Fabric workspace**
+3.  **Create or Use an Existing Microsoft Fabric Workspace**
+
+    > ℹ️ **Note:** If you already have an existing Microsoft Fabric Workspace, you can skip workspace creation and **continue from Point 5 (Environment Creation)**.
     1.  Navigate to ([Fabric Workspace](https://app.fabric.microsoft.com/))
     2.  Click on Data Engineering experience
     3.  Click on Workspaces from left Navigation
@@ -38,7 +40,7 @@ If you'd like to customize the solution accelerator, here are some ways you migh
          4.   ```cd ./Conversation-Knowledge-Mining-Solution-Accelerator/Deployment/scripts/fabric_scripts```
          5.   ```sh ./run_fabric_items_scripts.sh keyvault_param workspaceid_param solutionprefix_param```
               1.   keyvault_param - the name of the keyvault that was created in Step 1
-              2.   workspaceid_param - the workspaceid created in Step 2
+              2.   workspaceid_param - Existing workspaceid or workspaceid created in Step 3
               3.   solutionprefix_param - prefix used to append to lakehouse upon creation
 5.  **Add App Authentication**
 
@@ -53,4 +55,4 @@ All files WAV files can be uploaded in the corresponding Lakehouse in the data/F
 
 ### Post-deployment
 - To process additional files, manually execute the pipeline_notebook after uploading new files.
-- The OpenAI prompt can be modified within the Fabric notebooks.
+- The OpenAI prompt can be modified within the Fabric notebooks.

infra/deploy_ai_foundry.bicep

@@ -5,11 +5,14 @@ param keyVaultName string
 param cuLocation string
 param deploymentType string
 param gptModelName string
+param gptModelVersion string
 param azureOpenAIApiVersion string
 param gptDeploymentCapacity int
 param embeddingModel string
 param embeddingDeploymentCapacity int
 param managedIdentityObjectId string
+param existingLogAnalyticsWorkspaceId string = ''
+
 var abbrs = loadJsonContent('./abbreviations.json')
 var storageName = '${solutionName}hubstorage'
 var storageSkuName = 'Standard_LRS'
@@ -32,6 +35,7 @@ var aiModelDeployments = [
       name: deploymentType
       capacity: gptDeploymentCapacity
     }
+    version: gptModelVersion
     raiPolicyName: 'Microsoft.Default'
   }
   {
@@ -47,11 +51,21 @@ var aiModelDeployments = [
 
 var containerRegistryNameCleaned = replace(containerRegistryName, '-', '')
 
+var useExisting = !empty(existingLogAnalyticsWorkspaceId)
+var existingLawSubscription = useExisting ? split(existingLogAnalyticsWorkspaceId, '/')[2] : ''
+var existingLawResourceGroup = useExisting ? split(existingLogAnalyticsWorkspaceId, '/')[4] : ''
+var existingLawName = useExisting ? split(existingLogAnalyticsWorkspaceId, '/')[8] : ''
+
 resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' existing = {
   name: keyVaultName
 }
 
-resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2023-09-01' = {
+resource existingLogAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' existing = if (useExisting) {
+  name: existingLawName
+  scope: resourceGroup(existingLawSubscription ,existingLawResourceGroup)
+}
+
+resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2023-09-01' = if (!useExisting){
   name: workspaceName
   location: location
   tags: {}
@@ -71,7 +85,7 @@ resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = {
     Application_Type: 'web'
     publicNetworkAccessForIngestion: 'Enabled'
     publicNetworkAccessForQuery: 'Disabled'
-    WorkspaceResourceId: logAnalytics.id
+    WorkspaceResourceId: useExisting ? existingLogAnalyticsWorkspace.id : logAnalytics.id
   }
 }
 
@@ -542,7 +556,10 @@ output aiSearchService string = aiSearch.name
 output aiProjectName string = aiProject.name
 
 output applicationInsightsId string = applicationInsights.id
-output logAnalyticsWorkspaceResourceName string = logAnalytics.name
+output logAnalyticsWorkspaceResourceName string = useExisting ? existingLogAnalyticsWorkspace.name : logAnalytics.name
+output logAnalyticsWorkspaceResourceGroup string = useExisting ? existingLawResourceGroup : resourceGroup().name
+output logAnalyticsWorkspaceSubscription string = useExisting ? existingLawSubscription : subscription().subscriptionId
+
 output storageAccountName string = storageNameCleaned
 
 output azureOpenAIKeyName string = azureOpenAIApiKeyEntry.name

infra/deploy_post_deployment_scripts.bicep

@@ -17,11 +17,13 @@ param sqlDbName string
 param sqlUsers array = [
 ]
 param logAnalyticsWorkspaceResourceName string
+param logAnalyticsWorkspaceResourceGroup string
+param logAnalyticsWorkspaceSubscription string
 var resourceGroupName = resourceGroup().name
 
 resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2020-10-01' existing = {
   name: logAnalyticsWorkspaceResourceName
-  scope: resourceGroup()
+  scope: resourceGroup(logAnalyticsWorkspaceSubscription, logAnalyticsWorkspaceResourceGroup)
 }
 
 resource containerAppEnv 'Microsoft.App/managedEnvironments@2022-03-01' = {

infra/main.bicep

@@ -6,6 +6,9 @@ var abbrs = loadJsonContent('./abbreviations.json')
 @description('A unique prefix for all resources in this deployment. This should be 3-20 characters long:')
 param environmentName string
 
+@description('Optional: Existing Log Analytics Workspace Resource ID')
+param existingLogAnalyticsWorkspaceId string = ''
+
 @minLength(1)
 @description('Location for the Content Understanding service deployment:')
 @allowed(['swedencentral', 'australiaeast'])
@@ -28,18 +31,12 @@ param secondaryLocation string
 ])
 param deploymentType string = 'GlobalStandard'
 
-@minLength(1)
 @description('Name of the GPT model to deploy:')
-@allowed([
-  'gpt-4o-mini'
-  'gpt-4o'
-  'gpt-4'
-])
 param gptModelName string = 'gpt-4o-mini'
 
-// @minLength(1)
-// @description('Version of the GPT model to deploy:')
-// param gptModelVersion string = '2024-02-15-preview' //'2024-08-06'
+@description('Version of the GPT model to deploy:')
+param gptModelVersion string = '2024-07-18'
+
 var azureOpenAIApiVersion = '2025-01-01-preview'
 
 @minValue(10)
@@ -102,11 +99,14 @@ module aifoundry 'deploy_ai_foundry.bicep' = {
     cuLocation: contentUnderstandingLocation
     deploymentType: deploymentType
     gptModelName: gptModelName
+    gptModelVersion: gptModelVersion
     azureOpenAIApiVersion: azureOpenAIApiVersion
     gptDeploymentCapacity: gptDeploymentCapacity
     embeddingModel: embeddingModel
     embeddingDeploymentCapacity: embeddingDeploymentCapacity
     managedIdentityObjectId: managedIdentityModule.outputs.managedIdentityOutput.objectId
+    existingLogAnalyticsWorkspaceId: existingLogAnalyticsWorkspaceId
+
   }
   scope: resourceGroup(resourceGroup().name)
 }
@@ -168,6 +168,8 @@ module uploadFiles 'deploy_post_deployment_scripts.bicep' = {
     managedIdentityClientId:managedIdentityModule.outputs.managedIdentityOutput.clientId
     keyVaultName:aifoundry.outputs.keyvaultName
     logAnalyticsWorkspaceResourceName: aifoundry.outputs.logAnalyticsWorkspaceResourceName
+    logAnalyticsWorkspaceResourceGroup: aifoundry.outputs.logAnalyticsWorkspaceResourceGroup
+    logAnalyticsWorkspaceSubscription: aifoundry.outputs.logAnalyticsWorkspaceSubscription
     sqlServerName: sqlDBModule.outputs.sqlServerName
     sqlDbName: sqlDBModule.outputs.sqlDbName
     sqlUsers: [

infra/main.bicepparam

@@ -6,6 +6,8 @@ param contentUnderstandingLocation = readEnvironmentVariable('AZURE_CONTENT_UNDE
 param secondaryLocation = readEnvironmentVariable('AZURE_SECONDARY_LOCATION', 'eastus2')
 param deploymentType = readEnvironmentVariable('AZURE_OPEN_AI_MODEL_DEPLOYMENT_TYPE', 'GlobalStandard')
 param gptModelName = readEnvironmentVariable('AZURE_OPEN_AI_DEPLOYMENT_MODEL', 'gpt-4o-mini')
+param gptModelVersion = readEnvironmentVariable('AZURE_ENV_MODEL_VERSION', '2024-07-18')
 param gptDeploymentCapacity = int(readEnvironmentVariable('AZURE_OPEN_AI_DEPLOYMENT_MODEL_CAPACITY', '30'))
 param embeddingModel = readEnvironmentVariable('AZURE_OPENAI_EMBEDDING_MODEL', 'text-embedding-ada-002')
 param embeddingDeploymentCapacity = int(readEnvironmentVariable('AZURE_OPENAI_EMBEDDING_MODEL_CAPACITY', '80'))
+param existingLogAnalyticsWorkspaceId = readEnvironmentVariable('AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID', '')