feat: Unit test code changes and EXP changes for Log analytics

Author: Avijit-Microsoft

.github/workflows/test.yml

@@ -6,6 +6,7 @@ on:
       - main
       - dev
       - demo
+      - psl-backend-unittest
   pull_request:
     types:
       - opened
@@ -18,49 +19,48 @@ on:
       - demo
 
 jobs:
-#   frontend_tests:
-#     runs-on: ubuntu-latest
-
-#     steps:
-#       - name: Checkout code
-#         uses: actions/checkout@v3
-
-#       - name: Set up Node.js
-#         uses: actions/setup-node@v3
-#         with:
-#           node-version: '20'
-
-#       - name: Check if Frontend Test Files Exist
-#         id: check_frontend_tests
-#         run: |
-#           if [ -z "$(find App/frontend/src -type f -name '*.test.js' -o -name '*.test.ts' -o -name '*.test.tsx')" ]; then
-#             echo "No frontend test files found, skipping frontend tests."
-#             echo "skip_frontend_tests=true" >> $GITHUB_ENV
-#           else
-#             echo "Frontend test files found, running tests."
-#             echo "skip_frontend_tests=false" >> $GITHUB_ENV
-#           fi
-
-#       - name: Install Frontend Dependencies
-#         if: env.skip_frontend_tests == 'false'
-#         run: |
-#           cd App/frontend
-#           npm install
-
-#       - name: Run Frontend Tests with Coverage
-#         if: env.skip_frontend_tests == 'false'
-#         run: |
-#           cd App/frontend
-#           npm run test -- --coverage
-
-#       - name: Skip Frontend Tests
-#         if: env.skip_frontend_tests == 'true'
-#         run: |
-#           echo "Skipping frontend tests because no test files were found."
+  #   frontend_tests:
+  #     runs-on: ubuntu-latest
+
+  #     steps:
+  #       - name: Checkout code
+  #         uses: actions/checkout@v3
+
+  #       - name: Set up Node.js
+  #         uses: actions/setup-node@v3
+  #         with:
+  #           node-version: '20'
+
+  #       - name: Check if Frontend Test Files Exist
+  #         id: check_frontend_tests
+  #         run: |
+  #           if [ -z "$(find App/frontend/src -type f -name '*.test.js' -o -name '*.test.ts' -o -name '*.test.tsx')" ]; then
+  #             echo "No frontend test files found, skipping frontend tests."
+  #             echo "skip_frontend_tests=true" >> $GITHUB_ENV
+  #           else
+  #             echo "Frontend test files found, running tests."
+  #             echo "skip_frontend_tests=false" >> $GITHUB_ENV
+  #           fi
+
+  #       - name: Install Frontend Dependencies
+  #         if: env.skip_frontend_tests == 'false'
+  #         run: |
+  #           cd App/frontend
+  #           npm install
+
+  #       - name: Run Frontend Tests with Coverage
+  #         if: env.skip_frontend_tests == 'false'
+  #         run: |
+  #           cd App/frontend
+  #           npm run test -- --coverage
+
+  #       - name: Skip Frontend Tests
+  #         if: env.skip_frontend_tests == 'true'
+  #         run: |
+  #           echo "Skipping frontend tests because no test files were found."
 
   backend_tests:
     runs-on: ubuntu-latest
-    
 
     steps:
       - name: Checkout code
@@ -69,7 +69,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
+          python-version: "3.11"
 
       - name: Install Backend Dependencies
         run: |
@@ -81,7 +81,7 @@ jobs:
       - name: Check if Backend Test Files Exist
         id: check_backend_tests
         run: |
-          if [ -z "$(find src/api -type f -name 'test_*.py')" ]; then
+          if [ -z "$(find src/tests/api -type f -name 'test_*.py')" ]; then
             echo "No backend test files found, skipping backend tests."
             echo "skip_backend_tests=true" >> $GITHUB_ENV
           else
@@ -94,8 +94,6 @@ jobs:
         run: |
           pytest --cov=. --cov-report=term-missing --cov-report=xml
 
-
-
       - name: Skip Backend Tests
         if: env.skip_backend_tests == 'true'
         run: |

documents/CustomizingAzdParameters.md

@@ -45,4 +45,9 @@ Change the Embedding Deployment Capacity (choose a number based on available emb
 
 ```shell
 azd env set AZURE_OPENAI_EMBEDDING_MODEL_CAPACITY 80
+```
+
+Set the Log Analytics Workspace Id if you need to reuse the existing workspace which is already existing
+```shell
+azd env set AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID '<Existing Log Analytics Workspace Id>'
 ```

documents/DeploymentGuide.md

@@ -116,6 +116,8 @@ When you start the deployment, most parameters will have **default values**, but
 | **GPT Model Deployment Capacity** | Configure capacity for **GPT models**. | 30k |
 | **Embedding Model** | Default: **text-embedding-ada-002**. | text-embedding-ada-002 |
 | **Embedding Model Capacity** | Set the capacity for **embedding models**. | 80k |
+| **Existing Log analytics workspace** | To reuse the existing Log analytics workspace Id. |  |
+
 
 </details>
 

documents/Fabric_deployment.md

@@ -3,11 +3,13 @@
 ### How to customize 
 
 If you'd like to customize the solution accelerator, here are some ways you might do that:
-- Ingest your own [audio conversation files](./ConversationalDataFormat.md) by uploading them into the `cu_audio_files_all` lakehouse folder and run the data pipeline
-- Deploy with Microsoft Fabric by following the steps in [Fabric_deployment.md](./Fabric_deployment.md)
+1. Ingest your own [audio conversation files](./ConversationalDataFormat.md) by uploading them into the `cu_audio_files_all` lakehouse folder and run the data pipeline
+2. Deploy with Microsoft Fabric by following the steps in [Fabric_deployment.md](./Fabric_deployment.md)
 
 
-3.  **Create Fabric workspace**
+3.  **Create or Use an Existing Microsoft Fabric Workspace**
+
+    > ℹ️ **Note:** If you already have an existing Microsoft Fabric Workspace, you can skip workspace creation and **continue from Point 5 (Environment Creation)**.
     1.  Navigate to ([Fabric Workspace](https://app.fabric.microsoft.com/))
     2.  Click on Data Engineering experience
     3.  Click on Workspaces from left Navigation
@@ -38,7 +40,7 @@ If you'd like to customize the solution accelerator, here are some ways you migh
          4.   ```cd ./Conversation-Knowledge-Mining-Solution-Accelerator/Deployment/scripts/fabric_scripts```
          5.   ```sh ./run_fabric_items_scripts.sh keyvault_param workspaceid_param solutionprefix_param```
               1.   keyvault_param - the name of the keyvault that was created in Step 1
-              2.   workspaceid_param - the workspaceid created in Step 2
+              2.   workspaceid_param - Existing workspaceid or workspaceid created in Step 3
               3.   solutionprefix_param - prefix used to append to lakehouse upon creation
 5.  **Add App Authentication**
 
@@ -53,4 +55,4 @@ All files WAV files can be uploaded in the corresponding Lakehouse in the data/F
 
 ### Post-deployment
 - To process additional files, manually execute the pipeline_notebook after uploading new files.
-- The OpenAI prompt can be modified within the Fabric notebooks.
+- The OpenAI prompt can be modified within the Fabric notebooks.

infra/deploy_ai_foundry.bicep

@@ -5,11 +5,14 @@ param keyVaultName string
 param cuLocation string
 param deploymentType string
 param gptModelName string
+param gptModelVersion string
 param azureOpenAIApiVersion string
 param gptDeploymentCapacity int
 param embeddingModel string
 param embeddingDeploymentCapacity int
 param managedIdentityObjectId string
+param existingLogAnalyticsWorkspaceId string = ''
+
 var abbrs = loadJsonContent('./abbreviations.json')
 // var storageName = '${solutionName}hubstorage'
 // var storageSkuName = 'Standard_LRS'
@@ -58,6 +61,7 @@ var aiModelDeployments = [
       name: deploymentType
       capacity: gptDeploymentCapacity
     }
+    version: gptModelVersion
     raiPolicyName: 'Microsoft.Default'
   }
   {
@@ -73,11 +77,20 @@ var aiModelDeployments = [
 
 var containerRegistryNameCleaned = replace(containerRegistryName, '-', '')
 
+var useExisting = !empty(existingLogAnalyticsWorkspaceId)
+var existingLawResourceGroup = useExisting ? split(existingLogAnalyticsWorkspaceId, '/')[4] : ''
+var existingLawName = useExisting ? split(existingLogAnalyticsWorkspaceId, '/')[8] : ''
+
 resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' existing = {
   name: keyVaultName
 }
 
-resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2023-09-01' = {
+resource existingLogAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' existing = if (useExisting) {
+  name: existingLawName
+  scope: resourceGroup(existingLawResourceGroup)
+}
+
+resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2023-09-01' = if (!useExisting){
   name: workspaceName
   location: location
   tags: {}
@@ -97,7 +110,7 @@ resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = {
     Application_Type: 'web'
     publicNetworkAccessForIngestion: 'Enabled'
     publicNetworkAccessForQuery: 'Disabled'
-    WorkspaceResourceId: logAnalytics.id
+    WorkspaceResourceId: useExisting ? existingLogAnalyticsWorkspace.id : logAnalytics.id
   }
 }
 
@@ -714,7 +727,9 @@ output aiSearchService string = aiSearch.name
 output aiProjectName string = aiHubProject.name
 
 output applicationInsightsId string = applicationInsights.id
-output logAnalyticsWorkspaceResourceName string = logAnalytics.name
+output logAnalyticsWorkspaceResourceName string = useExisting ? existingLogAnalyticsWorkspace.name : logAnalytics.name
+output logAnalyticsWorkspaceResourceGroup string = useExisting ? existingLawResourceGroup : resourceGroup().name
+
 output storageAccountName string = storageNameCleaned
 
 output azureOpenAIKeyName string = azureOpenAIApiKeyEntry.name

infra/deploy_post_deployment_scripts.bicep

@@ -17,11 +17,12 @@ param sqlDbName string
 param sqlUsers array = [
 ]
 param logAnalyticsWorkspaceResourceName string
+param logAnalyticsWorkspaceResourceGroup string
 var resourceGroupName = resourceGroup().name
 
 resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2020-10-01' existing = {
   name: logAnalyticsWorkspaceResourceName
-  scope: resourceGroup()
+  scope: resourceGroup(logAnalyticsWorkspaceResourceGroup)
 }
 
 resource containerAppEnv 'Microsoft.App/managedEnvironments@2022-03-01' = {

infra/main.bicep

@@ -6,6 +6,9 @@ var abbrs = loadJsonContent('./abbreviations.json')
 @description('A unique prefix for all resources in this deployment. This should be 3-20 characters long:')
 param environmentName string
 
+@description('Optional: Existing Log Analytics Workspace Resource ID')
+param existingLogAnalyticsWorkspaceId string = ''
+
 @minLength(1)
 @description('Location for the Content Understanding service deployment:')
 @allowed(['swedencentral', 'australiaeast'])
@@ -28,18 +31,12 @@ param secondaryLocation string
 ])
 param deploymentType string = 'GlobalStandard'
 
-@minLength(1)
 @description('Name of the GPT model to deploy:')
-@allowed([
-  'gpt-4o-mini'
-  'gpt-4o'
-  'gpt-4'
-])
 param gptModelName string = 'gpt-4o-mini'
 
-// @minLength(1)
-// @description('Version of the GPT model to deploy:')
-// param gptModelVersion string = '2024-02-15-preview' //'2024-08-06'
+@description('Version of the GPT model to deploy:')
+param gptModelVersion string = '2024-07-18'
+
 var azureOpenAIApiVersion = '2024-02-15-preview'
 
 @minValue(10)
@@ -102,11 +99,14 @@ module aifoundry 'deploy_ai_foundry.bicep' = {
     cuLocation: contentUnderstandingLocation
     deploymentType: deploymentType
     gptModelName: gptModelName
+    gptModelVersion: gptModelVersion
     azureOpenAIApiVersion: azureOpenAIApiVersion
     gptDeploymentCapacity: gptDeploymentCapacity
     embeddingModel: embeddingModel
     embeddingDeploymentCapacity: embeddingDeploymentCapacity
     managedIdentityObjectId: managedIdentityModule.outputs.managedIdentityOutput.objectId
+    existingLogAnalyticsWorkspaceId: existingLogAnalyticsWorkspaceId
+
   }
   scope: resourceGroup(resourceGroup().name)
 }
@@ -168,6 +168,7 @@ module uploadFiles 'deploy_post_deployment_scripts.bicep' = {
     managedIdentityClientId:managedIdentityModule.outputs.managedIdentityOutput.clientId
     keyVaultName:aifoundry.outputs.keyvaultName
     logAnalyticsWorkspaceResourceName: aifoundry.outputs.logAnalyticsWorkspaceResourceName
+    logAnalyticsWorkspaceResourceGroup: aifoundry.outputs.logAnalyticsWorkspaceResourceGroup
     sqlServerName: sqlDBModule.outputs.sqlServerName
     sqlDbName: sqlDBModule.outputs.sqlDbName
     sqlUsers: [

infra/main.bicepparam

@@ -6,6 +6,8 @@ param contentUnderstandingLocation = readEnvironmentVariable('AZURE_CONTENT_UNDE
 param secondaryLocation = readEnvironmentVariable('AZURE_SECONDARY_LOCATION', 'eastus2')
 param deploymentType = readEnvironmentVariable('AZURE_OPEN_AI_MODEL_DEPLOYMENT_TYPE', 'GlobalStandard')
 param gptModelName = readEnvironmentVariable('AZURE_OPEN_AI_DEPLOYMENT_MODEL', 'gpt-4o-mini')
+param gptModelVersion = readEnvironmentVariable('AZURE_ENV_MODEL_VERSION', '2024-07-18')
 param gptDeploymentCapacity = int(readEnvironmentVariable('AZURE_OPEN_AI_DEPLOYMENT_MODEL_CAPACITY', '30'))
 param embeddingModel = readEnvironmentVariable('AZURE_OPENAI_EMBEDDING_MODEL', 'text-embedding-ada-002')
 param embeddingDeploymentCapacity = int(readEnvironmentVariable('AZURE_OPENAI_EMBEDDING_MODEL_CAPACITY', '80'))
+param existingLogAnalyticsWorkspaceId = readEnvironmentVariable('AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID', '')

infra/main.json

@@ -5,7 +5,7 @@
     "_generator": {
       "name": "bicep",
       "version": "0.35.1.17967",
-      "templateHash": "15074305105133532037"
+      "templateHash": "17280998536631599018"
     }
   },
   "parameters": {
@@ -53,16 +53,17 @@
     "gptModelName": {
       "type": "string",
       "defaultValue": "gpt-4o-mini",
-      "allowedValues": [
-        "gpt-4o-mini",
-        "gpt-4o",
-        "gpt-4"
-      ],
-      "minLength": 1,
       "metadata": {
         "description": "Name of the GPT model to deploy:"
       }
     },
+    "gptModelVersion": {
+      "type": "string",
+      "defaultValue": "2024-07-18",
+      "metadata": {
+        "description": "Version of the GPT model to deploy:"
+      }
+    },
     "gptDeploymentCapacity": {
       "type": "int",
       "defaultValue": 30,
@@ -589,6 +590,9 @@
           "gptModelName": {
             "value": "[parameters('gptModelName')]"
           },
+          "gptModelVersion": {
+            "value": "[parameters('gptModelVersion')]"
+          },
           "azureOpenAIApiVersion": {
             "value": "[variables('azureOpenAIApiVersion')]"
           },
@@ -612,7 +616,7 @@
             "_generator": {
               "name": "bicep",
               "version": "0.35.1.17967",
-              "templateHash": "5794496619234186044"
+              "templateHash": "796973952642771216"
             }
           },
           "parameters": {
@@ -634,6 +638,9 @@
             "gptModelName": {
               "type": "string"
             },
+            "gptModelVersion": {
+              "type": "string"
+            },
             "azureOpenAIApiVersion": {
               "type": "string"
             },
@@ -903,6 +910,7 @@
                   "name": "[parameters('deploymentType')]",
                   "capacity": "[parameters('gptDeploymentCapacity')]"
                 },
+                "version": "[parameters('gptModelVersion')]",
                 "raiPolicyName": "Microsoft.Default"
               },
               {

pytest.ini

@@ -0,0 +1,7 @@
+[pytest]
+markers =
+    unittest: Unit Tests (relatively fast)
+    functional: Functional Tests (tests that require a running server, with stubbed downstreams)
+    azure: marks tests as extended (run less frequently, relatively slow)
+pythonpath = ./src/api
+log_level=debug