Merge pull request #428 from microsoft/dev

fix: merge dev to main- improving development environment setup, deployment configurations, and documentation

Author: Roopan-Microsoft

.devcontainer/devcontainer.json

@@ -3,7 +3,21 @@
     "image": "mcr.microsoft.com/devcontainers/python:3.11-bullseye",
     "forwardPorts": [50505],
     "features": {
-        "ghcr.io/azure/azure-dev/azd:latest": {}
+        "ghcr.io/devcontainers/features/node:1": {
+            "nodeGypDependencies": true,
+            "installYarnUsingApt": true,
+            "version": "lts",
+            "pnpmVersion": "latest",
+            "nvmVersion": "latest"
+        },
+        "ghcr.io/devcontainers/features/azure-cli:1": {
+            "installBicep": true,
+            "version": "latest",
+            "bicepVersion": "latest"
+        },
+        "ghcr.io/azure/azure-dev/azd:0": {
+            "version": "stable"
+        }
     },
     "customizations": {
         "vscode": {
@@ -16,7 +30,7 @@
             ]
         }
     },
-    "postStartCommand": "git pull origin main && python3 -m pip install -r infra/scripts/index_scripts/requirements.txt && curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash && chmod +x ./scripts/quota_check_params.sh",
+    "postCreateCommand": "bash ./.devcontainer/setup_env.sh",
     "remoteUser": "vscode",
     "hostRequirements": {
         "memory": "4gb"

.devcontainer/setup_env.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+git fetch
+git pull
+
+# provide execute permission to quotacheck script
+sudo chmod +x ./scripts/quota_check_params.sh

.github/workflows/deploy.yml

@@ -7,6 +7,10 @@ on:
     schedule:
       - cron: '0 9,21 * * *'  # Runs at 9:00 AM and 9:00 PM GMT
 
+env:
+  GPT_MIN_CAPACITY: 10
+  TEXT_EMBEDDING_MIN_CAPACITY: 10
+
 jobs:
   deploy:
     runs-on: ubuntu-latest
@@ -30,8 +34,8 @@ jobs:
           export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}
           export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}
           export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
-          export GPT_MIN_CAPACITY="30"
-          export TEXT_EMBEDDING_MIN_CAPACITY="30"
+          export GPT_MIN_CAPACITY=${{ env.GPT_MIN_CAPACITY }}
+          export TEXT_EMBEDDING_MIN_CAPACITY=${{ env.TEXT_EMBEDDING_MIN_CAPACITY }}
           export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}"
 
           chmod +x scripts/checkquota.sh
@@ -114,17 +118,17 @@ jobs:
             --template-file infra/main.json \
             --parameters \
                 environmentName="${{ env.SOLUTION_PREFIX }}" \
-                secondaryLocation="eastus2" \
+                secondaryLocation="northcentralus" \
                 deploymentType="GlobalStandard" \
                 gptModelName="gpt-4o" \
                 azureOpenaiAPIVersion="2024-05-01-preview" \
-                gptDeploymentCapacity="10" \
+                gptDeploymentCapacity=${{ env.GPT_MIN_CAPACITY }} \
                 embeddingModel="text-embedding-ada-002" \
-                embeddingDeploymentCapacity="30" \
+                embeddingDeploymentCapacity=${{ env.TEXT_EMBEDDING_MIN_CAPACITY }} \
                 imageTag="latest"
       
       - name: Extract AI Services and Key Vault Names
-        if: always()
+        if: always() && steps.check_create_rg.outcome == 'success'
         run: |
           echo "Fetching AI Services and Key Vault names before deletion..."
           
@@ -145,7 +149,7 @@ jobs:
           echo "AI_SERVICES=$AI_SERVICES" >> $GITHUB_ENV
 
       - name: Delete Bicep Deployment
-        if: success()
+        if: always()
         run: |
           set -e  
           echo "Checking if resource group exists..."
@@ -162,6 +166,7 @@ jobs:
           fi
 
       - name: Wait for Resource Deletion to Complete
+        if: always()
         run: |
           echo "Waiting for all deployed resources (including AI Services) to be deleted..."
       
@@ -289,7 +294,7 @@ jobs:
 
           EMAIL_BODY=$(cat <<EOF
           {
-            "body": "<p>Dear Team,</p><p>We would like to inform you that the Research Assistant Automation process has encountered an issue and has failed to complete successfully.</p><p><strong>Build URL:</strong> ${RUN_URL}<br> ${OUTPUT}</p><p>Please investigate the matter at your earliest convenience.</p><p>Best regards,<br>Your Automation Team</p>"
+            "body": "<p>Dear Team,</p><p>We would like to inform you that the DocGen Automation process has encountered an issue and has failed to complete successfully.</p><p><strong>Build URL:</strong> ${RUN_URL}<br> ${OUTPUT}</p><p>Please investigate the matter at your earliest convenience.</p><p>Best regards,<br>Your Automation Team</p>"
           }
           EOF
           )

.gitignore

@@ -11,4 +11,5 @@ myenv
 
 scriptsenv/
 
-scriptenv
+scriptenv
+pdf

.vscode/launch.json

@@ -6,7 +6,7 @@
         "type": "python",
         "request": "launch",
         "module": "quart",
-        "cwd": "${workspaceFolder}",
+        "cwd": "${workspaceFolder}/src",
         "env": {
             "QUART_APP": "app:app",
             "QUART_ENV": "development",

README.md

@@ -41,7 +41,7 @@ The sample data is sourced from generic AI-generated promissory notes.The docume
 
 ### Key features
 <details open>
-  <summary>Click to learn more about the key features this solution enables</summary>
+  <summary>Click to learn more about the key features this solution enables</summary>
 
   - **Semantic search** <br/>
   Azure AI Search to enable RAG and grounding of the application on the processed dataset.​
@@ -62,7 +62,7 @@ Quick deploy
 </h2>
 
 ### How to install or deploy
-Follow the quick deploy steps on the deployment guide to deploy this solution to your own Azure subscription.
+Follow the quick deploy steps on the deployment guide to deploy this solution to your own Azure subscription.
 
 [Click here to launch the deployment guide](./docs/DeploymentGuide.md)
 <br/><br/>
@@ -130,7 +130,7 @@ Put your data to work by reducing blank page anxiety, speeding up document draft
 
 ### Business value
 <details>
-  <summary>Click to learn more about what value this solution provides</summary>
+  <summary>Click to learn more about what value this solution provides</summary>
 
   - **Draft templates quickly** <br/>
   Put your data to work to create any kind of document that is supported by a large data library.
@@ -217,4 +217,4 @@ You acknowledge that the Software and Microsoft Products and Services (1) are no
 
 You acknowledge the Software is not subject to SOC 1 and SOC 2 compliance audits. No Microsoft technology, nor any of its component technologies, including the Software, is intended or made available as a substitute for the professional advice, opinion, or judgement of a certified financial services professional. Do not use the Software to replace, substitute, or provide professional financial advice or judgment.
 
-BY ACCESSING OR USING THE SOFTWARE, YOU ACKNOWLEDGE THAT THE SOFTWARE IS NOT DESIGNED OR INTENDED TO SUPPORT ANY USE IN WHICH A SERVICE INTERRUPTION, DEFECT, ERROR, OR OTHER FAILURE OF THE SOFTWARE COULD RESULT IN THE DEATH OR SERIOUS BODILY INJURY OF ANY PERSON OR IN PHYSICAL OR ENVIRONMENTAL DAMAGE (COLLECTIVELY, “HIGH-RISK USE”), AND THAT YOU WILL ENSURE THAT, IN THE EVENT OF ANY INTERRUPTION, DEFECT, ERROR, OR OTHER FAILURE OF THE SOFTWARE, THE SAFETY OF PEOPLE, PROPERTY, AND THE ENVIRONMENT ARE NOT REDUCED BELOW A LEVEL THAT IS REASONABLY, APPROPRIATE, AND LEGAL, WHETHER IN GENERAL OR IN A SPECIFIC INDUSTRY. BY ACCESSING THE SOFTWARE, YOU FURTHER ACKNOWLEDGE THAT YOUR HIGH-RISK USE OF THE SOFTWARE IS AT YOUR OWN RISK.  
+BY ACCESSING OR USING THE SOFTWARE, YOU ACKNOWLEDGE THAT THE SOFTWARE IS NOT DESIGNED OR INTENDED TO SUPPORT ANY USE IN WHICH A SERVICE INTERRUPTION, DEFECT, ERROR, OR OTHER FAILURE OF THE SOFTWARE COULD RESULT IN THE DEATH OR SERIOUS BODILY INJURY OF ANY PERSON OR IN PHYSICAL OR ENVIRONMENTAL DAMAGE (COLLECTIVELY, “HIGH-RISK USE”), AND THAT YOU WILL ENSURE THAT, IN THE EVENT OF ANY INTERRUPTION, DEFECT, ERROR, OR OTHER FAILURE OF THE SOFTWARE, THE SAFETY OF PEOPLE, PROPERTY, AND THE ENVIRONMENT ARE NOT REDUCED BELOW A LEVEL THAT IS REASONABLY, APPROPRIATE, AND LEGAL, WHETHER IN GENERAL OR IN A SPECIFIC INDUSTRY. BY ACCESSING THE SOFTWARE, YOU FURTHER ACKNOWLEDGE THAT YOUR HIGH-RISK USE OF THE SOFTWARE IS AT YOUR OWN RISK.  

azure.yaml

@@ -32,7 +32,7 @@ hooks:
         Write-Host "Web app URL: "
         Write-Host "$env:WEB_APP_URL" -ForegroundColor Cyan
         Write-Host "`nIf you want to use the Sample Data, run the following command in the Bash terminal to process it:"
-        Write-Host "bash ./infra/scripts/process_sample_data.sh $env:STORAGE_ACCOUNT_NAME $env:STORAGE_CONTAINER_NAME $env:KEY_VAULT_NAME $env:COSMOSDB_ACCOUNT_NAME $env:RESOURCE_GROUP_NAME" -ForegroundColor Cyan
+        Write-Host "bash ./infra/scripts/process_sample_data.sh" -ForegroundColor Cyan
       shell: pwsh
       continueOnError: false
       interactive: true
@@ -41,8 +41,8 @@ hooks:
         echo "Web app URL: "
         echo $WEB_APP_URL
         echo ""
-        echo "If you want to use the Sample Data, run the following command in the terminal to process it:"
-        echo "bash ./infra/scripts/process_sample_data.sh $STORAGE_ACCOUNT_NAME $STORAGE_CONTAINER_NAME $KEY_VAULT_NAME $COSMOSDB_ACCOUNT_NAME $RESOURCE_GROUP_NAME"
+        echo "If you want to use the Sample Data, run the following command in the bash terminal to process it:"
+        echo "bash ./infra/scripts/process_sample_data.sh"
       shell: sh
       continueOnError: false
       interactive: true

docs/DeploymentGuide.md

@@ -148,12 +148,16 @@ Once you've opened the project in [Codespaces](#github-codespaces), [Dev Contain
     azd up
     ```
 
-3. Provide an `azd` environment name (e.g., "cpsapp").
+3. Provide an `azd` environment name (e.g., "dgapp").
 4. Select a subscription from your Azure account and choose a location that has quota for all the resources. 
     - This deployment will take *7-10 minutes* to provision the resources in your account and set up the solution with sample data.
     - If you encounter an error or timeout during deployment, changing the location may help, as there could be availability constraints for the resources.
 
 5. Once the deployment has completed successfully and you would like to use the sample data, run the bash command printed in the terminal. The bash command will look like the following: 
+    ```shell 
+    bash ./infra/scripts/process_sample_data.sh
+    ```
+    If you don't have azd env then you need to pass parameters along with the command. Then the command will look like the following:
     ```shell 
     bash ./infra/scripts/process_sample_data.sh <Storage-Account-name> <Storgae-Account-container-name> <Key-Vault-name> <CosmosDB-Account-name> <Resource-Group-name>
     ```

docs/README_LOCAL.md

@@ -1,7 +1,7 @@
 ### Deploy from your local machine
 
 #### Local Setup: Basic Chat Experience
-1. Copy `.env.sample` to a new file called `.env` and configure the settings as described in the [Environment variables](#environment-variables) section.
+1. Copy `.env.sample` present in `src` folder to a new file called `.env` and configure the settings as described in the [Environment variables](#environment-variables) section.
 
     These variables are required:
     - `AZURE_OPENAI_RESOURCE`
@@ -17,7 +17,7 @@
 
     See the [documentation](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/reference#example-response-2) for more information on these parameters.
 
-2. Start the app with `start.cmd`. This will build the frontend, install backend dependencies, and then start the app. Or, just run the backend in debug mode using the VSCode debug configuration in `.vscode/launch.json`.
+2. Start the app with `start.cmd` or `start.sh`. This will build the frontend, install backend dependencies, and then start the app. Or, just run the backend in debug mode using the VSCode debug configuration in `.vscode/launch.json`.
 
 3. You can see the local running app at http://127.0.0.1:50505. If you experience a port conflict and the app does not load, stop the application in the terminal (CTRL-C on Windows), edit the `start.cmd` file and change the port to a value not in use (i.e., 5000).
 
@@ -50,7 +50,7 @@ NOTE: You may find you need to set: MacOS: `export NODE_OPTIONS="--max-old-space
     - `AZURE_SEARCH_STRICTNESS`
     - `AZURE_OPENAI_EMBEDDING_NAME`
 
-3. Start the app with `start.cmd`. This will build the frontend, install backend dependencies, and then start the app. Or, just run the backend in debug mode using the VSCode debug configuration in `.vscode/launch.json`.
+3. Start the app with `start.cmd` or `start.sh`. This will build the frontend, install backend dependencies, and then start the app. Or, just run the backend in debug mode using the VSCode debug configuration in `.vscode/launch.json`.
 4. You can see the local running app at http://127.0.0.1:50505. If you experience a port conflict and the app does not load, stop the application in the terminal (CTRL-C on Windows), edit the `start.cmd` file and change the port to a value not in use (i.e., 5000).
 
 NOTE: You may find you need to set: MacOS: `export NODE_OPTIONS="--max-old-space-size=8192"` or Windows: `set NODE_OPTIONS=--max-old-space-size=8192` to avoid running out of memory when building the frontend.
@@ -62,7 +62,7 @@ To enable chat history, you will need to set up CosmosDB resources. The ARM temp
 - `AZURE_COSMOSDB_CONVERSATIONS_CONTAINER`
 - `AZURE_COSMOSDB_ACCOUNT_KEY`
 
-As above, start the app with `start.cmd`, then visit the local running app at http://127.0.0.1:50505. Or, just run the backend in debug mode using the VSCode debug configuration in `.vscode/launch.json`. If you experience a port conflict and the app does not load, stop the application in the terminal (CTRL-C on Windows), edit the `start.cmd` file and change the port to a value not in use (i.e., 5000).
+As above, start the app with `start.cmd` or `start.sh`, then visit the local running app at http://127.0.0.1:50505. Or, just run the backend in debug mode using the VSCode debug configuration in `.vscode/launch.json`. If you experience a port conflict and the app does not load, stop the application in the terminal (CTRL-C on Windows), edit the `start.cmd` file and change the port to a value not in use (i.e., 5000).
 
 #### Local Setup: Enable Message Feedback
 To enable message feedback, you will need to set up CosmosDB resources. Then specify these additional environment variable:
@@ -75,10 +75,10 @@ To enable message feedback, you will need to set up CosmosDB resources. Then spe
 
 You can use the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) to deploy the app from your local machine. Make sure you have version 2.48.1 or later.
 
-If this is your first time deploying the app, you can use [az webapp up](https://learn.microsoft.com/en-us/cli/azure/webapp?view=azure-cli-latest#az-webapp-up). Run the following two commands from the root folder of the repo, updating the placeholder values to your desired app name, resource group, location, and subscription. You can also change the SKU if desired.
+If this is your first time deploying the app, you can use [az webapp up](https://learn.microsoft.com/en-us/cli/azure/webapp?view=azure-cli-latest#az-webapp-up). Run the following two commands from the `src` folder of the repo, updating the placeholder values to your desired app name, resource group, location, and subscription. You can also change the SKU if desired.
 
 1. `az webapp up --runtime PYTHON:3.11 --sku B1 --name <new-app-name> --resource-group <resource-group-name> --location <azure-region> --subscription <subscription-name>`
-1. `az webapp config set --startup-file "python3 -m gunicorn app:app" --name <new-app-name>`
+1. `az webapp config set --startup-file "python3 -m gunicorn app:app" --name <new-app-name> --resource-group <resource-group-name>`
 
 If you've deployed the app previously, first run this command to update the appsettings to allow local code deployment:
 
@@ -88,10 +88,10 @@ Check the runtime stack for your app by viewing the app service resource in the
 
 Check the SKU in the same way. Use the abbreviated SKU name in the argument below, e.g. for "Basic (B1)" the SKU is `B1`. 
 
-Then, use these commands to deploy your local code to the existing app:
+Then, use these commands from `src` folder to deploy your local code to the existing app:
 
 1. `az webapp up --runtime <runtime-stack> --sku <sku> --name <existing-app-name> --resource-group <resource-group-name>`
-1. `az webapp config set --startup-file "python3 -m gunicorn app:app" --name <existing-app-name>`
+1. `az webapp config set --startup-file "python3 -m gunicorn app:app" --name <existing-app-name> --resource-group <resource-group-name>`
 
 Make sure that the app name and resource group match exactly for the app that was previously deployed.
 

infra/deploy_ai_foundry.bicep

@@ -4,7 +4,7 @@ param solutionLocation string
 param keyVaultName string
 param deploymentType string
 param gptModelName string
-param gptModelVersion string
+param azureOpenaiAPIVersion string
 param gptDeploymentCapacity int
 param embeddingModel string
 param embeddingDeploymentCapacity int
@@ -141,6 +141,7 @@ resource aiServices 'Microsoft.CognitiveServices/accounts@2021-10-01' = {
   }
   kind: 'AIServices'
   properties: {
+    publicNetworkAccess: 'Enabled'
     customSubDomainName: aiServicesName
     apiProperties: {
       //statisticsEnabled: false
@@ -363,7 +364,7 @@ resource azureOpenAIApiVersionEntry 'Microsoft.KeyVault/vaults/secrets@2021-11-0
   parent: keyVault
   name: 'AZURE-OPENAI-PREVIEW-API-VERSION'
   properties: {
-    value: gptModelVersion  //'2024-02-15-preview'
+    value: azureOpenaiAPIVersion  //'2024-02-15-preview'
   }
 }