Update deployment and fix azurecli issue

bkeller108 · bkeller108 · commit 563451a6f8c9 · 2025-01-16T11:38:02.000-06:00
diff --git a/AzureFunctions/km-charts-function/function_app.py b/AzureFunctions/km-charts-function/function_app.py
@@ -1,3 +1,4 @@
+from datetime import datetime
 import azure.functions as func
 import logging
 import json
@@ -25,6 +26,33 @@ def get_metrics(req: func.HttpRequest) -> func.HttpResponse:
 
     conn = pymssql.connect(server, username, password, database)
     cursor = conn.cursor()
+
+    # Adjust the dates to the current date
+    today = datetime.today()
+    cursor.execute("SELECT MAX(CAST(StartTime AS DATETIME)) FROM [dbo].[processed_data]")
+    max_start_time = cursor.fetchone()[0]
+    
+    if max_start_time:
+        days_difference = (today - max_start_time).days - 1
+        if days_difference != 0:
+            # Update processed_data table
+            cursor.execute(
+                "UPDATE [dbo].[processed_data] SET StartTime = FORMAT(DATEADD(DAY, %s, StartTime), 'yyyy-MM-dd HH:mm:ss'), EndTime = FORMAT(DATEADD(DAY, %s, EndTime), 'yyyy-MM-dd HH:mm:ss')",
+                (days_difference, days_difference)
+            )
+            # Update km_processed_data table
+            cursor.execute(
+                "UPDATE [dbo].[km_processed_data] SET StartTime = FORMAT(DATEADD(DAY, %s, StartTime), 'yyyy-MM-dd HH:mm:ss'), EndTime = FORMAT(DATEADD(DAY, %s, EndTime), 'yyyy-MM-dd HH:mm:ss')",
+                (days_difference, days_difference)
+            )
+            # Update processed_data_key_phrases table
+            cursor.execute(
+                "UPDATE [dbo].[processed_data_key_phrases] SET StartTime = FORMAT(DATEADD(DAY, %s, StartTime), 'yyyy-MM-dd HH:mm:ss')",
+                (days_difference,)
+            )
+            # Commit the changes
+            conn.commit()
+            
     if data_type == 'filters':
  
         sql_stmt = '''select 'Topic' as filter_name, mined_topic as displayValue, mined_topic as key1 from 
diff --git a/Deployment/bicep/deploy_upload_files_script.bicep b/Deployment/bicep/deploy_upload_files_script.bicep
@@ -3,8 +3,6 @@ param solutionLocation string
 param keyVaultName string
 param baseUrl string
 param managedIdentityObjectId string
-@secure()
-param storageAccountKey string
 param storageAccountName string
 param containerName string
 // param identity string
@@ -40,7 +38,7 @@ resource copy_demo_Data 'Microsoft.Resources/deploymentScripts@2020-10-01' = {
   properties: {
     azCliVersion: '2.50.0'
     primaryScriptUri: '${baseUrl}Deployment/scripts/copy_kb_files.sh' // deploy-azure-synapse-pipelines.sh
-    arguments: '${storageAccountName} ${containerName} ${storageAccountKey} ${baseUrl}' // Specify any arguments for the script
+    arguments: '${storageAccountName} ${containerName} ${baseUrl}' // Specify any arguments for the script
     timeout: 'PT1H' // Specify the desired timeout duration
     retentionInterval: 'PT1H' // Specify the desired retention interval
     cleanupPreference:'OnSuccess'
diff --git a/Deployment/bicep/main.bicep b/Deployment/bicep/main.bicep
@@ -13,6 +13,7 @@ param otherLocation string
 // param fabricWorkspaceId string
 
 var resourceGroupLocation = resourceGroup().location
+var resourceGroupName = resourceGroup().name
 
 var solutionLocation = resourceGroupLocation
 var baseUrl = 'https://raw.githubusercontent.com/microsoft/Customer-Service-Conversational-Insights-with-Azure-OpenAI-Services/km-v3-staging/'
@@ -84,7 +85,6 @@ module uploadFiles 'deploy_upload_files_script.bicep' = {
     baseUrl: baseUrl
     storageAccountName: storageAccount.outputs.storageName
     containerName: storageAccount.outputs.storageContainer
-    storageAccountKey:keyVault.getSecret('ADLS-ACCOUNT-KEY')
     managedIdentityObjectId:managedIdentityModule.outputs.managedIdentityOutput.id
   }
   dependsOn:[storageAccount,keyVault]
diff --git a/Deployment/bicep/main.json b/Deployment/bicep/main.json
@@ -5,7 +5,7 @@
     "_generator": {
       "name": "bicep",
       "version": "0.32.4.45862",
-      "templateHash": "7717305353028371958"
+      "templateHash": "2383153489316953097"
     }
   },
   "parameters": {
@@ -26,6 +26,7 @@
   },
   "variables": {
     "resourceGroupLocation": "[resourceGroup().location]",
+    "resourceGroupName": "[resourceGroup().name]",
     "solutionLocation": "[variables('resourceGroupLocation')]",
     "baseUrl": "https://raw.githubusercontent.com/microsoft/Customer-Service-Conversational-Insights-with-Azure-OpenAI-Services/km-v3-staging/"
   },
@@ -1434,14 +1435,6 @@
           "containerName": {
             "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_storage_account'), '2022-09-01').outputs.storageContainer.value]"
           },
-          "storageAccountKey": {
-            "reference": {
-              "keyVault": {
-                "id": "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.KeyVault/vaults', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_ai_foundry'), '2022-09-01').outputs.keyvaultName.value)]"
-              },
-              "secretName": "ADLS-ACCOUNT-KEY"
-            }
-          },
           "managedIdentityObjectId": {
             "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_managed_identity'), '2022-09-01').outputs.managedIdentityOutput.value.id]"
           }
@@ -1453,7 +1446,7 @@
             "_generator": {
               "name": "bicep",
               "version": "0.32.4.45862",
-              "templateHash": "2240564084997274937"
+              "templateHash": "16162981464101235300"
             }
           },
           "parameters": {
@@ -1472,9 +1465,6 @@
             "managedIdentityObjectId": {
               "type": "string"
             },
-            "storageAccountKey": {
-              "type": "securestring"
-            },
             "storageAccountName": {
               "type": "string"
             },
@@ -1498,7 +1488,7 @@
               "properties": {
                 "azCliVersion": "2.50.0",
                 "primaryScriptUri": "[format('{0}Deployment/scripts/copy_kb_files.sh', parameters('baseUrl'))]",
-                "arguments": "[format('{0} {1} {2} {3}', parameters('storageAccountName'), parameters('containerName'), parameters('storageAccountKey'), parameters('baseUrl'))]",
+                "arguments": "[format('{0} {1} {2}', parameters('storageAccountName'), parameters('containerName'), parameters('baseUrl'))]",
                 "timeout": "PT1H",
                 "retentionInterval": "PT1H",
                 "cleanupPreference": "OnSuccess"
diff --git a/Deployment/scripts/copy_kb_files.sh b/Deployment/scripts/copy_kb_files.sh
@@ -3,8 +3,7 @@
 # Variables
 storageAccount="$1"
 fileSystem="$2"
-accountKey="$3"
-baseUrl="$4"
+baseUrl="$3"
 # azureOpenAIApiKey="$4"
 # azureOpenAIEndpoint="$5"
 # azureSearchAdminKey="$6"
@@ -85,7 +84,12 @@ echo "Script Started"
 # az storage blob upload-batch --account-name "$storageAccount" --destination data/"$extractedFolder1" --source /mnt/azscripts/azscriptinput/"$extractedFolder1" --auth-mode login --pattern '*'
 # az storage blob upload-batch --account-name "$storageAccount" --destination data/"$extractedFolder2" --source /mnt/azscripts/azscriptinput/"$extractedFolder2" --auth-mode login --pattern '*'
 
-az storage fs directory upload -f "$fileSystem" --account-name "$storageAccount" -s "$extractedFolder1" --account-key "$accountKey" --recursive
+# Authenticate with Azure using managed identity
+az login --identity
+# Using az storage blob upload-batch to upload files with managed identity authentication, as the az storage fs directory upload command is not working with managed identity authentication.
+az storage blob upload-batch --account-name "$storageAccount" --destination data/"$extractedFolder1" --source /mnt/azscripts/azscriptinput/"$extractedFolder1" --auth-mode login --pattern '*'
+
+#az storage fs directory upload -f "$fileSystem" --account-name "$storageAccount" -s "$extractedFolder1" --account-key "$accountKey" --recursive
 
 # az storage blob upload-batch --account-name "$storageAccount" --destination "data/audiofiles" --source /mnt/azscripts/azscriptinput/"$extractedFolder2" --account-key "$accountKey" --pattern '*'
 # az storage blob upload-batch --account-name "$storageAccount" --destination "data/audiofiles" --source /mnt/azscripts/azscriptinput/"$extractedFolder3" --account-key "$accountKey" --pattern '*'
@@ -105,4 +109,4 @@ az storage fs directory upload -f "$fileSystem" --account-name "$storageAccount"
 # # pip install graphrag==0.3.6
 
 # # python -m graphrag.index --root /mnt/azscripts/azscriptinput/ragtest
-# # python -m graphrag index --root ./ragtest
+# # python -m graphrag index --root ./ragtest
