Skip to content

Commit bae1291

Browse files
Roopan-MicrosoftRoopan-Microsoft
authored
Merge pull request #143 from Harmanpreet-Microsoft/main
chore: Addition of Deployment and test pipeline
2 parents b215ac5 + 7098856 commit bae1291

File tree

4 files changed

+286
-9
lines changed

4 files changed

+286
-9
lines changed

‎.github/workflows/CI.yml‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
name: Deploy Resources
2+
3+
on:
4+
push:
5+
branches:
6+
- main # Adjust this to the branch you want to trigger the deployment on
7+
schedule:
8+
- cron: '0 6,18 * * *' # Runs at 6:00 AM and 6:00 PM GMT
9+
10+
11+
jobs:
12+
deploy:
13+
runs-on: windows-latest # Use a Windows runner for PowerShell scripts
14+
15+
steps:
16+
- name: Checkout Code
17+
uses: actions/checkout@v3 # Checks out your repository
18+
# Install Azure CLI
19+
- name: Install Azure CLI
20+
shell: pwsh
21+
run: |
22+
Invoke-WebRequest -Uri https://aka.ms/installazurecliwindows -OutFile AzureCLI.msi
23+
Start-Process msiexec.exe -ArgumentList '/I AzureCLI.msi /quiet' -Wait
24+
25+
# Install kubectl (Windows method)
26+
- name: Install kubectl
27+
shell: pwsh
28+
run: |
29+
Invoke-WebRequest -Uri https://dl.k8s.io/release/v1.28.0/bin/windows/amd64/kubectl.exe -OutFile kubectl.exe
30+
Move-Item -Path ./kubectl.exe -Destination "C:\kubectl.exe"
31+
[Environment]::SetEnvironmentVariable('PATH', $env:PATH + ';C:\', [System.EnvironmentVariableTarget]::Machine)
32+
33+
34+
# Install Helm (Windows method)
35+
- name: Install Helm
36+
shell: pwsh
37+
run: |
38+
Invoke-WebRequest -Uri https://get.helm.sh/helm-v3.13.0-windows-amd64.zip -OutFile helm.zip
39+
Expand-Archive helm.zip -DestinationPath helm
40+
Move-Item -Path ./helm/windows-amd64/helm.exe -Destination "C:\helm.exe"
41+
[Environment]::SetEnvironmentVariable('PATH', $env:PATH + ';C:\', [System.EnvironmentVariableTarget]::Machine)
42+
43+
44+
- name: Set Docker environment variables
45+
run: echo "DOCKER_BUILDKIT=0" >> $GITHUB_ENV
46+
47+
# Set up Docker
48+
- name: Set up Docker
49+
uses: docker/setup-buildx-action@v2
50+
with:
51+
driver: docker
52+
53+
- name: Setup PowerShell
54+
shell: pwsh
55+
run: |
56+
$PSVersionTable.PSVersion
57+
58+
- name: Run Deployment Script with Input
59+
shell: pwsh
60+
run: |
61+
cd Deployment
62+
$input = @"
63+
${{ secrets.AZURE_SUBSCRIPTION_ID }}
64+
CanadaCentral
65+
WestUS3
66+
${{ secrets.EMAIL }}
67+
yes
68+
"@
69+
$input | pwsh ./resourcedeployment.ps1
70+
echo "Resource Group Name is ${{ env.rg_name }}"
71+
echo "Kubernetes resource group are ${{ env.krg_name }}"
72+
env:
73+
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
74+
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
75+
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
76+
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
77+
- name: Cleanup Resource Group
78+
if: always() # Ensures this step runs even if the deployment fails
79+
shell: pwsh
80+
run: |
81+
az login --service-principal --username ${{ secrets.AZURE_CLIENT_ID }} --password ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }}
82+
az group delete --name ${{ env.rg_name }} --yes --no-wait
83+
az group delete --name ${{ env.krg_name }} --yes --no-wait
84+
env:
85+
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
86+
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
87+
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
88+
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
89+
90+
- name: Wait for Resource Deletion to Complete
91+
shell: pwsh
92+
run: |
93+
$retries = 0
94+
$maxRetries = 3
95+
$sleepIntervals = @(700, 200, 200)
96+
97+
while ($retries -lt $maxRetries) {
98+
$rgStatus = az group exists --name ${{ env.rg_name }}
99+
$krgStatus = az group exists --name ${{ env.krg_name }}
100+
101+
102+
# if (-not $rgStatus -and -not $krgStatus) {
103+
# Write-Host "Both resource groups deleted successfully."
104+
# break
105+
# }
106+
if ($rgStatus -eq "false" -and $krgStatus -eq "false") {
107+
Write-Host "Both resource groups deleted successfully."
108+
break
109+
}
110+
111+
$retries++
112+
if ($retries -eq $maxRetries) {
113+
Write-Host "Resource groups deletion not confirmed after $maxRetries attempts. Exiting."
114+
exit 1
115+
}
116+
117+
Write-Host "Resource groups still exist. Retrying in $($sleepIntervals[$retries - 1]) seconds..."
118+
Start-Sleep -Seconds $sleepIntervals[$retries - 1]
119+
}
120+
121+
- name: Purging the Resources
122+
if: success()
123+
shell: pwsh
124+
run: |
125+
# Set variables using GitHub Actions environment values
126+
$solutionPrefix = "${{ env.SOLUTION_PREFIX }}"
127+
$subscriptionId = "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
128+
$resourceGroupName = "${{ env.rg_name }}"
129+
130+
$openai_name = "openaiservice-$solutionPrefix"
131+
$cognitiveservice_name = "cognitiveservice-$solutionPrefix"
132+
133+
# Debug: Print resource names
134+
Write-Host "Purging OpenAI resource: $openai_name"
135+
Write-Host "Purging CognitiveService Account: $cognitiveservice_name"
136+
137+
# Construct resource IDs
138+
$openaiResourceId = "/subscriptions/$subscriptionId/providers/Microsoft.CognitiveServices/locations/westus3/resourceGroups/$resourceGroupName/deletedAccounts/$openai_name"
139+
$cognitiveResourceId = "/subscriptions/$subscriptionId/providers/Microsoft.CognitiveServices/locations/eastus/resourceGroups/$resourceGroupName/deletedAccounts/$cognitiveservice_name"
140+
141+
# Debug: Print constructed resource IDs
142+
Write-Host "Command to purge OpenAI resource: az resource delete --ids `"$openaiResourceId`" --verbose"
143+
Write-Host "Command to purge CognitiveService Account: az resource delete --ids `"$cognitiveResourceId`" --verbose"
144+
# Purge OpenAI Resource
145+
az resource delete --ids $openaiResourceId --verbose
146+
if (-not $?) {
147+
Write-Host "Failed to purge OpenAI resource: $openaiResourceId"
148+
}
149+
150+
# Purge CognitiveService Account
151+
152+
153+
az resource delete --ids $cognitiveResourceId --verbose
154+
if (-not $?) {
155+
Write-Host "Failed to purge CognitiveService Account."
156+
}
157+
158+
159+
- name: Send Notification on Failure
160+
if: failure()
161+
shell: pwsh
162+
run: |
163+
# Define the RUN_URL variable
164+
$RUN_URL = "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
165+
166+
# Construct the email body using a Here-String
167+
$EMAIL_BODY = @"
168+
{
169+
"body": "<p>Dear Team,</p><p>The Document Knowledge Mining Automation process encountered an issue.</p><p><strong>Build URL:</strong> <a href='$RUN_URL'>$RUN_URL</a></p><p>Please investigate promptly.</p><p>Best regards,<br>Your Automation Team</p>"
170+
}
171+
"@
172+
173+
# Send the notification with error handling
174+
try {
175+
curl -X POST "${{ secrets.LOGIC_APP_URL }}" `
176+
-H "Content-Type: application/json" `
177+
-d "$EMAIL_BODY"
178+
} catch {
179+
Write-Output "Failed to send notification."
180+
}
181+

‎App/kernel-memory/service/Core/Prompts/extract-keywords.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,4 @@ CATEGORY COUNT CAN BE UP TO 10
1818
[END EXAMPLE]
1919

2020
Extract Keywords from this :
21-
{{$input}}
21+
{{$input}}

‎Deployment/main.bicep‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@ module gs_appconfig 'bicep/azureappconfigservice.bicep' = {
158158

159159
// return all resource names as a output
160160
output gs_resourcegroup_name string = 'rg-${resourceprefix_name}${resourceprefix}'
161+
output gs_solution_prefix string = '${resourceprefix_name}${resourceprefix}'
161162
output gs_storageaccount_name string = gs_storageaccount.outputs.storageAccountName
162163
output gs_azsearch_name string = gs_azsearch.outputs.searchServiceName
163164

‎Deployment/resourcedeployment.ps1‎

Lines changed: 103 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,35 @@ function successBanner(){
4141
Write-Host " |_| |___/ "
4242
}
4343

44+
function failureBanner(){
45+
Write-Host " _____ _ _ "
46+
Write-Host "| __ \ | | | | "
47+
Write-Host "| | | | ___ _ __ | | ___ _ _ _ __ ___ ___ _ __ | |_ "
48+
Write-Host "| | | |/ _ \ '_ \| |/ _ \| | | | '_ ` _ \ / _ \ '_ \| __| "
49+
Write-Host "| |__| | __/ |_) | | (_) | |_| | | | | | | __/ | | | |_ "
50+
Write-Host "|_____/ \___| .__/|_|\___/ \__, |_| |_| |_|\___|_| |_|\__| "
51+
Write-Host " | | __/ | "
52+
Write-Host " ______ _|_| _ |___/ "
53+
Write-Host "| ____| (_) | | | "
54+
Write-Host "| |__ __ _ _| | ___ __| | "
55+
Write-Host "| __/ _` | | |/ _ \/ _` | "
56+
Write-Host "| | | (_| | | | __/ (_| | "
57+
Write-Host "|_| \__,_|_|_|\___|\__,_| "
58+
}
59+
60+
# Common function to check if a variable is null or empty
61+
function ValidateVariableIsNullOrEmpty {
62+
param (
63+
[string]$variableValue,
64+
[string]$variableName
65+
)
66+
67+
if ([string]::IsNullOrEmpty($variableValue)) {
68+
Write-Host "Error: $variableName is null or empty." -ForegroundColor Red
69+
failureBanner
70+
exit 1
71+
}
72+
}
4473
# Function to prompt for parameters with kind messages
4574
function PromptForParameters {
4675
param(
@@ -112,8 +141,19 @@ $modelLocation = $params.modelLocation
112141
$email = $params.email
113142

114143
function LoginAzure([string]$subscriptionID) {
115-
Write-Host "Log in to Azure.....`r`n" -ForegroundColor Yellow
116-
az login
144+
Write-Host "Log in to Azure.....`r`n" -ForegroundColor Yellow
145+
if ($env:CI -eq "true"){
146+
147+
az login --service-principal `
148+
--username $env:AZURE_CLIENT_ID `
149+
--password $env:AZURE_CLIENT_SECRET `
150+
--tenant $env:AZURE_TENANT_ID
151+
write-host "CI deployment mode"
152+
}
153+
else{
154+
az login
155+
write-host "manual deployment mode"
156+
}
117157
az account set --subscription $subscriptionID
118158
Write-Host "Switched subscription to '$subscriptionID' `r`n" -ForegroundColor Yellow
119159
}
@@ -163,9 +203,12 @@ function DeployAzureResources([string]$location, [string]$modelLocation) {
163203

164204
function DisplayResult([pscustomobject]$jsonString) {
165205
$resourcegroupName = $jsonString.properties.outputs.gs_resourcegroup_name.value
206+
$solutionPrefix = $jsonString.properties.outputs.gs_solution_prefix.value
207+
166208
$storageAccountName = $jsonString.properties.outputs.gs_storageaccount_name.value
167209
$azsearchServiceName = $jsonString.properties.outputs.gs_azsearch_name.value
168210
$aksName = $jsonString.properties.outputs.gs_aks_name.value
211+
169212
$containerRegistryName = $jsonString.properties.outputs.gs_containerregistry_name.value
170213
$azcognitiveserviceName = $jsonString.properties.outputs.gs_azcognitiveservice_name.value
171214
$azopenaiServiceName = $jsonString.properties.outputs.gs_openaiservice_name.value
@@ -186,6 +229,9 @@ function DisplayResult([pscustomobject]$jsonString) {
186229
Write-Host "* Azure Storage Account " -ForegroundColor Yellow -NoNewline; Write-Host "$storageAccountName" -ForegroundColor Green
187230
Write-Host "* Azure Cosmos DB " -ForegroundColor Yellow -NoNewline; Write-Host "$azcosmosDBName" -ForegroundColor Green
188231
Write-Host "* Azure App Configuration Endpoint " -ForegroundColor Yellow -NoNewline; Write-Host "$azappConfigEndpoint" -ForegroundColor Green
232+
Write-Output "rg_name=$resourcegroupName" >> $Env:GITHUB_ENV
233+
234+
Write-Output "SOLUTION_PREFIX=$solutionPrefix" >> $Env:GITHUB_ENV
189235
}
190236

191237
# Function to replace placeholders in a template with actual values
@@ -395,10 +441,33 @@ try {
395441
###############################################################
396442
# Get the storage account key
397443
$storageAccountKey = az storage account keys list --account-name $deploymentResult.StorageAccountName --resource-group $deploymentResult.ResourceGroupName --query "[0].value" -o tsv
444+
445+
# Validate if the storage account key is empty or null
446+
ValidateVariableIsNullOrEmpty -variableValue $storageAccountKey -variableName "Storage account key"
447+
398448
## Construct the connection string manually
399449
$storageAccountConnectionString = "DefaultEndpointsProtocol=https;AccountName=$($deploymentResult.StorageAccountName);AccountKey=$storageAccountKey;EndpointSuffix=core.windows.net"
450+
# Validate if the Storage Account Connection String is empty or null
451+
ValidateVariableIsNullOrEmpty -variableValue $storageAccountConnectionString -variableName "Storage Account Connection String"
452+
400453
## Assign the connection string to the deployment result object
401-
$deploymentResult.StorageAccountConnectionString = $storageAccountConnectionString
454+
$deploymentResult.StorageAccountConnectionString = $storageAccountConnectionString
455+
456+
# Check if ResourceGroupName is valid
457+
ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.ResourceGroupName -variableName "Resource group name"
458+
459+
# Check if AzCosmosDBName is valid
460+
ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.AzCosmosDBName -variableName "Az Cosmos DB name"
461+
462+
# Check if AzCognitiveServiceName is valid
463+
ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.AzCognitiveServiceName -variableName "Az Cognitive Service name"
464+
465+
# Check if AzSearchServiceName is valid
466+
ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.AzSearchServiceName -variableName "Az Search Service name"
467+
468+
# Check if AzOpenAiServiceName is valid
469+
ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.AzOpenAiServiceName -variableName "Az OpenAI Service name"
470+
402471
# Get MongoDB connection string
403472
$deploymentResult.AzCosmosDBConnectionString = az cosmosdb keys list --name $deploymentResult.AzCosmosDBName --resource-group $deploymentResult.ResourceGroupName --type connection-strings --query "connectionStrings[0].connectionString" -o tsv
404473
# Get Azure Cognitive Service API Key
@@ -538,6 +607,8 @@ try {
538607
Write-Host "Getting the Kubernetes resource group..." -ForegroundColor Cyan
539608
$aksResourceGroupName = $(az aks show --resource-group $deploymentResult.ResourceGroupName --name $deploymentResult.AksName --query nodeResourceGroup --output tsv)
540609
Write-Host "Kubernetes resource group: $aksResourceGroupName" -ForegroundColor Green
610+
Write-Output "krg_name=$aksResourceGroupName" >> $Env:GITHUB_ENV
611+
541612
}
542613
catch {
543614
Write-Host "Failed to get the Kubernetes resource group." -ForegroundColor Red
@@ -601,26 +672,49 @@ try {
601672
# 6-1. Get Az Network resource Name with the public IP address
602673
Write-Host "Assign DNS Name to the public IP address" -ForegroundColor Green
603674
$publicIpName=$(az network public-ip list --query "[?ipAddress=='$externalIP'].name" --output tsv)
604-
605675
# 6-2. Generate Unique backend API fqdn Name - esgdocanalysis-3 digit random number with padding 0
606676
$dnsName = "kmgs$($(Get-Random -Minimum 0 -Maximum 9999).ToString("D4"))"
677+
678+
# Validate if the AKS Resource Group Name, Public IP name and DNS Name are provided
679+
ValidateVariableIsNullOrEmpty -variableValue $aksResourceGroupName -variableName "AKS Resource Group name"
680+
681+
ValidateVariableIsNullOrEmpty -variableValue $publicIpName -variableName "Public IP name"
607682

683+
ValidateVariableIsNullOrEmpty -variableValue $dnsName -variableName "DNS Name"
684+
608685
# 6-3. Assign DNS Name to the public IP address
609686
az network public-ip update --resource-group $aksResourceGroupName --name $publicIpName --dns-name $dnsName
610-
# 6-4. Get FQDN for the public IP address
611-
$fqdn = az network public-ip show --resource-group $aksResourceGroupName --name $publicIpName --query "dnsSettings.fqdn" --output tsv
612-
Write-Host "FQDN for the public IP address is: $fqdn" -ForegroundColor Green
613687

688+
# 6-4. Get FQDN for the public IP address
689+
$fqdn = az network public-ip show --resource-group $aksResourceGroupName --name $publicIpName --query "dnsSettings.fqdn" --output tsv
690+
691+
# Validate if the FQDN is null or empty
692+
ValidateVariableIsNullOrEmpty -variableValue $fqdn -variableName "FQDN"
693+
614694
# 7. Assign the role for aks system assigned managed identity to App Configuration Data Reader role with the scope of Resourcegroup
615695
Write-Host "Assign the role for aks system assigned managed identity to App Configuration Data Reader role" -ForegroundColor Green
696+
# Ensure that the required fields are not null or empty
697+
ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.ResourceGroupName -variableName "Resource group name"
698+
699+
ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.AksName -variableName "AKS cluster name"
700+
616701
# Get vmss resource group name
617702
$vmssResourceGroupName = $(az aks show --resource-group $deploymentResult.ResourceGroupName --name $deploymentResult.AksName --query nodeResourceGroup --output tsv)
703+
704+
# Validate if vmss Resource Group Name is null or empty
705+
ValidateVariableIsNullOrEmpty -variableValue $vmssResourceGroupName -variableName "VMSS resource group"
706+
618707
# Get vmss name
619708
$vmssName = $(az vmss list --resource-group $vmssResourceGroupName --query "[0].name" --output tsv)
709+
710+
# Validate if vmss Name is null or empty
711+
ValidateVariableIsNullOrEmpty -variableValue $vmssName -variableName "VMSS name"
712+
620713
# Create System Assigned Managed Identity
621714
$systemAssignedIdentity = $(az vmss identity assign --resource-group $vmssResourceGroupName --name $vmssName --query systemAssignedIdentity --output tsv)
622715

623-
716+
# Validate if System Assigned Identity is null or empty
717+
ValidateVariableIsNullOrEmpty -variableValue $systemAssignedIdentity -variableName "System-assigned managed identity"
624718

625719
# Assign the role for aks system assigned managed identity to App Configuration Data Reader role with the scope of Resourcegroup
626720
az role assignment create --assignee $systemAssignedIdentity --role "App Configuration Data Reader" --scope $deploymentResult.ResourceGroupId
@@ -641,6 +735,7 @@ try {
641735
Write-Host "Upgrading node pool: $nodePool" -ForegroundColor Cyan
642736
Write-Host "Node pool $nodePool upgrade initiated." -ForegroundColor Green
643737
az aks nodepool upgrade --resource-group $deploymentResult.ResourceGroupName --cluster-name $deploymentResult.AksName --name $nodePool
738+
644739
}
645740
}
646741
catch {

0 commit comments

Comments
 (0)