Skip to content

Commit 2fb2567

Browse files
marktayl1marktayl1
committed
2 parents f2bd8ad + 8c9a95f commit 2fb2567

22 files changed

+1299
-103
lines changed

‎.github/workflows/deploy-v2.yml‎

Lines changed: 346 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,346 @@
1+
name: Validate Deployment v2
2+
3+
on:
4+
workflow_run:
5+
workflows: ["Build Docker and Optional Push"]
6+
types:
7+
- completed
8+
branches:
9+
- macae-v2
10+
- dev
11+
schedule:
12+
- cron: "0 11,23 * * *" # Runs at 11:00 AM and 11:00 PM GMT
13+
workflow_dispatch: #Allow manual triggering
14+
env:
15+
GPT_MIN_CAPACITY: 150
16+
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
17+
18+
jobs:
19+
deploy:
20+
runs-on: ubuntu-latest
21+
outputs:
22+
RESOURCE_GROUP_NAME: ${{ steps.check_create_rg.outputs.RESOURCE_GROUP_NAME }}
23+
WEBAPP_URL: ${{ steps.get_output.outputs.WEBAPP_URL }}
24+
DEPLOYMENT_SUCCESS: ${{ steps.deployment_status.outputs.SUCCESS }}
25+
MACAE_URL_API: ${{ steps.get_backend_url.outputs.MACAE_URL_API }}
26+
CONTAINER_APP: ${{steps.get_backend_url.outputs.CONTAINER_APP}}
27+
steps:
28+
- name: Checkout Code
29+
uses: actions/checkout@v3
30+
31+
- name: Run Quota Check
32+
id: quota-check
33+
run: |
34+
export AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}
35+
export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}
36+
export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}
37+
export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
38+
export GPT_MIN_CAPACITY="150"
39+
export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}"
40+
41+
chmod +x infra/scripts/checkquota.sh
42+
if ! infra/scripts/checkquota.sh; then
43+
# If quota check fails due to insufficient quota, set the flag
44+
if grep -q "No region with sufficient quota found" infra/scripts/checkquota.sh; then
45+
echo "QUOTA_FAILED=true" >> $GITHUB_ENV
46+
fi
47+
exit 1 # Fail the pipeline if any other failure occurs
48+
fi
49+
50+
- name: Send Notification on Quota Failure
51+
if: env.QUOTA_FAILED == 'true'
52+
run: |
53+
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
54+
EMAIL_BODY=$(cat <<EOF
55+
{
56+
"body": "<p>Dear Team,</p><p>The quota check has failed, and the pipeline cannot proceed.</p><p><strong>Build URL:</strong> ${RUN_URL}</p><p>Please take necessary action.</p><p>Best regards,<br>Your Automation Team</p>"
57+
}
58+
EOF
59+
)
60+
61+
curl -X POST "${{ secrets.AUTO_LOGIC_APP_URL }}" \
62+
-H "Content-Type: application/json" \
63+
-d "$EMAIL_BODY" || echo "Failed to send notification"
64+
65+
- name: Fail Pipeline if Quota Check Fails
66+
if: env.QUOTA_FAILED == 'true'
67+
run: exit 1
68+
69+
- name: Set Deployment Region
70+
run: |
71+
echo "Selected Region: $VALID_REGION"
72+
echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV
73+
74+
- name: Setup Azure CLI
75+
run: |
76+
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
77+
az --version # Verify installation
78+
79+
- name: Login to Azure
80+
run: |
81+
az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }}
82+
83+
- name: Install Bicep CLI
84+
run: az bicep install
85+
86+
- name: Generate Resource Group Name
87+
id: generate_rg_name
88+
run: |
89+
ACCL_NAME="macae"
90+
SHORT_UUID=$(uuidgen | cut -d'-' -f1)
91+
UNIQUE_RG_NAME="arg-${ACCL_NAME}-${SHORT_UUID}"
92+
echo "RESOURCE_GROUP_NAME=${UNIQUE_RG_NAME}" >> $GITHUB_ENV
93+
echo "Generated Resource_GROUP_PREFIX: ${UNIQUE_RG_NAME}"
94+
95+
- name: Check and Create Resource Group
96+
id: check_create_rg
97+
run: |
98+
set -e
99+
rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
100+
if [ "$rg_exists" = "false" ]; then
101+
az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.AZURE_LOCATION }}
102+
fi
103+
echo "RESOURCE_GROUP_NAME=${{ env.RESOURCE_GROUP_NAME }}" >> $GITHUB_OUTPUT
104+
105+
- name: Generate Unique Solution Prefix
106+
id: generate_solution_prefix
107+
run: |
108+
COMMON_PART="macae"
109+
TIMESTAMP=$(date +%s)
110+
UPDATED_TIMESTAMP=$(echo $TIMESTAMP | tail -c 6)
111+
UNIQUE_SOLUTION_PREFIX="${COMMON_PART}${UPDATED_TIMESTAMP}"
112+
echo "SOLUTION_PREFIX=${UNIQUE_SOLUTION_PREFIX}" >> $GITHUB_ENV
113+
114+
- name: Deploy Bicep Template
115+
id: deploy
116+
run: |
117+
if [[ "${{ env.BRANCH_NAME }}" == "macae-v2" ]]; then
118+
IMAGE_TAG="latest"
119+
elif [[ "${{ env.BRANCH_NAME }}" == "dev" ]]; then
120+
IMAGE_TAG="dev"
121+
# elif [[ "${{ env.BRANCH_NAME }}" == "hotfix" ]]; then
122+
# IMAGE_TAG="hotfix"
123+
# else
124+
# IMAGE_TAG="latest"
125+
fi
126+
127+
az deployment group create \
128+
--resource-group ${{ env.RESOURCE_GROUP_NAME }} \
129+
--template-file infra/main.bicep \
130+
--parameters \
131+
solutionName=${{ env.SOLUTION_PREFIX }} \
132+
location="${{ env.AZURE_LOCATION }}" \
133+
gptModelDeploymentType="GlobalStandard" \
134+
gptModelName="gpt-4o" \
135+
gptModelVersion="2024-08-06" \
136+
backendContainerImageTag="${IMAGE_TAG}" \
137+
frontendContainerImageTag="${IMAGE_TAG}" \
138+
azureAiServiceLocation='${{ env.AZURE_LOCATION }}' \
139+
gptModelCapacity=150 \
140+
createdBy="Pipeline" \
141+
--output json
142+
143+
- name: Extract Web App and API App URLs
144+
id: get_output
145+
run: |
146+
WEBAPP_NAMES=$(az webapp list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --query "[].name" -o tsv)
147+
for NAME in $WEBAPP_NAMES; do
148+
if [[ $NAME == app-* ]]; then
149+
WEBAPP_URL="https://${NAME}.azurewebsites.net"
150+
echo "WEBAPP_URL=$WEBAPP_URL" >> $GITHUB_OUTPUT
151+
fi
152+
done
153+
154+
- name: Get Container App Backend URL
155+
id: get_backend_url
156+
run: |
157+
CONTAINER_APP_NAME=$(az containerapp list \
158+
--resource-group ${{ env.RESOURCE_GROUP_NAME }} \
159+
--query "[0].name" -o tsv)
160+
161+
MACAE_URL_API=$(az containerapp show \
162+
--name "$CONTAINER_APP_NAME" \
163+
--resource-group ${{ env.RESOURCE_GROUP_NAME }} \
164+
--query "properties.configuration.ingress.fqdn" -o tsv)
165+
166+
echo "MACAE_URL_API=https://${MACAE_URL_API}" >> $GITHUB_OUTPUT
167+
echo "CONTAINER_APP=${CONTAINER_APP_NAME}" >> $GITHUB_OUTPUT
168+
169+
- name: Set Deployment Status
170+
id: deployment_status
171+
if: always()
172+
run: |
173+
if [ "${{ job.status }}" == "success" ]; then
174+
echo "SUCCESS=true" >> $GITHUB_OUTPUT
175+
else
176+
echo "SUCCESS=false" >> $GITHUB_OUTPUT
177+
fi
178+
179+
e2e-test:
180+
needs: deploy
181+
if: needs.deploy.outputs.DEPLOYMENT_SUCCESS == 'true'
182+
uses: ./.github/workflows/test-automation.yml
183+
with:
184+
MACAE_WEB_URL: ${{ needs.deploy.outputs.WEBAPP_URL }}
185+
MACAE_URL_API: ${{ needs.deploy.outputs.MACAE_URL_API }}
186+
MACAE_RG: ${{ needs.deploy.outputs.RESOURCE_GROUP_NAME }}
187+
MACAE_CONTAINER_APP: ${{ needs.deploy.outputs.CONTAINER_APP }}
188+
secrets: inherit
189+
190+
cleanup-deployment:
191+
if: always() && needs.deploy.outputs.RESOURCE_GROUP_NAME != ''
192+
needs: [deploy, e2e-test]
193+
runs-on: ubuntu-latest
194+
env:
195+
RESOURCE_GROUP_NAME: ${{ needs.deploy.outputs.RESOURCE_GROUP_NAME }}
196+
steps:
197+
- name: Setup Azure CLI
198+
run: |
199+
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
200+
az --version
201+
- name: Login to Azure
202+
run: |
203+
az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }}
204+
az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
205+
206+
- name: Extract AI Services and Key Vault Names
207+
if: always()
208+
run: |
209+
echo "Fetching AI Services and Key Vault names before deletion..."
210+
211+
# Get Key Vault name
212+
KEYVAULT_NAME=$(az resource list --resource-group "${{ env.RESOURCE_GROUP_NAME }}" --resource-type "Microsoft.KeyVault/vaults" --query "[].name" -o tsv)
213+
echo "Detected Key Vault: $KEYVAULT_NAME"
214+
echo "KEYVAULT_NAME=$KEYVAULT_NAME" >> $GITHUB_ENV
215+
# Extract AI Services names
216+
echo "Fetching AI Services..."
217+
AI_SERVICES=$(az resource list --resource-group '${{ env.RESOURCE_GROUP_NAME }}' --resource-type "Microsoft.CognitiveServices/accounts" --query "[].name" -o tsv)
218+
# Flatten newline-separated values to space-separated
219+
AI_SERVICES=$(echo "$AI_SERVICES" | paste -sd ' ' -)
220+
echo "Detected AI Services: $AI_SERVICES"
221+
echo "AI_SERVICES=$AI_SERVICES" >> $GITHUB_ENV
222+
223+
- name: Get OpenAI Resource from Resource Group
224+
id: get_openai_resource
225+
run: |
226+
227+
set -e
228+
echo "Fetching OpenAI resource from resource group ${{ env.RESOURCE_GROUP_NAME }}..."
229+
230+
# Run the az resource list command to get the OpenAI resource name
231+
openai_resource_name=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --resource-type "Microsoft.CognitiveServices/accounts" --query "[0].name" -o tsv)
232+
233+
if [ -z "$openai_resource_name" ]; then
234+
echo "No OpenAI resource found in resource group ${{ env.RESOURCE_GROUP_NAME }}."
235+
exit 0
236+
else
237+
echo "OPENAI_RESOURCE_NAME=${openai_resource_name}" >> $GITHUB_ENV
238+
echo "OpenAI resource name: ${openai_resource_name}"
239+
fi
240+
241+
- name: Delete Bicep Deployment
242+
if: always()
243+
run: |
244+
set -e
245+
echo "Checking if resource group exists..."
246+
rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
247+
if [ "$rg_exists" = "true" ]; then
248+
echo "Resource group exist. Cleaning..."
249+
az group delete \
250+
--name ${{ env.RESOURCE_GROUP_NAME }} \
251+
--yes \
252+
--no-wait
253+
echo "Resource group deleted... ${{ env.RESOURCE_GROUP_NAME }}"
254+
else
255+
echo "Resource group does not exists."
256+
fi
257+
258+
- name: Wait for resource deletion to complete
259+
run: |
260+
261+
# Add resources to the array
262+
resources_to_check=("${{ env.OPENAI_RESOURCE_NAME }}")
263+
264+
echo "List of resources to check: ${resources_to_check[@]}"
265+
266+
# Maximum number of retries
267+
max_retries=3
268+
269+
# Retry intervals in seconds (30, 60, 120)
270+
retry_intervals=(30 60 120)
271+
272+
# Retry mechanism to check resources
273+
retries=0
274+
while true; do
275+
resource_found=false
276+
277+
# Get the list of resources in YAML format again on each retry
278+
resource_list=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --output yaml)
279+
280+
# Iterate through the resources to check
281+
for resource in "${resources_to_check[@]}"; do
282+
echo "Checking resource: $resource"
283+
if echo "$resource_list" | grep -q "name: $resource"; then
284+
echo "Resource '$resource' exists in the resource group."
285+
resource_found=true
286+
else
287+
echo "Resource '$resource' does not exist in the resource group."
288+
fi
289+
done
290+
291+
# If any resource exists, retry
292+
if [ "$resource_found" = true ]; then
293+
retries=$((retries + 1))
294+
if [ "$retries" -gt "$max_retries" ]; then
295+
echo "Maximum retry attempts reached. Exiting."
296+
break
297+
else
298+
# Wait for the appropriate interval for the current retry
299+
echo "Waiting for ${retry_intervals[$retries-1]} seconds before retrying..."
300+
sleep ${retry_intervals[$retries-1]}
301+
fi
302+
else
303+
echo "No resources found. Exiting."
304+
break
305+
fi
306+
done
307+
308+
- name: Purging the Resources
309+
if: always()
310+
run: |
311+
312+
set -e
313+
echo "Azure OpenAI: ${{ env.OPENAI_RESOURCE_NAME }}"
314+
315+
# Purge OpenAI Resource
316+
echo "Purging the OpenAI Resource..."
317+
if ! az resource delete --ids /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/providers/Microsoft.CognitiveServices/locations/eastus/resourceGroups/${{ env.RESOURCE_GROUP_NAME }}/deletedAccounts/${{ env.OPENAI_RESOURCE_NAME }} --verbose; then
318+
echo "Failed to purge openai resource: ${{ env.OPENAI_RESOURCE_NAME }}"
319+
else
320+
echo "Purged the openai resource: ${{ env.OPENAI_RESOURCE_NAME }}"
321+
fi
322+
323+
echo "Resource purging completed successfully"
324+
325+
- name: Send Notification on Failure
326+
if: failure()
327+
run: |
328+
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
329+
330+
# Construct the email body
331+
EMAIL_BODY=$(cat <<EOF
332+
{
333+
"body": "<p>Dear Team,</p><p>We would like to inform you that the Multi-Agent-Custom-Automation-Engine-Solution-Accelerator Automation process has encountered an issue and has failed to complete successfully.</p><p><strong>Build URL:</strong> ${RUN_URL}<br> ${OUTPUT}</p><p>Please investigate the matter at your earliest convenience.</p><p>Best regards,<br>Your Automation Team</p>"
334+
}
335+
EOF
336+
)
337+
338+
# Send the notification
339+
curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
340+
-H "Content-Type: application/json" \
341+
-d "$EMAIL_BODY" || echo "Failed to send notification"
342+
- name: Logout from Azure
343+
if: always()
344+
run: |
345+
az logout
346+
echo "Logged out from Azure."

0 commit comments

Comments
 (0)