added retry mechanism for client advisor

Author: Prashant-Microsoft

.github/workflows/CAdeploy.yml

@@ -76,6 +76,7 @@ jobs:
       - name: List KeyVaults and Store in Array
         id: list_keyvaults
         run: |
+
           set -e
           echo "Listing all KeyVaults in the resource group ${RESOURCE_GROUP_NAME}..."
           
@@ -108,6 +109,7 @@ jobs:
       - name: Update PowerBI URL
         if: success()
         run: |
+
           set -e
 
           COMMON_PART="-app-service"
@@ -144,12 +146,65 @@ jobs:
 
       - name: Wait for resource deletion to complete
         run: |
-          echo "Waiting for resource group deletion to complete..."
-          sleep 120  # Wait for 2 minutes (120 seconds) to allow deletion to complete
+
+          # List of keyvaults 
+          KEYVAULTS="${{ env.KEYVAULTS }}"
+
+          # Remove the surrounding square brackets, if they exist
+          stripped_keyvaults=$(echo "$KEYVAULTS" | sed 's/\[\|\]//g')
+          
+          # Convert the comma-separated string into an array
+          IFS=',' read -r -a resources_to_check <<< "$stripped_keyvaults"
+          
+          # Append new resources to the array
+          resources_to_check+=("${{ env.SOLUTION_PREFIX }}-openai" "${{ env.SOLUTION_PREFIX }}-cogser")
+
+          echo "List of resources to check: ${resources_to_check[@]}"
+         
+          # Get the list of resources in YAML format
+          resource_list=$(az resource list --resource-group myResourceGroup4 --output yaml)
+
+          # Maximum number of retries
+          max_retries=3
+
+          # Retry intervals in seconds (30, 60, 120)
+          retry_intervals=(30 60 120)
+
+          while true; do
+            resource_found=false
+
+            # Iterate through the resources to check
+            for resource in "${resources_to_check[@]}"; do
+              echo "Checking resource: $resource"
+              if echo "$resource_list" | grep -q "name: $resource"; then
+                echo "Resource '$resource' exists in the resource group."
+                resource_found=true
+              else
+                echo "Resource '$resource' does not exist in the resource group."
+              fi
+            done
+
+            # If any resource exists, retry
+            if [ "$resource_found" = true ]; then
+              retries=$((retries + 1))
+              if [ "$retries" -ge "$max_retries" ]; then
+                echo "Maximum retry attempts reached. Exiting."
+                break
+              else
+                # Wait for the appropriate interval for the current retry
+                echo "Waiting for ${retry_intervals[$retries-1]} seconds before retrying..."
+                sleep ${retry_intervals[$retries-1]}
+              fi
+            else
+              echo "No resources found. Exiting."
+              break
+            fi
+          done
 
       - name: Purging the Resources
         if: success()
         run: |
+
           set -e 
           # Define variables
           OPENAI_COMMON_PART="-openai"
@@ -205,6 +260,7 @@ jobs:
       - name: Send Notification on Failure
         if: failure()
         run: |
+
           RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
       
           # Construct the email body