Another try of fixing the workflow

Author: nullchimp

.github/workflows/deploy-azure.yml

@@ -101,46 +101,105 @@ jobs:
         fi
 
     ################################################################
-    # 2.5 · Reconcile AKS cluster if it exists
+    # 2.5 · Check for ongoing operations and abort/wait
     ################################################################
-    - name: Reconcile AKS cluster
+    - name: Check AKS operations and abort if needed
       run: |
         set -euo pipefail
         RG="rg-aks-memgraph-${{ env.ENVIRONMENT }}"
         AKS="aks-memgraph-${{ env.ENVIRONMENT }}"
         
         if az aks show --resource-group "$RG" --name "$AKS" &>/dev/null; then
-          echo "🔄 Reconciling AKS cluster to fix node resource group..."
-          az aks update --resource-group "$RG" --name "$AKS" --no-wait
+          # Check for ongoing operations
+          echo "🔍 Checking for ongoing AKS operations..."
+          ONGOING_OPS=$(az aks operation list --resource-group "$RG" --name "$AKS" --query "[?status=='Running' || status=='Pending']" -o json)
           
-          echo "⏳ Waiting for reconciliation to complete..."
+          if [[ $(echo "$ONGOING_OPS" | jq 'length') -gt 0 ]]; then
+            echo "⚠️ Found ongoing operations. Attempting to abort them..."
+            OPERATION_IDS=$(echo "$ONGOING_OPS" | jq -r '.[].operationId')
+            
+            for OP_ID in $OPERATION_IDS; do
+              echo "🛑 Aborting operation $OP_ID..."
+              az aks operation-abort --resource-group "$RG" --name "$AKS" --operation-id "$OP_ID" || echo "⚠️ Could not abort operation, may require manual intervention"
+            done
+            
+            # Wait for all operations to complete or be aborted
+            echo "⏳ Waiting for operations to complete or be aborted..."
+            max_retries=20
+            retry_count=0
+            while [ $retry_count -lt $max_retries ]; do
+              ONGOING_COUNT=$(az aks operation list --resource-group "$RG" --name "$AKS" --query "length([?status=='Running' || status=='Pending'])" -o tsv)
+              if [ "$ONGOING_COUNT" -eq 0 ]; then
+                echo "✅ No more ongoing operations"
+                break
+              fi
+              echo "⏳ Still have $ONGOING_COUNT ongoing operations, waiting 15s... ($(($retry_count + 1))/$max_retries)"
+              sleep 15
+              retry_count=$((retry_count + 1))
+            done
+            
+            if [ $retry_count -eq $max_retries ]; then
+              echo "::warning::Timed out waiting for operations to complete. This might cause Terraform to fail."
+            fi
+          else
+            echo "✅ No ongoing operations found"
+          fi
+          
+          # Wait for the cluster to be in a Succeeded provisioning state
+          echo "⏳ Ensuring cluster is in a Succeeded state..."
           max_retries=10
           retry_count=0
           while [ $retry_count -lt $max_retries ]; do
-            if az aks show --resource-group "$RG" --name "$AKS" --query "provisioningState" -o tsv | grep -q "Succeeded"; then
-              echo "✅ AKS reconciliation completed successfully"
+            PROVISION_STATE=$(az aks show --resource-group "$RG" --name "$AKS" --query "provisioningState" -o tsv)
+            if [ "$PROVISION_STATE" = "Succeeded" ]; then
+              echo "✅ AKS cluster is in Succeeded state"
               break
             fi
-            echo "⏳ AKS still reconciling, waiting 30s... ($(($retry_count + 1))/$max_retries)"
+            echo "⏳ AKS state: $PROVISION_STATE, waiting 30s... ($(($retry_count + 1))/$max_retries)"
             sleep 30
             retry_count=$((retry_count + 1))
           done
           
           if [ $retry_count -eq $max_retries ]; then
-            echo "⚠️ AKS reconciliation timed out but continuing with deployment"
+            echo "::warning::Timed out waiting for AKS to reach Succeeded state. This might cause Terraform to fail."
           fi
         else
-          echo "⏭️ No existing AKS cluster to reconcile"
+          echo "⏭️ No existing AKS cluster to check"
         fi
 
     ################################################################
-    # 3 · Terraform apply
+    # 3 · Terraform apply (with retries)
     ################################################################
     - name: Terraform apply
       env: { TF_IN_AUTOMATION: true }
       run: |
-        terraform -chdir="$TF_DIR" apply -auto-approve -input=false -no-color \
-          -var="node_vm_size=${{ env.NODE_VM_SIZE }}"
+        max_retries=5
+        retry_count=0
+        
+        while [ $retry_count -lt $max_retries ]; do
+          echo "🔄 Running Terraform apply (attempt $(($retry_count + 1))/$max_retries)..."
+          
+          if terraform -chdir="$TF_DIR" apply -auto-approve -input=false -no-color \
+            -var="node_vm_size=${{ env.NODE_VM_SIZE }}"; then
+            echo "✅ Terraform apply succeeded!"
+            exit 0
+          fi
+          
+          exit_code=$?
+          echo "⚠️ Terraform apply failed with exit code $exit_code"
+          
+          # Check if this is the last attempt
+          if [ $(($retry_count + 1)) -eq $max_retries ]; then
+            echo "❌ All retry attempts exhausted"
+            exit $exit_code
+          fi
+          
+          # Wait before retrying
+          retry_count=$((retry_count + 1))
+          wait_time=$((30 * retry_count))
+          echo "⏳ Waiting ${wait_time}s before retry..."
+          sleep $wait_time
+        done
 
     ################################################################
     # 4 · kubeconfig

infra/azure/main.tf

@@ -51,11 +51,17 @@ resource "azurerm_kubernetes_cluster" "this" {
     name       = "default"
     node_count = 1
     vm_size    = var.node_vm_size
+    
+    # Use lifecycle to ignore changes to upgrade_settings to prevent conflicts
+    # This lets Azure manage these settings instead of forcing our values
+  }
 
-    # Explicitly define upgrade settings to match Azure defaults
-    upgrade_settings {
-      max_surge = "33%"
-    }
+  # Prevent Terraform from modifying certain attributes that might cause conflicts
+  lifecycle {
+    ignore_changes = [
+      default_node_pool[0].upgrade_settings,
+      tags
+    ]
   }
 
   identity { type = "SystemAssigned" }