...

Author: nullchimp

.github/workflows/deploy-azure.yml

@@ -101,75 +101,88 @@ jobs:
         fi
 
     ################################################################
-    # 2.5 · Check for ongoing operations and abort/wait
+    # 2.5 · Wait for AKS to be in a stable state
     ################################################################
-    - name: Check AKS operations and abort if needed
+    - name: Wait for AKS stability
       run: |
         set -euo pipefail
         RG="rg-aks-memgraph-${{ env.ENVIRONMENT }}"
         AKS="aks-memgraph-${{ env.ENVIRONMENT }}"
         
         if az aks show --resource-group "$RG" --name "$AKS" &>/dev/null; then
-          # Check for ongoing operations
-          echo "🔍 Checking for ongoing AKS operations..."
-          ONGOING_OPS=$(az aks operation list --resource-group "$RG" --name "$AKS" --query "[?status=='Running' || status=='Pending']" -o json)
-          
-          if [[ $(echo "$ONGOING_OPS" | jq 'length') -gt 0 ]]; then
-            echo "⚠️ Found ongoing operations. Attempting to abort them..."
-            OPERATION_IDS=$(echo "$ONGOING_OPS" | jq -r '.[].operationId')
-            
-            for OP_ID in $OPERATION_IDS; do
-              echo "🛑 Aborting operation $OP_ID..."
-              az aks operation-abort --resource-group "$RG" --name "$AKS" --operation-id "$OP_ID" || echo "⚠️ Could not abort operation, may require manual intervention"
-            done
-            
-            # Wait for all operations to complete or be aborted
-            echo "⏳ Waiting for operations to complete or be aborted..."
-            max_retries=20
-            retry_count=0
-            while [ $retry_count -lt $max_retries ]; do
-              ONGOING_COUNT=$(az aks operation list --resource-group "$RG" --name "$AKS" --query "length([?status=='Running' || status=='Pending'])" -o tsv)
-              if [ "$ONGOING_COUNT" -eq 0 ]; then
-                echo "✅ No more ongoing operations"
-                break
-              fi
-              echo "⏳ Still have $ONGOING_COUNT ongoing operations, waiting 15s... ($(($retry_count + 1))/$max_retries)"
-              sleep 15
-              retry_count=$((retry_count + 1))
-            done
-            
-            if [ $retry_count -eq $max_retries ]; then
-              echo "::warning::Timed out waiting for operations to complete. This might cause Terraform to fail."
-            fi
-          else
-            echo "✅ No ongoing operations found"
-          fi
+          echo "🔍 Checking AKS cluster state..."
           
           # Wait for the cluster to be in a Succeeded provisioning state
-          echo "⏳ Ensuring cluster is in a Succeeded state..."
-          max_retries=10
+          echo "⏳ Waiting for cluster to be in a stable state..."
+          max_retries=15
           retry_count=0
+          
           while [ $retry_count -lt $max_retries ]; do
             PROVISION_STATE=$(az aks show --resource-group "$RG" --name "$AKS" --query "provisioningState" -o tsv)
+            
             if [ "$PROVISION_STATE" = "Succeeded" ]; then
               echo "✅ AKS cluster is in Succeeded state"
+              # Force a small delay to ensure all backend operations are truly complete
+              sleep 30
               break
             fi
+            
             echo "⏳ AKS state: $PROVISION_STATE, waiting 30s... ($(($retry_count + 1))/$max_retries)"
             sleep 30
             retry_count=$((retry_count + 1))
           done
           
           if [ $retry_count -eq $max_retries ]; then
-            echo "::warning::Timed out waiting for AKS to reach Succeeded state. This might cause Terraform to fail."
+            echo "::warning::Timed out waiting for AKS to reach Succeeded state."
+            echo "⚠️ Attempting to proceed anyway, but Terraform might fail."
           fi
+          
+          # Display detailed cluster information for debugging
+          echo "📊 Current AKS cluster status:"
+          az aks show --resource-group "$RG" --name "$AKS" --query "{provisioningState:provisioningState,powerState:powerState,kubernetesVersion:kubernetesVersion,nodeResourceGroup:nodeResourceGroup}" -o table || true
+          
+          # This pause gives the system time to finalize any background operations
+          echo "⏱️ Allowing 60 seconds of stabilization time before proceeding..."
+          sleep 60
         else
           echo "⏭️ No existing AKS cluster to check"
         fi
 
     ################################################################
     # 3 · Terraform apply (with retries)
     ################################################################
+    - name: Reconcile node resource group if needed
+      run: |
+        set -euo pipefail
+        RG="rg-aks-memgraph-${{ env.ENVIRONMENT }}"
+        AKS="aks-memgraph-${{ env.ENVIRONMENT }}"
+        
+        if az aks show --resource-group "$RG" --name "$AKS" &>/dev/null; then
+          # Check if the node resource group exists
+          NODE_RG=$(az aks show --resource-group "$RG" --name "$AKS" --query "nodeResourceGroup" -o tsv)
+          echo "🔍 Node resource group is: $NODE_RG"
+          
+          if ! az group exists --name "$NODE_RG" | grep -q true; then
+            echo "⚠️ Node resource group $NODE_RG doesn't exist, attempting to reconcile..."
+            az aks update --resource-group "$RG" --name "$AKS" --yes || echo "⚠️ Reconciliation failed, proceeding anyway"
+            
+            # Wait for any update operation to complete
+            echo "⏱️ Waiting 2 minutes for update operation to complete..."
+            sleep 120
+            
+            # Check again
+            if ! az group exists --name "$NODE_RG" | grep -q true; then
+              echo "::warning::Node resource group still doesn't exist after reconciliation attempt"
+            else
+              echo "✅ Node resource group now exists"
+            fi
+          else
+            echo "✅ Node resource group exists"
+          fi
+        else
+          echo "⏭️ No existing AKS cluster to reconcile"
+        fi
+
     - name: Terraform apply
       env: { TF_IN_AUTOMATION: true }
       run: |

infra/azure/main.tf

@@ -59,8 +59,17 @@ resource "azurerm_kubernetes_cluster" "this" {
   # Prevent Terraform from modifying certain attributes that might cause conflicts
   lifecycle {
     ignore_changes = [
+      # Ignore all node pool settings that Azure might modify internally
       default_node_pool[0].upgrade_settings,
-      tags
+      default_node_pool[0].orchestrator_version,
+      default_node_pool[0].node_labels,
+      default_node_pool[0].node_taints,
+      # Ignore other common attributes that cause drift
+      kubernetes_version,
+      sku_tier,
+      tags,
+      # Azure sometimes modifies these internal networking settings
+      network_profile
     ]
   }