feat(merge queue): add merge queue based CI and address comments

Author: jshiwamV

.github/setup/aws/backend.tf

@@ -1,7 +1,7 @@
 terraform {
   backend "s3" {
     bucket  = "materialize-terraform-self-managed-state"
-    key     = "github-setup/oidc/terraform.tfstate"
+    key     = "github-setup/oidc/aws/terraform.tfstate"
     region  = "us-east-1"
     encrypt = true
     # profile = "" # Add your profile name here since backend block doesn't accept variables

.github/setup/azure/accessTokenLifeTime.sh

@@ -1,3 +1,9 @@
+
+#!/bin/bash
+
+# Exit early on error/unset var/pipe failure
+set -euo pipefail
+
 # 1. Create a 4-hour token lifetime policy and capture the policy ID
 POLICY_RESPONSE=$(az rest --method POST \
   --url "https://graph.microsoft.com/v1.0/policies/tokenLifetimePolicies" \
@@ -21,4 +27,4 @@ az rest --method POST \
   --headers "Content-Type=application/json" \
   --body "{\"@odata.id\": \"https://graph.microsoft.com/v1.0/policies/tokenLifetimePolicies/${POLICY_ID}\"}"
 
-echo "✅ Token lifetime policy applied successfully!"
+echo "✅ Token lifetime policy applied successfully!"

.github/setup/azure/main.tf

@@ -49,6 +49,22 @@ data "azuread_client_config" "current" {}
 # Get current subscription
 data "azurerm_client_config" "current" {}
 
+# Get built-in role definitions for ABAC conditions
+data "azurerm_role_definition" "owner" {
+  name  = "Owner"
+  scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}"
+}
+
+data "azurerm_role_definition" "user_access_administrator" {
+  name  = "User Access Administrator"
+  scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}"
+}
+
+data "azurerm_role_definition" "rbac_administrator" {
+  name  = "Role Based Access Control Administrator"
+  scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}"
+}
+
 
 # Principle of Least Privilege: Minimal role assignments based on fixture requirements
 # Refer roles from https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles to follow the principle of least privilege
@@ -155,25 +171,26 @@ resource "azurerm_role_assignment" "github_actions_rbac_admin" {
 
   # ABAC condition to block assignment of high-privilege roles
   # Allows assignment of any role EXCEPT: Owner, User Access Administrator, RBAC Administrator
-  # Role GUIDs: 8e3af657... (Owner), 18d7d88d... (User Access Admin), f58310d9... (RBAC Admin)
+  # Using data sources to get role GUIDs dynamically instead of hardcoding
+  # TODO: Test and validate this configuration, havent tested due to Perms issue.
   condition = <<-EOT
     (
       (!(ActionMatches{'Microsoft.Authorization/roleAssignments/write'})) 
       OR 
       (@Request[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAllValues:GuidNotEquals {
-        8e3af657-a8ff-443c-a75c-2fe8c4bcb635, 
-        18d7d88d-d35e-4fb5-a5c3-7773c20a72d9, 
-        f58310d9-a9f6-439a-9e8d-f62e7b41a168
+        ${data.azurerm_role_definition.owner.id}, 
+        ${data.azurerm_role_definition.user_access_administrator.id}, 
+        ${data.azurerm_role_definition.rbac_administrator.id}
       })
     ) 
     AND 
     (
       (!(ActionMatches{'Microsoft.Authorization/roleAssignments/delete'})) 
       OR 
       (@Resource[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAllValues:GuidNotEquals {
-        8e3af657-a8ff-443c-a75c-2fe8c4bcb635, 
-        18d7d88d-d35e-4fb5-a5c3-7773c20a72d9, 
-        f58310d9-a9f6-439a-9e8d-f62e7b41a168
+        ${data.azurerm_role_definition.owner.id}, 
+        ${data.azurerm_role_definition.user_access_administrator.id}, 
+        ${data.azurerm_role_definition.rbac_administrator.id}
       })
     )
   EOT

.github/workflows/README.md

@@ -1,14 +1,15 @@
 # Infrastructure Testing Workflows
 
-## 🛡️ **Approval-Gated Testing**
+## 🛡️ **Merge Queue Integration**
 
-Infrastructure tests **require PR approval** to prevent accidental resource provisioning and manage costs.
+Infrastructure tests **integrate with GitHub's merge queue** to ensure only approved, tested code reaches `main`.
 
 ### **How It Works**
-1. **Create PR** → No tests run initially
-2. **Get approval** → Tests run automatically  
-3. **Push changes** → Tests re-run automatically (if PR approved)
-4. **Manual trigger** → Use `gh workflow run test-<cloud>.yml` if needed
+1. **Create PR** → Request review
+2. **Get approval** → PR enters merge queue automatically
+3. **Tests run** → Only affected cloud providers tested (smart path filtering)
+4. **Auto-merge** → When tests pass, code merges to `main`
+5. **Manual trigger** → Use `gh workflow run test-<cloud>.yml` if needed
 
 
 ### **What Gets Tested**
@@ -22,13 +23,19 @@ Infrastructure tests **require PR approval** to prevent accidental resource prov
 
 ### **Features**
 - ✅ **Granular path filtering** - Only tests infrastructure changes (excludes docs/README)
-- ✅ **Smart cloud detection** - Tests only affected clouds, or all clouds for shared changes
-- ✅ **Race condition prevention** - One workflow per PR
-- ✅ **Parallel cloud testing** - AWS/GCP/Azure run simultaneously  
-- ✅ **Auto-retest** - New pushes trigger tests if PR approved
+- ✅ **Smart cloud detection** - Tests only affected clouds, or all clouds for shared changes  
+- ✅ **Merge queue integration** - Automatic testing on approved PRs
+- ✅ **Conflict resolution** - Auto-retests when merge conflicts occur
+- ✅ **Parallel cloud testing** - AWS/GCP/Azure run simultaneously when needed
 
 ## **Setup Requirements**
 
+**Branch Protection + Merge Queue:**
+- Enable merge queue for `main` branch
+- Require PR approvals (dismisses stale approvals)  
+- Add required status checks: `AWS Tests`, `GCP Tests`, `Azure Tests`
+
+
 **Repository Secrets:**
 ```
 MATERIALIZE_LICENSE_KEY
@@ -43,11 +50,14 @@ TF_TEST_S3_BUCKET, TF_TEST_S3_REGION, TF_TEST_S3_PREFIX
 GOOGLE_PROJECT, AWS_REGION
 ```
 
-## **Manual Override**
+## **Manual Testing**
 
 ```bash
-# Run tests without approval (requires repo access)
+# Run individual cloud tests manually (for debugging/testing)
 gh workflow run test-aws.yml --ref your-branch
 gh workflow run test-gcp.yml --ref your-branch  
 gh workflow run test-azure.yml --ref your-branch
+
+# Note: Manual runs bypass merge queue but still require proper authentication
+# Production merges should always go through the merge queue process
 ```

.github/workflows/test-aws.yml

@@ -1,9 +1,29 @@
 name: AWS Tests
 
 on:
-  # Only run via approval workflow or manual trigger
-  # Removed pull_request trigger to prevent duplicate runs
-  workflow_call: # Called by test-on-approval.yml
+  # Primary trigger: GitHub merge queue with smart path filtering
+  merge_group:
+    paths-ignore:
+      # Exclude other cloud providers
+      - "gcp/**"
+      - "azure/**"
+      - "test/gcp/**"
+      - "test/azure/**"
+      # Exclude setup files from other cloud providers
+      - ".github/setup/gcp/**"
+      - ".github/setup/azure/**"
+      # Exclude examples
+      - "aws/examples/**"
+      # Exclude documentation and config
+      - "**.md"
+      - "**.env"
+      - ".gitignore"
+      - "LICENSE"
+      # Exclude Azure/GCP specific workflows
+      - ".github/workflows/test-gcp*.yml"
+      - ".github/workflows/test-azure*.yml"
+      - ".github/scripts/**"
+  # Manual trigger: For testing and debugging purposes
   workflow_dispatch:
     inputs:
       test_stage:
@@ -15,9 +35,9 @@ on:
           - network-only
           - full-test
 
-  # TODO: Add scheduled and PR triggers later
+  # Future enhancement: Add scheduled runs for nightly testing
   # schedule:
-  #   - cron: '30 4 * * *'  # 10 AM IST (4:30 AM UTC)
+  #   - cron: '30 4 * * *'  # 10 AM IST (4:30 AM UTC) for nightly infrastructure validation
 
 permissions:
   id-token: write # Required for OIDC
@@ -27,7 +47,8 @@ jobs:
   test-aws:
     name: AWS Infrastructure Tests
     runs-on: ubuntu-latest
-    # Note: Approval-gated tests now handled by test-on-approval.yml workflow
+    # Note: Triggered automatically when approved PRs enter GitHub's merge queue
+    # Smart path filtering ensures tests run only when AWS infrastructure changes
 
     steps:
       - name: Checkout code