Refactor VMSS infra: manage Key Vault and images in Terraform

Moves Key Vault and custom image resource group management into Terraform, replacing data sources with resource definitions. Updates GitHub Actions workflow to import additional resources and use a subscription variable. Adds RBAC role assignments and a wait for RBAC propagation before storing the GitHub runner token in Key Vault. Updates documentation to clarify test duration and cost calculation.

Author: potatoqualitee

.github/workflows/vmss-deploy.yml

@@ -33,6 +33,7 @@ jobs:
           echo "ARM_CLIENT_SECRET=$(echo '${{ secrets.VMSS_AZURE_CREDENTIALS }}' | jq -r '.clientSecret')" >> $GITHUB_ENV
           echo "ARM_SUBSCRIPTION_ID=$(echo '${{ secrets.VMSS_AZURE_CREDENTIALS }}' | jq -r '.subscriptionId')" >> $GITHUB_ENV
           echo "ARM_TENANT_ID=$(echo '${{ secrets.VMSS_AZURE_CREDENTIALS }}' | jq -r '.tenantId')" >> $GITHUB_ENV
+          echo "ARM_USE_AZUREAD=true" >> $GITHUB_ENV
 
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v3
@@ -82,20 +83,38 @@ jobs:
         continue-on-error: true
         working-directory: ./gh-runners
         run: |
+          SUBSCRIPTION_ID="65a430fb-5a9a-49ff-969e-05d1beaa88fb"
+
+          # Import main resource group if it exists outside Terraform state
+          terraform import azurerm_resource_group.rg /subscriptions/${SUBSCRIPTION_ID}/resourceGroups/dbatools-ci-runners 2>/dev/null || true
+
+          # Import images resource group if it exists outside Terraform state
+          terraform import azurerm_resource_group.images /subscriptions/${SUBSCRIPTION_ID}/resourceGroups/dbatools-ci-images 2>/dev/null || true
+
+          # Import storage account if it exists outside Terraform state
+          terraform import azurerm_storage_account.tfstate /subscriptions/${SUBSCRIPTION_ID}/resourceGroups/dbatools-ci-runners/providers/Microsoft.Storage/storageAccounts/dbatoolstfstate 2>/dev/null || true
+
+          # Import storage container if it exists outside Terraform state
+          terraform import azurerm_storage_container.tfstate https://dbatoolstfstate.blob.core.windows.net/tfstate 2>/dev/null || true
+
+          # Import Key Vault if it exists outside Terraform state
+          terraform import azurerm_key_vault.vmss /subscriptions/${SUBSCRIPTION_ID}/resourceGroups/dbatools-ci-runners/providers/Microsoft.KeyVault/vaults/dbatoolsci 2>/dev/null || true
+
           # Import VNet if it exists outside Terraform state
-          terraform import azurerm_virtual_network.vnet /subscriptions/65a430fb-5a9a-49ff-969e-05d1beaa88fb/resourceGroups/dbatools-ci-runners/providers/Microsoft.Network/virtualNetworks/dbatools-runner-vmss-vnet 2>/dev/null || true
+          terraform import azurerm_virtual_network.vnet /subscriptions/${SUBSCRIPTION_ID}/resourceGroups/dbatools-ci-runners/providers/Microsoft.Network/virtualNetworks/dbatools-runner-vmss-vnet 2>/dev/null || true
 
           # Import subnet if it exists outside Terraform state
-          terraform import azurerm_subnet.subnet /subscriptions/65a430fb-5a9a-49ff-969e-05d1beaa88fb/resourceGroups/dbatools-ci-runners/providers/Microsoft.Network/virtualNetworks/dbatools-runner-vmss-vnet/subnets/dbatools-runner-vmss-subnet 2>/dev/null || true
+          terraform import azurerm_subnet.subnet /subscriptions/${SUBSCRIPTION_ID}/resourceGroups/dbatools-ci-runners/providers/Microsoft.Network/virtualNetworks/dbatools-runner-vmss-vnet/subnets/dbatools-runner-vmss-subnet 2>/dev/null || true
 
           # Import VMSS if it exists outside Terraform state
-          terraform import azurerm_windows_virtual_machine_scale_set.vmss /subscriptions/65a430fb-5a9a-49ff-969e-05d1beaa88fb/resourceGroups/dbatools-ci-runners/providers/Microsoft.Compute/virtualMachineScaleSets/dbatools-runner-vmss 2>/dev/null || true
+          terraform import azurerm_windows_virtual_machine_scale_set.vmss /subscriptions/${SUBSCRIPTION_ID}/resourceGroups/dbatools-ci-runners/providers/Microsoft.Compute/virtualMachineScaleSets/dbatools-runner-vmss 2>/dev/null || true
 
           # Import VMSS extension if it exists outside Terraform state
-          terraform import azurerm_virtual_machine_scale_set_extension.vmss /subscriptions/65a430fb-5a9a-49ff-969e-05d1beaa88fb/resourceGroups/dbatools-ci-runners/providers/Microsoft.Compute/virtualMachineScaleSets/dbatools-runner-vmss/extensions/CustomScriptExtension 2>/dev/null || true
+          terraform import azurerm_virtual_machine_scale_set_extension.vmss /subscriptions/${SUBSCRIPTION_ID}/resourceGroups/dbatools-ci-runners/providers/Microsoft.Compute/virtualMachineScaleSets/dbatools-runner-vmss/extensions/CustomScriptExtension 2>/dev/null || true
 
-          # Import role assignment if it exists outside Terraform state
-          terraform import azurerm_role_assignment.vmss_kv_secrets_user /subscriptions/65a430fb-5a9a-49ff-969e-05d1beaa88fb/resourceGroups/dbatools-ci-runners/providers/Microsoft.KeyVault/vaults/dbatoolsci|Key_Vault_Secrets_User 2>/dev/null || true
+          # Import role assignments if they exist outside Terraform state
+          # Note: Role assignment IDs are GUIDs and may need to be discovered first
+          echo "Note: Role assignments for Key Vault access will be created if they don't exist"
 
       - name: Terraform Apply
         if: github.event_name == 'push'

MIGRATION-COMPLETE.md

@@ -55,11 +55,13 @@ Successfully migrated from AppVeyor to GitHub Actions with Azure VMSS self-hoste
 2. **CI tests queue** (ci.yml) - 10 jobs waiting for runners
 3. **Autoscaler detects queue** (autoscaler.yml) - Scales VMSS to 3 instances within 1 minute
 4. **Runners register** - VMs boot, register with GitHub (3 minutes)
-5. **Tests run** - Jobs execute automatically (2.5 hours for full matrix)
+5. **Tests run** - Jobs execute automatically (duration depends on your test suite)
 6. **Runners destroy** - Ephemeral runners self-destruct after each job
 7. **Autoscaler scales down** - Detects empty queue, scales VMSS to 0 (2 minutes after completion)
 
-**Total cost per push: ~$1.25** (3 runners × 2.5 hours × $0.166/hr)
+**Cost per push:** 3 runners × actual_test_hours × $0.166/hr
+- **Each job times out at 60 minutes max**
+- **Actual cost depends on real test duration** (unknown until first run)
 
 **Your involvement: Push code. That's it.** ✅
 

gh-runners/backend.tf

@@ -1,20 +1,9 @@
-# Resource group for Terraform state storage
-resource "azurerm_resource_group" "tfstate" {
-  name     = "dbatools-ci-runners"
-  location = var.location
-
-  tags = {
-    Environment = "CI"
-    ManagedBy   = "Terraform"
-    Purpose     = "Terraform-State"
-  }
-}
-
 # Storage account for Terraform state
+# Note: Resource group is created in vmss.tf and bootstrapped via GitHub Actions
 resource "azurerm_storage_account" "tfstate" {
   name                     = "dbatoolstfstate"
-  resource_group_name      = azurerm_resource_group.tfstate.name
-  location                 = azurerm_resource_group.tfstate.location
+  resource_group_name      = azurerm_resource_group.rg.name
+  location                 = azurerm_resource_group.rg.location
   account_tier             = "Standard"
   account_replication_type = "LRS"
   min_tls_version          = "TLS1_2"

gh-runners/version.tf

@@ -17,6 +17,10 @@ terraform {
       source  = "integrations/github"
       version = "~> 5.0"
     }
+    time = {
+      source  = "hashicorp/time"
+      version = "~> 0.9"
+    }
   }
 }
 

gh-runners/vmss.tf

@@ -13,16 +13,54 @@ resource "azurerm_resource_group" "rg" {
   }
 }
 
-# Reference existing Key Vault
-data "azurerm_key_vault" "vmss" {
-  name                = var.keyvault_name
-  resource_group_name = data.azurerm_resource_group.rg.name
+# Create Key Vault for storing GitHub runner token
+resource "azurerm_key_vault" "vmss" {
+  name                       = var.keyvault_name
+  resource_group_name        = azurerm_resource_group.rg.name
+  location                   = azurerm_resource_group.rg.location
+  tenant_id                  = data.azurerm_client_config.current.tenant_id
+  sku_name                   = "standard"
+  soft_delete_retention_days = 7
+  purge_protection_enabled   = false
+
+  # Enable RBAC authorization
+  enable_rbac_authorization = true
+
+  # Network ACLs - allow Azure services
+  network_acls {
+    bypass         = "AzureServices"
+    default_action = "Allow"
+  }
+
+  tags = {
+    Environment = "CI"
+    ManagedBy   = "Terraform"
+  }
 }
 
-# Reference existing custom image
+# Resource group for custom images
+# This must exist and contain the golden image with GitHub Actions runner pre-installed
+resource "azurerm_resource_group" "images" {
+  name     = var.image_resource_group
+  location = var.location
+
+  tags = {
+    Environment = "CI"
+    ManagedBy   = "Terraform"
+    Purpose     = "Custom-Images"
+  }
+}
+
+# Reference custom golden image
+# PREREQUISITE: The golden image must be created separately and contain:
+# - Windows Server with SQL Server pre-installed
+# - GitHub Actions runner binaries at C:\actions-runner
+# - All required dependencies (PowerShell modules, etc.)
 data "azurerm_image" "golden" {
   name                = var.image_name
-  resource_group_name = var.image_resource_group
+  resource_group_name = azurerm_resource_group.images.name
+
+  depends_on = [azurerm_resource_group.images]
 }
 
 # Create virtual network for VMSS
@@ -116,10 +154,41 @@ resource "azurerm_virtual_machine_scale_set_extension" "vmss" {
 
 # Grant VMSS managed identity access to Key Vault secrets
 resource "azurerm_role_assignment" "vmss_kv_secrets_user" {
-  scope                = data.azurerm_key_vault.vmss.id
+  scope                = azurerm_key_vault.vmss.id
   role_definition_name = "Key Vault Secrets User"
   principal_id         = azurerm_windows_virtual_machine_scale_set.vmss.identity[0].principal_id
 
-  # Wait for VMSS to be created
-  depends_on = [azurerm_windows_virtual_machine_scale_set.vmss]
+  depends_on = [
+    azurerm_windows_virtual_machine_scale_set.vmss,
+    azurerm_key_vault.vmss
+  ]
+}
+
+# Grant Terraform service principal access to manage Key Vault secrets
+resource "azurerm_role_assignment" "terraform_kv_secrets_officer" {
+  scope                = azurerm_key_vault.vmss.id
+  role_definition_name = "Key Vault Secrets Officer"
+  principal_id         = data.azurerm_client_config.current.object_id
+
+  depends_on = [azurerm_key_vault.vmss]
+}
+
+# Wait for role assignment to propagate
+resource "time_sleep" "wait_for_rbac" {
+  create_duration = "60s"
+
+  depends_on = [azurerm_role_assignment.terraform_kv_secrets_officer]
+}
+
+# Store GitHub runner token in Key Vault
+resource "azurerm_key_vault_secret" "github_runner_token" {
+  name         = "GITHUB-RUNNER-TOKEN"
+  value        = var.github_token
+  key_vault_id = azurerm_key_vault.vmss.id
+
+  depends_on = [
+    azurerm_key_vault.vmss,
+    azurerm_role_assignment.terraform_kv_secrets_officer,
+    time_sleep.wait_for_rbac
+  ]
 }