Migrate Azure Firewall and route tables to core configuration

CHANGELOG.md

@@ -6,6 +6,7 @@ ENHANCEMENTS:
 * Enabled Structured Azure Firewall logs for TRE firewall. [#4430](https://github.com/microsoft/AzureTRE/issues/4430)
 * Deny public access to TRE management storage account, and add private endpoint for TRE core [#4353](https://github.com/microsoft/AzureTRE/issues/4353)
 * Added anonymous access enablement for Nexus by default issue. [#4387](https://github.com/microsoft/AzureTRE/pull/4387)
+* Migrate Azure Firewall and Route Tables to Core Terraform ([#4342](https://github.com/microsoft/AzureTRE/pull/4342))
 
 BUG FIXES:
 * Fix the management storage access error while executing `make show-core-output` command, and remove redundant error messages from `mgmtstorage_enable_public_access.sh` script ([#4404](https://github.com/microsoft/AzureTRE/issues/4404))

Makefile

@@ -25,10 +25,6 @@ build-and-push-airlock-processor: build-airlock-processor push-airlock-processor
 help: ## 💬 This help message :)
 	@grep -E '[a-zA-Z_-]+:.*?## .*$$' $(firstword $(MAKEFILE_LIST)) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}'
 
-# to move your environment from the single 'core' deployment (which includes the firewall)
-# toward the shared services model, where it is split out - run the following make target before a tre-deploy
-# This will remove + import the resource state into a shared service
-migrate-firewall-state: prepare-tf-state
 
 bootstrap:
 	$(call target_title, "Bootstrap Terraform") \
@@ -96,15 +92,12 @@ push-resource-processor-vm-porter-image:
 push-airlock-processor:
 	$(call push_image,"airlock-processor","${MAKEFILE_DIR}/airlock_processor/_version.py")
 
-# # These targets are for a graceful migration of Firewall
-# # from terraform state in Core to a Shared Service.
-# # See https://github.com/microsoft/AzureTRE/issues/1177
-prepare-tf-state:
+migrate-firewall-state:
 	$(call target_title, "Preparing terraform state") \
 	&& . ${MAKEFILE_DIR}/devops/scripts/check_dependencies.sh nodocker,env \
-	&& pushd ${MAKEFILE_DIR}/core/terraform > /dev/null && ../../shared_services/firewall/terraform/remove_state.sh && popd > /dev/null \
-	&& pushd ${MAKEFILE_DIR}/templates/shared_services/firewall/terraform > /dev/null && ./import_state.sh && popd > /dev/null
-# / End migration targets
+	&& pushd ${MAKEFILE_DIR}/templates/shared_services/firewall/terraform > /dev/null && ${MAKEFILE_DIR}/core/terraform/firewall/remove_state.sh && popd > /dev/null \
+	&& pushd ${MAKEFILE_DIR}/core/terraform > /dev/null && ${MAKEFILE_DIR}/core/terraform/firewall/import_state.sh && popd > /dev/null
+
 
 deploy-core: tre-start
 	$(call target_title, "Deploying TRE") \
@@ -318,10 +311,10 @@ deploy-shared-service:
 	&& ${MAKEFILE_DIR}/devops/scripts/deploy_shared_service.sh $${PROPS}
 
 firewall-install:
+	$(MAKE) migrate-firewall-state
 	. ${MAKEFILE_DIR}/devops/scripts/check_dependencies.sh env \
 	&& $(MAKE) bundle-build bundle-publish bundle-register deploy-shared-service \
-	DIR=${MAKEFILE_DIR}/templates/shared_services/firewall/ BUNDLE_TYPE=shared_service \
-	PROPS="$${FIREWALL_SKU+--firewall_sku $${FIREWALL_SKU} }$${FIREWALL_FORCE_TUNNEL_IP+--firewall_force_tunnel_ip $${FIREWALL_FORCE_TUNNEL_IP} }"
+	DIR=${MAKEFILE_DIR}/templates/shared_services/firewall/ BUNDLE_TYPE=shared_service
 
 static-web-upload:
 	$(call target_title, "Uploading to static website") \

templates/shared_services/firewall/terraform/firewall.tf → core/terraform/firewall/firewall.tf

@@ -2,10 +2,10 @@ resource "azurerm_public_ip" "fwtransit" {
   count               = var.firewall_force_tunnel_ip != "" ? 0 : 1
   name                = "pip-fw-${var.tre_id}"
   resource_group_name = local.core_resource_group_name
-  location            = data.azurerm_resource_group.rg.location
+  location            = var.location
   allocation_method   = "Static"
   sku                 = "Standard"
-  tags                = local.tre_shared_service_tags
+  tags                = var.tre_core_tags
 
   lifecycle { ignore_changes = [tags, zones] }
 }
@@ -18,35 +18,34 @@ moved {
 resource "azurerm_public_ip" "fwmanagement" {
   count               = (var.firewall_force_tunnel_ip != "" || local.effective_firewall_sku == "Basic") ? 1 : 0
   name                = "pip-fw-management-${var.tre_id}"
-  resource_group_name = local.core_resource_group_name
-  location            = data.azurerm_resource_group.rg.location
+  resource_group_name = var.resource_group_name
+  location            = var.location
   allocation_method   = "Static"
   sku                 = "Standard"
-  tags                = local.tre_shared_service_tags
+  tags                = var.tre_core_tags
 
   lifecycle { ignore_changes = [tags, zones] }
 }
 
-
 resource "azurerm_firewall" "fw" {
   name                = local.firewall_name
   resource_group_name = local.core_resource_group_name
-  location            = data.azurerm_resource_group.rg.location
+  location            = var.location
   sku_tier            = local.effective_firewall_sku
   sku_name            = "AZFW_VNet"
   firewall_policy_id  = azurerm_firewall_policy.root.id
-  tags                = local.tre_shared_service_tags
+  tags                = var.tre_core_tags
   ip_configuration {
     name                 = "fw-ip-configuration"
-    subnet_id            = data.azurerm_subnet.firewall.id
+    subnet_id            = var.firewall_subnet_id
     public_ip_address_id = var.firewall_force_tunnel_ip != "" ? null : azurerm_public_ip.fwtransit[0].id
   }
 
   dynamic "management_ip_configuration" {
     for_each = (var.firewall_force_tunnel_ip != "" || local.effective_firewall_sku == "Basic") ? [1] : []
     content {
       name                 = "mgmtconfig"
-      subnet_id            = data.azurerm_subnet.firewall_management.id
+      subnet_id            = var.firewall_management_subnet_id
       public_ip_address_id = azurerm_public_ip.fwmanagement[0].id
     }
   }
@@ -61,7 +60,7 @@ data "azurerm_monitor_diagnostic_categories" "firewall" {
 resource "azurerm_monitor_diagnostic_setting" "firewall" {
   name                           = "diagnostics-fw-${var.tre_id}"
   target_resource_id             = azurerm_firewall.fw.id
-  log_analytics_workspace_id     = data.azurerm_log_analytics_workspace.tre.id
+  log_analytics_workspace_id     = var.log_analytics_workspace_id
   log_analytics_destination_type = "Dedicated"
 
   dynamic "enabled_log" {
@@ -79,10 +78,10 @@ resource "azurerm_monitor_diagnostic_setting" "firewall" {
 
 resource "azurerm_firewall_policy" "root" {
   name                = local.firewall_policy_name
-  resource_group_name = local.core_resource_group_name
-  location            = data.azurerm_resource_group.rg.location
+  resource_group_name = var.resource_group_name
+  location            = var.location
   sku                 = local.effective_firewall_sku
-  tags                = local.tre_shared_service_tags
+  tags                = var.tre_core_tags
 
   lifecycle { ignore_changes = [tags] }
 }

core/terraform/firewall/import_state.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+# shellcheck disable=SC2154
+
+# See remove_state.sh for the purpose of these scripts
+echo "IMPORTING STATE FOR FIREWALL..."
+
+# check for the existence of the RG. If it's not there it's because we're in CI and building from scratch - we can skip this script
+set +e
+RESOURCE_GROUP_ID="rg-${TRE_ID}"
+if ! az group show -n "$RESOURCE_GROUP_ID"; then
+  echo "RG not found, skipping import_state"
+  exit 0
+fi
+
+set -e
+
+# shellcheck disable=SC1091
+source "$(dirname "$0")/../../../devops/scripts/mgmtstorage_enable_public_access.sh"
+
+# Initialise state for Terraform
+terraform init -input=false -backend=true -reconfigure -upgrade \
+    -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \
+    -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \
+    -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \
+    -backend-config="key=${TRE_ID}"
+
+# Import a resource if it exists in Azure but doesn't exist in Terraform
+tf_state_list="$(terraform state list)"
+function import_if_exists() {
+  ADDRESS=$1
+  ID=$2
+  CMD=$3
+
+  # Check if the resource exists in Terraform
+  echo "Checking if ${ADDRESS} exists in Terraform state..."
+  ESCAPED_ADDRESS=$(printf '%q' "${ADDRESS}")
+  TF_RESOURCE_EXISTS=$(echo "$tf_state_list" | grep -q ^"${ESCAPED_ADDRESS}"$; echo $?)
+
+  if [[ ${TF_RESOURCE_EXISTS} -eq 0 ]]; then
+    echo "${ADDRESS} already in TF State, ignoring..."
+    return
+  fi
+
+  # Some resources, e.g. Firewall rules and Diagnostics, don't show up in `az resource show`,
+  # so we need a way to set up a custom command for them
+  if [[ -z ${CMD} ]]; then
+    CMD="az resource show --ids ${ID}"
+  fi
+  ${CMD} > /dev/null
+  AZ_RESOURCE_EXISTS=$?
+
+
+
+  # If resource exists in Terraform, it's already managed -- don't do anything
+  # If resource doesn't exist in Terraform and doesn't exist in Azure, it will be created -- don't do anything
+  # If resource doesn't exist in Terraform but exist in Azure, we need to import it
+  if [[ ${TF_RESOURCE_EXISTS} -ne 0 && ${AZ_RESOURCE_EXISTS} -eq 0 ]]; then
+    echo "IMPORTING ${ADDRESS} ${ID}"
+    terraform import -var "tre_id=${TRE_ID}" -var "location=${LOCATION}" "${ADDRESS}" "${ID}"
+  fi
+}
+
+# Firewall
+import_if_exists module.firewall.azurerm_firewall.fw "/subscriptions/${ARM_SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_ID}/providers/Microsoft.Network/azureFirewalls/fw-${TRE_ID}"
+
+# Firewall IPs
+if [[ "${FIREWALL_SKU}" == "Basic" ]]; then
+  import_if_exists module.firewall.azurerm_public_ip.fwmanagement[0] "/subscriptions/${ARM_SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_ID}/providers/Microsoft.Network/publicIPAddresses/pip-fw-management-${TRE_ID}"
+fi
+
+import_if_exists module.firewall.azurerm_public_ip.fwtransit[0] "/subscriptions/${ARM_SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_ID}/providers/Microsoft.Network/publicIPAddresses/pip-fw-${TRE_ID}"
+
+# Firewall policy
+import_if_exists module.firewall.azurerm_firewall_policy.root "/subscriptions/${ARM_SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_ID}/providers/Microsoft.Network/firewallPolicies/fw-policy-${TRE_ID}"
+import_if_exists module.firewall.azurerm_firewall_policy_rule_collection_group.core \
+  "/subscriptions/${ARM_SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_ID}/providers/Microsoft.Network/firewallPolicies/fw-policy-${TRE_ID}/ruleCollectionGroups/rcg-core"
+
+
+# Diagnostic settings
+import_if_exists module.firewall.azurerm_monitor_diagnostic_setting.firewall \
+  "/subscriptions/${ARM_SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_ID}/providers/Microsoft.Network/azureFirewalls/fw-${TRE_ID}|diagnostics-fw-${TRE_ID}" \
+  "az monitor diagnostic-settings show --resource /subscriptions/${ARM_SUBSCRIPTION_ID}/resourceGroups/rg-${TRE_ID}/providers/microsoft.network/azureFirewalls/fw-${TRE_ID} --name diagnostics-fw-${TRE_ID}"
+
+# Route tables
+import_if_exists azurerm_route_table.rt \
+  "/subscriptions/${ARM_SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_ID}/providers/Microsoft.Network/routeTables/rt-${TRE_ID}"
+
+# import_if_exists azurerm_subnet_route_table_association.rt_shared_subnet_association \
+#   "/subscriptions/${ARM_SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_ID}/providers/Microsoft.Network/virtualNetworks/vnet-${TRE_ID}/subnets/SharedSubnet"
+
+# import_if_exists azurerm_subnet_route_table_association.rt_resource_processor_subnet_association \
+#   "/subscriptions/${ARM_SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_ID}/providers/Microsoft.Network/virtualNetworks/vnet-${TRE_ID}/subnets/ResourceProcessorSubnet"
+
+# import_if_exists azurerm_subnet_route_table_association.rt_web_app_subnet_association \
+#   "/subscriptions/${ARM_SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_ID}/providers/Microsoft.Network/virtualNetworks/vnet-${TRE_ID}/subnets/WebAppSubnet"
+
+# import_if_exists azurerm_subnet_route_table_association.rt_airlock_processor_subnet_association \
+#   "/subscriptions/${ARM_SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_ID}/providers/Microsoft.Network/virtualNetworks/vnet-${TRE_ID}/subnets/AirlockProcessorSubnet"
+
+# import_if_exists azurerm_subnet_route_table_association.rt_airlock_storage_subnet_association \
+#   "/subscriptions/${ARM_SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_ID}/providers/Microsoft.Network/virtualNetworks/vnet-${TRE_ID}/subnets/AirlockStorageSubnet"
+
+# import_if_exists azurerm_subnet_route_table_association.rt_airlock_events_subnet_association \
+#   "/subscriptions/${ARM_SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_ID}/providers/Microsoft.Network/virtualNetworks/vnet-${TRE_ID}/subnets/AirlockEventsSubnet"

core/terraform/firewall/locals.tf

@@ -0,0 +1,24 @@
+locals {
+  core_resource_group_name = "rg-${var.tre_id}"
+  firewall_name            = "fw-${var.tre_id}"
+  firewall_diagnostic_categories_enabled = [
+    "AZFWApplicationRule",
+    "AZFWApplicationRuleAggregation",
+    "AZFWDnsProxy",
+    "AZFWDnsQuery",
+    "AZFWFatFlow",
+    "AZFWFlowTrace",
+    "AZFWIdpsSignature",
+    "AZFWInternalFqdnResolutionFailure",
+    "AZFWNatRule",
+    "AZFWNatRuleAggregation",
+    "AZFWNetworkRule",
+    "AZFWNetworkRuleAggregation",
+    "AZFWThreatIntel"
+  ]
+
+  firewall_policy_name = "fw-policy-${var.tre_id}"
+
+  default_firewall_sku   = "Standard"
+  effective_firewall_sku = coalesce(var.firewall_sku, local.default_firewall_sku)
+}

core/terraform/firewall/main.tf

@@ -0,0 +1,9 @@
+terraform {
+  # In modules we should only specify the min version
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = ">= 3.117"
+    }
+  }
+}

core/terraform/firewall/outputs.tf

@@ -0,0 +1,3 @@
+output "private_ip_address" {
+  value = azurerm_firewall.fw.ip_configuration[0].private_ip_address
+}

templates/shared_services/firewall/terraform/remove_state.sh → core/terraform/firewall/remove_state.sh

@@ -8,16 +8,20 @@ echo "REMOVING STATE FOR FIREWALL..."
 
 set -e
 
+# shellcheck disable=SC1091
+source "$(dirname "$0")/../../../devops/scripts/mgmtstorage_enable_public_access.sh"
+
 terraform init -input=false -backend=true -reconfigure -upgrade \
     -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \
     -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \
     -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \
-    -backend-config="key=${TRE_ID}"
+    -backend-config="key=${TRE_ID}-shared-service-firewall"
 
 tf_state_list="$(terraform state list)"
 function remove_if_present() {
   echo -n "Checking $1 ..."
-  found=$(echo "$tf_state_list" | grep -q ^"$1"$; echo $?)
+  ESCAPED_ADDRESS=$(printf '%q' "${1}")
+  found=$(echo "$tf_state_list" | grep -q ^"$ESCAPED_ADDRESS"$; echo $?)
 
   if [[ $found -eq 0 ]]; then
     echo " removing"
@@ -27,17 +31,23 @@ function remove_if_present() {
   fi
 }
 
+# routetable.tf
 remove_if_present azurerm_route_table.rt
-remove_if_present azurerm_subnet_route_table_association.rt_resource_processor_subnet_association
 remove_if_present azurerm_subnet_route_table_association.rt_shared_subnet_association
+remove_if_present azurerm_subnet_route_table_association.rt_resource_processor_subnet_association
 remove_if_present azurerm_subnet_route_table_association.rt_web_app_subnet_association
-remove_if_present module.firewall
-remove_if_present module.firewall.azurerm_public_ip.fwpip
-remove_if_present module.firewall.azurerm_monitor_diagnostic_setting.firewall
-remove_if_present module.firewall.azurerm_firewall_network_rule_collection.web_app_subnet
-remove_if_present module.firewall.azurerm_firewall_network_rule_collection.resource_processor_subnet
-remove_if_present module.firewall.azurerm_firewall_network_rule_collection.general
-remove_if_present module.firewall.azurerm_firewall_application_rule_collection.web_app_subnet
-remove_if_present module.firewall.azurerm_firewall_application_rule_collection.shared_subnet
-remove_if_present module.firewall.azurerm_firewall_application_rule_collection.resource_processor_subnet
-remove_if_present module.firewall.azurerm_firewall.fw
+remove_if_present azurerm_subnet_route_table_association.rt_airlock_processor_subnet_association
+remove_if_present azurerm_subnet_route_table_association.rt_airlock_storage_subnet_association
+remove_if_present azurerm_subnet_route_table_association.rt_airlock_events_subnet_association
+
+# rules.tf
+remove_if_present azurerm_firewall_network_rule_collection.core
+
+# firewall.tf
+remove_if_present azurerm_public_ip.fwtransit[0]
+remove_if_present azurerm_public_ip.fwmanagement[0]
+remove_if_present azurerm_firewall.fw
+remove_if_present azurerm_monitor_diagnostic_categories.firewall
+remove_if_present azurerm_monitor_diagnostic_setting.firewall
+remove_if_present azurerm_firewall_policy_rule_collection_group.core
+remove_if_present azurerm_firewall_policy.root