Longw/high scale terraform

scripts/onboarding/aks/onboarding-msi-terraform-syslog/README.md

@@ -8,6 +8,6 @@ If you are deploying a new AKS cluster using Terraform with ama logs addon enabl
 
 **NOTE**
 - Please edit the main.tf file appropriately before running the terraform template
+- If resource group already exists, please run `terraform import azurerm_resource_group.rg /subscriptions/<Subscription_ID>/resourceGroups/<Resource_Group_Name>` before terraform plan
 - Data will start flowing after 10 minutes since the cluster needs to be ready first
 - Workspace ID needs to match format '/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.OperationalInsights/workspaces/workspaceValue'
-- If resource group already exists, please run `terraform import azurerm_resource_group.rg /subscriptions/<Subscription_ID>/resourceGroups/<Resource_Group_Name>` before terraform plan

scripts/onboarding/aks/onboarding-msi-terraform-syslog/main.tf

@@ -5,7 +5,7 @@ resource "azurerm_resource_group" "rg" {
 
 resource "azurerm_kubernetes_cluster" "k8s" {
   name                = var.cluster_name
-  location            = azurerm_resource_group.rg.location
+  location            = var.cluster_location
   resource_group_name = azurerm_resource_group.rg.name
   dns_prefix          = var.dns_prefix
 
@@ -27,6 +27,22 @@ resource "azurerm_kubernetes_cluster" "k8s" {
   }
 }
 
+locals {
+  enable_high_log_scale_mode = contains(var.streams, "Microsoft-ContainerLogV2-HighScale")
+  ingestion_dce_name_full    = "MSCI-ingest-${var.workspace_region}-${var.cluster_name}"
+  ingestion_dce_name_trimmed = substr(local.ingestion_dce_name_full, 0, 43)
+  ingestion_dce_name         = endswith(local.ingestion_dce_name_trimmed, "-") ? substr(local.ingestion_dce_name_trimmed, 0, 42) : local.ingestion_dce_name_trimmed
+}
+
+resource "azurerm_monitor_data_collection_endpoint" "ingestion_dce" {
+  count               = local.enable_high_log_scale_mode ? 1 : 0
+  name                = local.ingestion_dce_name
+  resource_group_name = azurerm_resource_group.rg.name
+  location            = var.workspace_region
+  kind                = "Linux"
+  tags                = var.resource_tag_values
+}
+
 resource "azurerm_monitor_data_collection_rule" "dcr" {
   name                = "MSCI-${var.workspace_region}-${var.cluster_name}"
   resource_group_name = azurerm_resource_group.rg.name
@@ -72,6 +88,8 @@ resource "azurerm_monitor_data_collection_rule" "dcr" {
     }
   }
 
+  data_collection_endpoint_id = local.enable_high_log_scale_mode ? azurerm_monitor_data_collection_endpoint.ingestion_dce[0].id : null
+
   description = "DCR for Azure Monitor Container Insights"
 }
 

scripts/onboarding/aks/onboarding-msi-terraform-syslog/variables.tf

@@ -1,82 +1,107 @@
 variable "agent_count" {
+  type    = number
   default = 3
 }
 
 variable "vm_size" {
-  type = string
+  type    = string
   default = "Standard_D2_v2"
 }
 
 variable "identity_type" {
-  type = string
+  type    = string
   default = "SystemAssigned"
 }
 
 variable "aks_resource_group_name" {
-  type = string
+  type    = string
   default = "<ResourceGroup>"
 }
 
 variable "resource_group_location" {
-  type = string
+  type    = string
   default = "<ResourceGroupLocation>"
 }
 
 variable "cluster_name" {
-  type = string
+  type    = string
   default = "<ClusterName>"
 }
 
+variable "cluster_location" {
+  type    = string
+  default = "<ClusterLocation>"
+}
+
 variable "dns_prefix" {
+  type    = string
   default = "k8stest"
 }
 
 variable "workspace_resource_id" {
-  type = string
+  type    = string
   default = "/subscriptions/<SubscriptionId>/resourceGroups/<ResourceGroup>/providers/Microsoft.OperationalInsights/workspaces/<workspaceName>"
 }
 
 variable "workspace_region" {
-  type = string
+  type    = string
   default = "<workspaceRegion>"
 }
 
 variable "syslog_levels" {
-  type = list(string)
+  type    = list(string)
   default = ["Debug", "Info", "Notice", "Warning", "Error", "Critical", "Alert", "Emergency"]
 }
 
 variable "syslog_facilities" {
-  type = list(string)
+  type    = list(string)
   default = ["auth", "authpriv", "cron", "daemon", "mark", "kern", "local0", "local1", "local2", "local3", "local4", "local5", "local6", "local7", "lpr", "mail", "news", "syslog", "user", "uucp"]
 }
 
 variable "resource_tag_values" {
   description = "Resource Tag Values"
-  type = map(string)
-  default = {
+  type        = map(string)
+  default     = {
     "<existingOrnew-tag-name1>" = "<existingOrnew-tag-value1>"
     "<existingOrnew-tag-name2>" = "<existingOrnew-tag-value2>"
     "<existingOrnew-tag-name3>" = "<existingOrnew-tag-value3>"
   }
 }
 
 variable "data_collection_interval" {
+  type    = string
   default = "1m"
 }
 
 variable "namespace_filtering_mode_for_data_collection" {
+  type    = string
   default = "Off"
 }
 
 variable "namespaces_for_data_collection" {
+  type    = list(string)
   default = ["kube-system", "gatekeeper-system", "azure-arc"]
 }
 
 variable "enableContainerLogV2" {
+  type    = bool
   default = true
 }
 
 variable "streams" {
- default = ["Microsoft-ContainerLog", "Microsoft-ContainerLogV2", "Microsoft-KubeEvents", "Microsoft-KubePodInventory", "Microsoft-KubeNodeInventory", "Microsoft-KubePVInventory","Microsoft-KubeServices", "Microsoft-KubeMonAgentEvents", "Microsoft-InsightsMetrics", "Microsoft-ContainerInventory",  "Microsoft-ContainerNodeInventory", "Microsoft-Perf"]
+  type    = list(string)
+  default = [
+    "Microsoft-ContainerLog",
+    "Microsoft-ContainerLogV2",
+    "Microsoft-KubeEvents",
+    "Microsoft-KubePodInventory",
+    "Microsoft-KubeNodeInventory",
+    "Microsoft-KubePVInventory",
+    "Microsoft-KubeServices",
+    "Microsoft-KubeMonAgentEvents",
+    "Microsoft-InsightsMetrics",
+    "Microsoft-ContainerInventory",
+    "Microsoft-ContainerNodeInventory",
+    "Microsoft-Perf"
+  ]
 }

scripts/onboarding/aks/onboarding-msi-terraform/README.md

@@ -8,6 +8,6 @@ If you are deploying a new AKS cluster using Terraform with ama logs addon enabl
 
 **NOTE**
 - Please edit the main.tf file appropriately before running the terraform template
+- If resource group already exists, please run `terraform import azurerm_resource_group.rg /subscriptions/<Subscription_ID>/resourceGroups/<Resource_Group_Name>` before terraform plan
 - Data will start flowing after 10 minutes since the cluster needs to be ready first
 - Workspace ID needs to match format '/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.OperationalInsights/workspaces/workspaceValue'
-- If resource group already exists, please run `terraform import azurerm_resource_group.rg /subscriptions/<Subscription_ID>/resourceGroups/<Resource_Group_Name>` before terraform plan

scripts/onboarding/aks/onboarding-msi-terraform/main.tf

@@ -5,7 +5,7 @@ resource "azurerm_resource_group" "rg" {
 
 resource "azurerm_kubernetes_cluster" "k8s" {
   name                = var.cluster_name
-  location            = azurerm_resource_group.rg.location
+  location            = var.cluster_location
   resource_group_name = azurerm_resource_group.rg.name
   dns_prefix          = var.dns_prefix
 
@@ -27,6 +27,22 @@ resource "azurerm_kubernetes_cluster" "k8s" {
   }
 }
 
+locals {
+  enable_high_log_scale_mode = contains(var.streams, "Microsoft-ContainerLogV2-HighScale")
+  ingestion_dce_name_full    = "MSCI-ingest-${var.workspace_region}-${var.cluster_name}"
+  ingestion_dce_name_trimmed = substr(local.ingestion_dce_name_full, 0, 43)
+  ingestion_dce_name         = endswith(local.ingestion_dce_name_trimmed, "-") ? substr(local.ingestion_dce_name_trimmed, 0, 42) : local.ingestion_dce_name_trimmed
+}
+
+resource "azurerm_monitor_data_collection_endpoint" "ingestion_dce" {
+  count               = local.enable_high_log_scale_mode ? 1 : 0
+  name                = local.ingestion_dce_name
+  resource_group_name = azurerm_resource_group.rg.name
+  location            = var.workspace_region
+  kind                = "Linux"
+  tags                = var.resource_tag_values
+}
+
 resource "azurerm_monitor_data_collection_rule" "dcr" {
   name                = "MSCI-${var.workspace_region}-${var.cluster_name}"
   resource_group_name = azurerm_resource_group.rg.name
@@ -60,6 +76,8 @@ resource "azurerm_monitor_data_collection_rule" "dcr" {
     }
   }
 
+  data_collection_endpoint_id = local.enable_high_log_scale_mode ? azurerm_monitor_data_collection_endpoint.ingestion_dce[0].id : null
+
   description = "DCR for Azure Monitor Container Insights"
 }
 

scripts/onboarding/aks/onboarding-msi-terraform/variables.tf

@@ -1,72 +1,97 @@
 variable "agent_count" {
+  type    = number
   default = 3
 }
 
 variable "vm_size" {
-  type = string
+  type    = string
   default = "Standard_D2_v2"
 }
 
 variable "identity_type" {
-  type = string
+  type    = string
   default = "SystemAssigned"
 }
 
 variable "aks_resource_group_name" {
-  type = string
+  type    = string
   default = "<ResourceGroup>"
 }
 
 variable "resource_group_location" {
-  type = string
+  type    = string
   default = "<ResourceGroupLocation>"
 }
 
 variable "cluster_name" {
-  type = string
+  type    = string
   default = "<ClusterName>"
 }
 
+variable "cluster_location" {
+  type    = string
+  default = "<ClusterLocation>"
+}
+
 variable "dns_prefix" {
+  type    = string
   default = "k8stest"
 }
 
 variable "workspace_resource_id" {
-  type = string
+  type    = string
   default = "/subscriptions/<SubscriptionId>/resourceGroups/<ResourceGroup>/providers/Microsoft.OperationalInsights/workspaces/<workspaceName>"
 }
 
 variable "workspace_region" {
-  type = string
+  type    = string
   default = "<workspaceRegion>"
 }
 
 variable "resource_tag_values" {
   description = "Resource Tag Values"
-  type = map(string)
-  default = {
+  type        = map(string)
+  default     = {
     "<existingOrnew-tag-name1>" = "<existingOrnew-tag-value1>"
     "<existingOrnew-tag-name2>" = "<existingOrnew-tag-value2>"
     "<existingOrnew-tag-name3>" = "<existingOrnew-tag-value3>"
   }
 }
 
 variable "data_collection_interval" {
+  type    = string
   default = "1m"
 }
 
 variable "namespace_filtering_mode_for_data_collection" {
+  type    = string
   default = "Off"
 }
 
 variable "namespaces_for_data_collection" {
+  type    = list(string)
   default = ["kube-system", "gatekeeper-system", "azure-arc"]
 }
 
 variable "enableContainerLogV2" {
+  type    = bool
   default = true
 }
 
 variable "streams" {
- default = ["Microsoft-ContainerLog", "Microsoft-ContainerLogV2", "Microsoft-KubeEvents", "Microsoft-KubePodInventory", "Microsoft-KubeNodeInventory", "Microsoft-KubePVInventory","Microsoft-KubeServices", "Microsoft-KubeMonAgentEvents", "Microsoft-InsightsMetrics", "Microsoft-ContainerInventory",  "Microsoft-ContainerNodeInventory", "Microsoft-Perf"]
+  type    = list(string)
+  default = [
+    "Microsoft-ContainerLog",
+    "Microsoft-ContainerLogV2",
+    "Microsoft-KubeEvents",
+    "Microsoft-KubePodInventory",
+    "Microsoft-KubeNodeInventory",
+    "Microsoft-KubePVInventory",
+    "Microsoft-KubeServices",
+    "Microsoft-KubeMonAgentEvents",
+    "Microsoft-InsightsMetrics",
+    "Microsoft-ContainerInventory",
+    "Microsoft-ContainerNodeInventory",
+    "Microsoft-Perf"
+  ]
 }