Skip to content

Commit 9081b11

Browse files
wanlonghenrywanlonghenry
authored
Longw/high scale terraform (#1488)
* terraform template for high log scale and networkflow logs
1 parent 78b2aab commit 9081b11

File tree

6 files changed

+112
-26
lines changed

6 files changed

+112
-26
lines changed

‎scripts/onboarding/aks/onboarding-msi-terraform-syslog/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,6 @@ If you are deploying a new AKS cluster using Terraform with ama logs addon enabl
88

99
**NOTE**
1010
- Please edit the main.tf file appropriately before running the terraform template
11+
- If resource group already exists, please run `terraform import azurerm_resource_group.rg /subscriptions/<Subscription_ID>/resourceGroups/<Resource_Group_Name>` before terraform plan
1112
- Data will start flowing after 10 minutes since the cluster needs to be ready first
1213
- Workspace ID needs to match format '/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.OperationalInsights/workspaces/workspaceValue'
13-
- If resource group already exists, please run `terraform import azurerm_resource_group.rg /subscriptions/<Subscription_ID>/resourceGroups/<Resource_Group_Name>` before terraform plan

‎scripts/onboarding/aks/onboarding-msi-terraform-syslog/main.tf‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ resource "azurerm_resource_group" "rg" {
55

66
resource "azurerm_kubernetes_cluster" "k8s" {
77
name = var.cluster_name
8-
location = azurerm_resource_group.rg.location
8+
location = var.cluster_location
99
resource_group_name = azurerm_resource_group.rg.name
1010
dns_prefix = var.dns_prefix
1111

@@ -27,6 +27,22 @@ resource "azurerm_kubernetes_cluster" "k8s" {
2727
}
2828
}
2929

30+
locals {
31+
enable_high_log_scale_mode = contains(var.streams, "Microsoft-ContainerLogV2-HighScale")
32+
ingestion_dce_name_full = "MSCI-ingest-${var.workspace_region}-${var.cluster_name}"
33+
ingestion_dce_name_trimmed = substr(local.ingestion_dce_name_full, 0, 43)
34+
ingestion_dce_name = endswith(local.ingestion_dce_name_trimmed, "-") ? substr(local.ingestion_dce_name_trimmed, 0, 42) : local.ingestion_dce_name_trimmed
35+
}
36+
37+
resource "azurerm_monitor_data_collection_endpoint" "ingestion_dce" {
38+
count = local.enable_high_log_scale_mode ? 1 : 0
39+
name = local.ingestion_dce_name
40+
resource_group_name = azurerm_resource_group.rg.name
41+
location = var.workspace_region
42+
kind = "Linux"
43+
tags = var.resource_tag_values
44+
}
45+
3046
resource "azurerm_monitor_data_collection_rule" "dcr" {
3147
name = "MSCI-${var.workspace_region}-${var.cluster_name}"
3248
resource_group_name = azurerm_resource_group.rg.name
@@ -72,6 +88,8 @@ resource "azurerm_monitor_data_collection_rule" "dcr" {
7288
}
7389
}
7490

91+
data_collection_endpoint_id = local.enable_high_log_scale_mode ? azurerm_monitor_data_collection_endpoint.ingestion_dce[0].id : null
92+
7593
description = "DCR for Azure Monitor Container Insights"
7694
}
7795

‎scripts/onboarding/aks/onboarding-msi-terraform-syslog/variables.tf‎

Lines changed: 37 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,82 +1,107 @@
11
variable "agent_count" {
2+
type = number
23
default = 3
34
}
45

56
variable "vm_size" {
6-
type = string
7+
type = string
78
default = "Standard_D2_v2"
89
}
910

1011
variable "identity_type" {
11-
type = string
12+
type = string
1213
default = "SystemAssigned"
1314
}
1415

1516
variable "aks_resource_group_name" {
16-
type = string
17+
type = string
1718
default = "<ResourceGroup>"
1819
}
1920

2021
variable "resource_group_location" {
21-
type = string
22+
type = string
2223
default = "<ResourceGroupLocation>"
2324
}
2425

2526
variable "cluster_name" {
26-
type = string
27+
type = string
2728
default = "<ClusterName>"
2829
}
2930

31+
variable "cluster_location" {
32+
type = string
33+
default = "<ClusterLocation>"
34+
}
35+
3036
variable "dns_prefix" {
37+
type = string
3138
default = "k8stest"
3239
}
3340

3441
variable "workspace_resource_id" {
35-
type = string
42+
type = string
3643
default = "/subscriptions/<SubscriptionId>/resourceGroups/<ResourceGroup>/providers/Microsoft.OperationalInsights/workspaces/<workspaceName>"
3744
}
3845

3946
variable "workspace_region" {
40-
type = string
47+
type = string
4148
default = "<workspaceRegion>"
4249
}
4350

4451
variable "syslog_levels" {
45-
type = list(string)
52+
type = list(string)
4653
default = ["Debug", "Info", "Notice", "Warning", "Error", "Critical", "Alert", "Emergency"]
4754
}
4855

4956
variable "syslog_facilities" {
50-
type = list(string)
57+
type = list(string)
5158
default = ["auth", "authpriv", "cron", "daemon", "mark", "kern", "local0", "local1", "local2", "local3", "local4", "local5", "local6", "local7", "lpr", "mail", "news", "syslog", "user", "uucp"]
5259
}
5360

5461
variable "resource_tag_values" {
5562
description = "Resource Tag Values"
56-
type = map(string)
57-
default = {
63+
type = map(string)
64+
default = {
5865
"<existingOrnew-tag-name1>" = "<existingOrnew-tag-value1>"
5966
"<existingOrnew-tag-name2>" = "<existingOrnew-tag-value2>"
6067
"<existingOrnew-tag-name3>" = "<existingOrnew-tag-value3>"
6168
}
6269
}
6370

6471
variable "data_collection_interval" {
72+
type = string
6573
default = "1m"
6674
}
6775

6876
variable "namespace_filtering_mode_for_data_collection" {
77+
type = string
6978
default = "Off"
7079
}
7180

7281
variable "namespaces_for_data_collection" {
82+
type = list(string)
7383
default = ["kube-system", "gatekeeper-system", "azure-arc"]
7484
}
7585

7686
variable "enableContainerLogV2" {
87+
type = bool
7788
default = true
7889
}
7990

8091
variable "streams" {
81-
default = ["Microsoft-ContainerLog", "Microsoft-ContainerLogV2", "Microsoft-KubeEvents", "Microsoft-KubePodInventory", "Microsoft-KubeNodeInventory", "Microsoft-KubePVInventory","Microsoft-KubeServices", "Microsoft-KubeMonAgentEvents", "Microsoft-InsightsMetrics", "Microsoft-ContainerInventory", "Microsoft-ContainerNodeInventory", "Microsoft-Perf"]
92+
type = list(string)
93+
default = [
94+
"Microsoft-ContainerLog",
95+
"Microsoft-ContainerLogV2",
96+
"Microsoft-KubeEvents",
97+
"Microsoft-KubePodInventory",
98+
"Microsoft-KubeNodeInventory",
99+
"Microsoft-KubePVInventory",
100+
"Microsoft-KubeServices",
101+
"Microsoft-KubeMonAgentEvents",
102+
"Microsoft-InsightsMetrics",
103+
"Microsoft-ContainerInventory",
104+
"Microsoft-ContainerNodeInventory",
105+
"Microsoft-Perf"
106+
]
82107
}

‎scripts/onboarding/aks/onboarding-msi-terraform/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,6 @@ If you are deploying a new AKS cluster using Terraform with ama logs addon enabl
88

99
**NOTE**
1010
- Please edit the main.tf file appropriately before running the terraform template
11+
- If resource group already exists, please run `terraform import azurerm_resource_group.rg /subscriptions/<Subscription_ID>/resourceGroups/<Resource_Group_Name>` before terraform plan
1112
- Data will start flowing after 10 minutes since the cluster needs to be ready first
1213
- Workspace ID needs to match format '/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.OperationalInsights/workspaces/workspaceValue'
13-
- If resource group already exists, please run `terraform import azurerm_resource_group.rg /subscriptions/<Subscription_ID>/resourceGroups/<Resource_Group_Name>` before terraform plan

‎scripts/onboarding/aks/onboarding-msi-terraform/main.tf‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ resource "azurerm_resource_group" "rg" {
55

66
resource "azurerm_kubernetes_cluster" "k8s" {
77
name = var.cluster_name
8-
location = azurerm_resource_group.rg.location
8+
location = var.cluster_location
99
resource_group_name = azurerm_resource_group.rg.name
1010
dns_prefix = var.dns_prefix
1111

@@ -27,6 +27,22 @@ resource "azurerm_kubernetes_cluster" "k8s" {
2727
}
2828
}
2929

30+
locals {
31+
enable_high_log_scale_mode = contains(var.streams, "Microsoft-ContainerLogV2-HighScale")
32+
ingestion_dce_name_full = "MSCI-ingest-${var.workspace_region}-${var.cluster_name}"
33+
ingestion_dce_name_trimmed = substr(local.ingestion_dce_name_full, 0, 43)
34+
ingestion_dce_name = endswith(local.ingestion_dce_name_trimmed, "-") ? substr(local.ingestion_dce_name_trimmed, 0, 42) : local.ingestion_dce_name_trimmed
35+
}
36+
37+
resource "azurerm_monitor_data_collection_endpoint" "ingestion_dce" {
38+
count = local.enable_high_log_scale_mode ? 1 : 0
39+
name = local.ingestion_dce_name
40+
resource_group_name = azurerm_resource_group.rg.name
41+
location = var.workspace_region
42+
kind = "Linux"
43+
tags = var.resource_tag_values
44+
}
45+
3046
resource "azurerm_monitor_data_collection_rule" "dcr" {
3147
name = "MSCI-${var.workspace_region}-${var.cluster_name}"
3248
resource_group_name = azurerm_resource_group.rg.name
@@ -60,6 +76,8 @@ resource "azurerm_monitor_data_collection_rule" "dcr" {
6076
}
6177
}
6278

79+
data_collection_endpoint_id = local.enable_high_log_scale_mode ? azurerm_monitor_data_collection_endpoint.ingestion_dce[0].id : null
80+
6381
description = "DCR for Azure Monitor Container Insights"
6482
}
6583

‎scripts/onboarding/aks/onboarding-msi-terraform/variables.tf‎

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,72 +1,97 @@
11
variable "agent_count" {
2+
type = number
23
default = 3
34
}
45

56
variable "vm_size" {
6-
type = string
7+
type = string
78
default = "Standard_D2_v2"
89
}
910

1011
variable "identity_type" {
11-
type = string
12+
type = string
1213
default = "SystemAssigned"
1314
}
1415

1516
variable "aks_resource_group_name" {
16-
type = string
17+
type = string
1718
default = "<ResourceGroup>"
1819
}
1920

2021
variable "resource_group_location" {
21-
type = string
22+
type = string
2223
default = "<ResourceGroupLocation>"
2324
}
2425

2526
variable "cluster_name" {
26-
type = string
27+
type = string
2728
default = "<ClusterName>"
2829
}
2930

31+
variable "cluster_location" {
32+
type = string
33+
default = "<ClusterLocation>"
34+
}
35+
3036
variable "dns_prefix" {
37+
type = string
3138
default = "k8stest"
3239
}
3340

3441
variable "workspace_resource_id" {
35-
type = string
42+
type = string
3643
default = "/subscriptions/<SubscriptionId>/resourceGroups/<ResourceGroup>/providers/Microsoft.OperationalInsights/workspaces/<workspaceName>"
3744
}
3845

3946
variable "workspace_region" {
40-
type = string
47+
type = string
4148
default = "<workspaceRegion>"
4249
}
4350

4451
variable "resource_tag_values" {
4552
description = "Resource Tag Values"
46-
type = map(string)
47-
default = {
53+
type = map(string)
54+
default = {
4855
"<existingOrnew-tag-name1>" = "<existingOrnew-tag-value1>"
4956
"<existingOrnew-tag-name2>" = "<existingOrnew-tag-value2>"
5057
"<existingOrnew-tag-name3>" = "<existingOrnew-tag-value3>"
5158
}
5259
}
5360

5461
variable "data_collection_interval" {
62+
type = string
5563
default = "1m"
5664
}
5765

5866
variable "namespace_filtering_mode_for_data_collection" {
67+
type = string
5968
default = "Off"
6069
}
6170

6271
variable "namespaces_for_data_collection" {
72+
type = list(string)
6373
default = ["kube-system", "gatekeeper-system", "azure-arc"]
6474
}
6575

6676
variable "enableContainerLogV2" {
77+
type = bool
6778
default = true
6879
}
6980

7081
variable "streams" {
71-
default = ["Microsoft-ContainerLog", "Microsoft-ContainerLogV2", "Microsoft-KubeEvents", "Microsoft-KubePodInventory", "Microsoft-KubeNodeInventory", "Microsoft-KubePVInventory","Microsoft-KubeServices", "Microsoft-KubeMonAgentEvents", "Microsoft-InsightsMetrics", "Microsoft-ContainerInventory", "Microsoft-ContainerNodeInventory", "Microsoft-Perf"]
82+
type = list(string)
83+
default = [
84+
"Microsoft-ContainerLog",
85+
"Microsoft-ContainerLogV2",
86+
"Microsoft-KubeEvents",
87+
"Microsoft-KubePodInventory",
88+
"Microsoft-KubeNodeInventory",
89+
"Microsoft-KubePVInventory",
90+
"Microsoft-KubeServices",
91+
"Microsoft-KubeMonAgentEvents",
92+
"Microsoft-InsightsMetrics",
93+
"Microsoft-ContainerInventory",
94+
"Microsoft-ContainerNodeInventory",
95+
"Microsoft-Perf"
96+
]
7297
}

0 commit comments

Comments
 (0)