Skip to content

Commit 16bdd6e

Browse files
Dev25Dev25
authored
feat: Add support for setting firewall rules (#470)
* Squash Commits Signed-off-by: Dev <[email protected]> * Fix example Signed-off-by: Dev <[email protected]> * Rename var + update README Signed-off-by: Dev <[email protected]> * Set to false as default Signed-off-by: Dev <[email protected]> * Enable firewall support in shared_vpc example Signed-off-by: Dev <[email protected]> * Remove network datasource and make subnetwork conditional on firewall Signed-off-by: Dev <[email protected]> * Fix attribute error Signed-off-by: Dev <[email protected]>
1 parent 5791ac1 commit 16bdd6e

File tree

46 files changed

+1010
-73
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+1010
-73
lines changed

‎README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ Then perform the following commands on the root folder:
100100

101101
| Name | Description | Type | Default | Required |
102102
|------|-------------|:----:|:-----:|:-----:|
103+
| add\_cluster\_firewall\_rules | Create additional firewall rules | bool | `"false"` | no |
103104
| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
104105
| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
105106
| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `"null"` | no |
@@ -109,6 +110,8 @@ Then perform the following commands on the root folder:
109110
| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
110111
| description | The description of the cluster | string | `""` | no |
111112
| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
113+
| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `<list>` | no |
114+
| firewall\_priority | Priority rule for firewall rules | number | `"1000"` | no |
112115
| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
113116
| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
114117
| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
@@ -219,6 +222,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog
219222
In order to execute this module you must have a Service Account with the
220223
following project roles:
221224
- roles/compute.viewer
225+
- roles/compute.securityAdmin (only required if `add_cluster_firewall_rules` is set to `true`)
222226
- roles/container.clusterAdmin
223227
- roles/container.developer
224228
- roles/iam.serviceAccountAdmin

‎autogen/main/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog
240240
In order to execute this module you must have a Service Account with the
241241
following project roles:
242242
- roles/compute.viewer
243+
- roles/compute.securityAdmin (only required if `add_cluster_firewall_rules` is set to `true`)
243244
- roles/container.clusterAdmin
244245
- roles/container.developer
245246
- roles/iam.serviceAccountAdmin

‎autogen/main/cluster.tf.tmpl‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ resource "google_container_cluster" "primary" {
8282
{% endif %}
8383

8484
default_max_pods_per_node = var.default_max_pods_per_node
85+
8586
{% if beta_cluster %}
8687
enable_binary_authorization = var.enable_binary_authorization
8788
enable_intranode_visibility = var.enable_intranode_visibility
@@ -159,7 +160,7 @@ resource "google_container_cluster" "primary" {
159160
}
160161

161162
dns_cache_config {
162-
enabled = var.dns_cache
163+
enabled = var.dns_cache
163164
}
164165
{% endif %}
165166
}
@@ -171,16 +172,16 @@ resource "google_container_cluster" "primary" {
171172

172173
maintenance_policy {
173174
{% if beta_cluster %}
174-
dynamic "recurring_window"{
175+
dynamic "recurring_window" {
175176
for_each = local.cluster_maintenance_window_is_recurring
176177
content {
177178
start_time = var.maintenance_start_time
178-
end_time = var.maintenance_end_time
179+
end_time = var.maintenance_end_time
179180
recurrence = var.maintenance_recurrence
180181
}
181182
}
182183

183-
dynamic "daily_maintenance_window"{
184+
dynamic "daily_maintenance_window" {
184185
for_each = local.cluster_maintenance_window_is_daily
185186
content {
186187
start_time = var.maintenance_start_time
@@ -352,7 +353,7 @@ resource "google_container_node_pool" "pools" {
352353
{% endif %}
353354
for_each = local.node_pools
354355
{% if update_variant %}
355-
name = {for k, v in random_id.name : k => v.hex}[each.key]
356+
name = { for k, v in random_id.name : k => v.hex }[each.key]
356357
{% else %}
357358
name = each.key
358359
{% endif %}
@@ -396,7 +397,7 @@ resource "google_container_node_pool" "pools" {
396397

397398
{% if beta_cluster %}
398399
upgrade_settings {
399-
max_surge = lookup(each.value, "max_surge", 1)
400+
max_surge = lookup(each.value, "max_surge", 1)
400401
max_unavailable = lookup(each.value, "max_unavailable", 0)
401402
}
402403
{% endif %}
@@ -433,8 +434,8 @@ resource "google_container_node_pool" "pools" {
433434
}
434435
{% endif %}
435436
tags = concat(
436-
lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [],
437-
lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${each.value["name"]}"] : [],
437+
lookup(local.node_pools_tags, "default_values", [true, true])[0] ? [local.cluster_network_tag] : [],
438+
lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["${local.cluster_network_tag}-${each.value["name"]}"] : [],
438439
local.node_pools_tags["all"],
439440
local.node_pools_tags[each.value["name"]],
440441
)

‎autogen/main/firewall.tf.tmpl‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/**
2+
* Copyright 2018 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
{{ autogeneration_note }}
18+
19+
20+
/******************************************
21+
Match the gke-<CLUSTER>-<ID>-all INGRESS
22+
firewall rule created by GKE but for EGRESS
23+
24+
Required for clusters when VPCs enforce
25+
a default-deny egress rule
26+
*****************************************/
27+
resource "google_compute_firewall" "intra_egress" {
28+
count = var.add_cluster_firewall_rules ? 1 : 0
29+
name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-intra-cluster-egress"
30+
description = "Managed by terraform gke module: Allow pods to communicate with each other and the master"
31+
project = local.network_project_id
32+
network = var.network
33+
priority = var.firewall_priority
34+
direction = "EGRESS"
35+
36+
target_tags = [local.cluster_network_tag]
37+
destination_ranges = [
38+
local.cluster_endpoint_for_nodes,
39+
local.cluster_subnet_cidr,
40+
local.cluster_alias_ranges_cidr[var.ip_range_pods],
41+
]
42+
43+
# Allow all possible protocols
44+
allow { protocol = "tcp" }
45+
allow { protocol = "udp" }
46+
allow { protocol = "icmp" }
47+
allow { protocol = "sctp" }
48+
allow { protocol = "esp" }
49+
allow { protocol = "ah" }
50+
51+
depends_on = [
52+
google_container_cluster.primary,
53+
]
54+
}
55+
56+
57+
/******************************************
58+
Allow GKE master to hit non 443 ports for
59+
Webhooks/Admission Controllers
60+
61+
https://github.com/kubernetes/kubernetes/issues/79739
62+
*****************************************/
63+
resource "google_compute_firewall" "master_webhooks" {
64+
count = var.add_cluster_firewall_rules ? 1 : 0
65+
name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-webhooks"
66+
description = "Managed by terraform gke module: Allow master to hit pods for admission controllers/webhooks"
67+
project = local.network_project_id
68+
network = var.network
69+
priority = var.firewall_priority
70+
direction = "INGRESS"
71+
72+
source_ranges = [local.cluster_endpoint_for_nodes]
73+
target_tags = [local.cluster_network_tag]
74+
75+
allow {
76+
protocol = "tcp"
77+
ports = var.firewall_inbound_ports
78+
}
79+
80+
depends_on = [
81+
google_container_cluster.primary,
82+
]
83+
84+
}

‎autogen/main/main.tf.tmpl‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -51,17 +51,17 @@ locals {
5151
node_pools = zipmap(local.node_pool_names, tolist(toset(var.node_pools)))
5252

5353
{% if beta_cluster %}
54-
release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : []
54+
release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : []
5555

5656
autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{
57-
resource_type = "cpu"
58-
minimum = var.cluster_autoscaling.min_cpu_cores
59-
maximum = var.cluster_autoscaling.max_cpu_cores
57+
resource_type = "cpu"
58+
minimum = var.cluster_autoscaling.min_cpu_cores
59+
maximum = var.cluster_autoscaling.max_cpu_cores
6060
}, {
61-
resource_type = "memory"
62-
minimum = var.cluster_autoscaling.min_memory_gb
63-
maximum = var.cluster_autoscaling.max_memory_gb
64-
}] : []
61+
resource_type = "memory"
62+
minimum = var.cluster_autoscaling.min_memory_gb
63+
maximum = var.cluster_autoscaling.max_memory_gb
64+
}] : []
6565

6666
{% endif %}
6767

@@ -74,6 +74,9 @@ locals {
7474
// auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous.
7575
default_auto_upgrade = var.regional ? true : false
7676

77+
cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null
78+
cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {}
79+
7780
cluster_network_policy = var.network_policy ? [{
7881
enabled = true
7982
provider = var.network_policy_provider
@@ -103,10 +106,12 @@ locals {
103106
cluster_output_zones = local.cluster_output_regional_zones
104107

105108
{% if private_cluster %}
106-
cluster_endpoint = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint) : google_container_cluster.primary.endpoint
107-
cluster_peering_name = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config.0.peering_name : null
109+
cluster_endpoint = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint) : google_container_cluster.primary.endpoint
110+
cluster_peering_name = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config.0.peering_name : null
111+
cluster_endpoint_for_nodes = var.master_ipv4_cidr_block
108112
{% else %}
109-
cluster_endpoint = google_container_cluster.primary.endpoint
113+
cluster_endpoint = google_container_cluster.primary.endpoint
114+
cluster_endpoint_for_nodes = "${google_container_cluster.primary.endpoint}/32"
110115
{% endif %}
111116

112117
cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, [])
@@ -144,6 +149,7 @@ locals {
144149
cluster_zones = sort(local.cluster_output_zones)
145150

146151
cluster_name = local.cluster_output_name
152+
cluster_network_tag = "gke-${var.name}"
147153
cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"]
148154
cluster_master_version = local.cluster_output_master_version
149155
cluster_min_master_version = local.cluster_output_min_master_version
@@ -164,14 +170,14 @@ locals {
164170
cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled
165171

166172
cluster_workload_identity_config = var.identity_namespace == null ? [] : var.identity_namespace == "enabled" ? [{
167-
identity_namespace = "${var.project_id}.svc.id.goog"}] : [{identity_namespace = var.identity_namespace
173+
identity_namespace = "${var.project_id}.svc.id.goog" }] : [{ identity_namespace = var.identity_namespace
168174
}]
169175
# /BETA features
170176
{% endif %}
171177

172178
{% if beta_cluster %}
173179
cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : []
174-
cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1]
180+
cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1]
175181
{% endif %}
176182
}
177183

‎autogen/main/networks.tf‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/**
2+
* Copyright 2018 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
{{ autogeneration_note }}
18+
19+
data "google_compute_subnetwork" "gke_subnetwork" {
20+
provider = google
21+
22+
count = var.add_cluster_firewall_rules ? 1 : 0
23+
name = var.subnetwork
24+
region = local.region
25+
project = local.network_project_id
26+
}

‎autogen/main/variables.tf.tmpl‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -420,7 +420,7 @@ variable "enable_binary_authorization" {
420420
}
421421

422422
variable "pod_security_policy_config" {
423-
type = list(object({ enabled = bool }))
423+
type = list(object({ enabled = bool }))
424424
description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created."
425425

426426
default = [{
@@ -482,3 +482,22 @@ variable "enable_shielded_nodes" {
482482
default = true
483483
}
484484
{% endif %}
485+
486+
487+
variable "add_cluster_firewall_rules" {
488+
type = bool
489+
description = "Create additional firewall rules"
490+
default = false
491+
}
492+
493+
variable "firewall_priority" {
494+
type = number
495+
description = "Priority rule for firewall rules"
496+
default = 1000
497+
}
498+
499+
variable "firewall_inbound_ports" {
500+
type = list(string)
501+
description = "List of TCP ports for admission/webhook controllers"
502+
default = ["8443", "9443", "15017"]
503+
}

‎autogen/safer-cluster/variables.tf.tmpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -317,7 +317,7 @@ variable "skip_provisioners" {
317317
}
318318

319319
variable "pod_security_policy_config" {
320-
type = list(object({ enabled = bool }))
320+
type = list(object({ enabled = bool }))
321321
description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created."
322322

323323
default = [{

‎cluster.tf‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ resource "google_container_cluster" "primary" {
5151

5252

5353
default_max_pods_per_node = var.default_max_pods_per_node
54+
5455
dynamic "master_authorized_networks_config" {
5556
for_each = local.master_authorized_networks_config
5657
content {
@@ -182,8 +183,8 @@ resource "google_container_node_pool" "pools" {
182183
},
183184
)
184185
tags = concat(
185-
lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [],
186-
lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${each.value["name"]}"] : [],
186+
lookup(local.node_pools_tags, "default_values", [true, true])[0] ? [local.cluster_network_tag] : [],
187+
lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["${local.cluster_network_tag}-${each.value["name"]}"] : [],
187188
local.node_pools_tags["all"],
188189
local.node_pools_tags[each.value["name"]],
189190
)

‎examples/private_zonal_with_networking/main.tf‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -58,12 +58,8 @@ module "gke" {
5858
region = var.region
5959
zones = slice(var.zones, 0, 1)
6060

61-
// This craziness gets a plain network name from the reference link which is the
62-
// only way to force cluster creation to wait on network creation without a
63-
// depends_on link. Tests use terraform 0.12.6, which does not have regex or regexall
64-
network = reverse(split("/", data.google_compute_subnetwork.subnetwork.network))[0]
65-
66-
subnetwork = data.google_compute_subnetwork.subnetwork.name
61+
network = module.gcp-network.network_name
62+
subnetwork = module.gcp-network.subnets_names[0]
6763
ip_range_pods = var.ip_range_pods_name
6864
ip_range_services = var.ip_range_services_name
6965
create_service_account = true

0 commit comments

Comments
 (0)