Fix issue with regional cluster roll outs causing version skews

thefirstofthe300 · thefirstofthe300 · commit 1d73e9d56ee0 · 2019-03-20T10:31:41.000-07:00
Regional clusters are created using the newest version of GKE that is
available across all zones in which the masters live. When a GKE version
roll out occurs, the available versions across the zones can become
skewed for zonal clusters with version x.y.z-gke.a being the only
available zonal version in one zone but version x.y[+1].z[+1]-gke.a[+1]
being the only zonal version available in another zone.

The Terraform module only checks for the version available in the first
zone returned by a call to the google_compute_zones data resource.

Consequently, the module will fail to create a regional cluster during a
roll out due to the version available in that zone not being available
across all the zones for regional clusters.
diff --git a/auth.tf b/auth.tf
@@ -31,4 +31,4 @@ provider "kubernetes" {
   host                   = "https://${local.cluster_endpoint}"
   token                  = "${data.google_client_config.default.access_token}"
   cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}"
-}
+}
diff --git a/autogen/cluster_regional.tf b/autogen/cluster_regional.tf
@@ -31,7 +31,7 @@ resource "google_container_cluster" "primary" {
 
   network            = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
   subnetwork         = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
-  min_master_version = "${local.kubernetes_version}"
+  min_master_version = "${local.kubernetes_version_regional}"
 
   logging_service    = "${var.logging_service}"
   monitoring_service = "${var.monitoring_service}"
@@ -104,7 +104,7 @@ resource "google_container_node_pool" "pools" {
   project            = "${var.project_id}"
   region             = "${var.region}"
   cluster            = "${var.name}"
-  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
+  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}"
   initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"
 
   autoscaling {
diff --git a/autogen/cluster_zonal.tf b/autogen/cluster_zonal.tf
@@ -31,7 +31,7 @@ resource "google_container_cluster" "zonal_primary" {
 
   network            = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
   subnetwork         = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
-  min_master_version = "${local.kubernetes_version}"
+  min_master_version = "${local.kubernetes_version_zonal}"
 
   logging_service    = "${var.logging_service}"
   monitoring_service = "${var.monitoring_service}"
@@ -104,7 +104,7 @@ resource "google_container_node_pool" "zonal_pools" {
   project            = "${var.project_id}"
   zone               = "${var.zones[0]}"
   cluster            = "${var.name}"
-  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
+  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}"
   initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"
 
   autoscaling {
diff --git a/autogen/main.tf b/autogen/main.tf
@@ -31,10 +31,12 @@ resource "random_shuffle" "available_zones" {
 }
 
 locals {
-  kubernetes_version     = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_node_version}"
-  node_version           = "${var.node_version != "" ? var.node_version : local.kubernetes_version}"
-  custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}"
-  network_project_id     = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"
+  kubernetes_version_regional = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version}"
+  kubernetes_version_zonal    = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version}"
+  node_version_regional       = "${var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional}"
+  node_version_zonal          = "${var.node_version != "" && !var.regional ? var.node_version : local.kubernetes_version_zonal}"
+  custom_kube_dns_config      = "${length(keys(var.stub_domains)) > 0 ? true : false}"
+  network_project_id          = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"
 
   cluster_type = "${var.regional ? "regional" : "zonal"}"
 
@@ -149,7 +151,17 @@ locals {
   Get available container engine versions
  *****************************************/
 data "google_container_engine_versions" "region" {
-  provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}"
-  zone     = "${data.google_compute_zones.available.names[0]}"
+  provider = "google-beta"
+  region   = "${var.region}"
+  project  = "${var.project_id}"
+}
+
+data "google_container_engine_versions" "zone" {
+  provider = "google-beta"
+  // Work around to prevent a lack of zone declaration from causing regional cluster creation from erroring out due to error
+  //
+  //     data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone.
+  //
+  zone     = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}"
   project  = "${var.project_id}"
 }
diff --git a/cluster_regional.tf b/cluster_regional.tf
@@ -31,7 +31,7 @@ resource "google_container_cluster" "primary" {
 
   network            = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
   subnetwork         = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
-  min_master_version = "${local.kubernetes_version}"
+  min_master_version = "${local.kubernetes_version_regional}"
 
   logging_service    = "${var.logging_service}"
   monitoring_service = "${var.monitoring_service}"
@@ -97,7 +97,7 @@ resource "google_container_node_pool" "pools" {
   project            = "${var.project_id}"
   region             = "${var.region}"
   cluster            = "${var.name}"
-  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
+  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}"
   initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"
 
   autoscaling {
diff --git a/cluster_zonal.tf b/cluster_zonal.tf
@@ -31,7 +31,7 @@ resource "google_container_cluster" "zonal_primary" {
 
   network            = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
   subnetwork         = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
-  min_master_version = "${local.kubernetes_version}"
+  min_master_version = "${local.kubernetes_version_zonal}"
 
   logging_service    = "${var.logging_service}"
   monitoring_service = "${var.monitoring_service}"
@@ -97,7 +97,7 @@ resource "google_container_node_pool" "zonal_pools" {
   project            = "${var.project_id}"
   zone               = "${var.zones[0]}"
   cluster            = "${var.name}"
-  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
+  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}"
   initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"
 
   autoscaling {
@@ -107,7 +107,7 @@ resource "google_container_node_pool" "zonal_pools" {
 
   management {
     auto_repair  = "${lookup(var.node_pools[count.index], "auto_repair", true)}"
-    auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", true)}"
+    auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", false)}"
   }
 
   node_config {
diff --git a/examples/deploy_service/main.tf b/examples/deploy_service/main.tf
@@ -24,6 +24,11 @@ provider "google" {
   version     = "~> 1.20"
 }
 
+provider "google-beta" {
+  credentials = "${file(var.credentials_path)}"
+  region      = "${var.region}"
+}
+
 provider "kubernetes" {
   load_config_file       = false
   host                   = "https://${module.gke.endpoint}"
diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf
@@ -24,6 +24,11 @@ provider "google" {
   region      = "${var.region}"
 }
 
+provider "google-beta" {
+  credentials = "${file(var.credentials_path)}"
+  region      = "${var.region}"
+}
+
 module "gke" {
   source                   = "../../"
   project_id               = "${var.project_id}"
@@ -43,6 +48,7 @@ module "gke" {
       min_count       = 1
       max_count       = 2
       service_account = "${var.compute_engine_service_account}"
+      auto_upgrade    = true
     },
     {
       name            = "pool-02"
@@ -53,7 +59,6 @@ module "gke" {
       disk_type       = "pd-standard"
       image_type      = "COS"
       auto_repair     = false
-      auto_upgrade    = false
       service_account = "${var.compute_engine_service_account}"
     },
   ]
diff --git a/examples/shared_vpc/main.tf b/examples/shared_vpc/main.tf
@@ -24,6 +24,11 @@ provider "google" {
   region      = "${var.region}"
 }
 
+provider "google-beta" {
+  credentials = "${file(var.credentials_path)}"
+  region      = "${var.region}"
+}
+
 module "gke" {
   source             = "../../"
   project_id         = "${var.project_id}"
diff --git a/examples/simple_regional/main.tf b/examples/simple_regional/main.tf
@@ -23,6 +23,11 @@ provider "google" {
   region      = "${var.region}"
 }
 
+provider "google-beta" {
+  credentials = "${file(var.credentials_path)}"
+  region      = "${var.region}"
+}
+
 module "gke" {
   source            = "../../"
   project_id        = "${var.project_id}"
diff --git a/examples/simple_zonal/main.tf b/examples/simple_zonal/main.tf
@@ -24,6 +24,11 @@ provider "google" {
   region      = "${var.region}"
 }
 
+provider "google-beta" {
+  credentials = "${file(var.credentials_path)}"
+  region      = "${var.region}"
+}
+
 module "gke" {
   source            = "../../"
   project_id        = "${var.project_id}"
diff --git a/examples/stub_domains/main.tf b/examples/stub_domains/main.tf
@@ -24,6 +24,11 @@ provider "google" {
   region      = "${var.region}"
 }
 
+provider "google-beta" {
+  credentials = "${file(var.credentials_path)}"
+  region      = "${var.region}"
+}
+
 module "gke" {
   source            = "../../"
   project_id        = "${var.project_id}"
diff --git a/main.tf b/main.tf
@@ -31,10 +31,12 @@ resource "random_shuffle" "available_zones" {
 }
 
 locals {
-  kubernetes_version     = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_node_version}"
-  node_version           = "${var.node_version != "" ? var.node_version : local.kubernetes_version}"
-  custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}"
-  network_project_id     = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"
+  kubernetes_version_regional = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version}"
+  kubernetes_version_zonal    = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version}"
+  node_version_regional       = "${var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional}"
+  node_version_zonal          = "${var.node_version != "" && !var.regional ? var.node_version : local.kubernetes_version_zonal}"
+  custom_kube_dns_config      = "${length(keys(var.stub_domains)) > 0 ? true : false}"
+  network_project_id          = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"
 
   cluster_type = "${var.regional ? "regional" : "zonal"}"
 
@@ -149,7 +151,17 @@ locals {
   Get available container engine versions
  *****************************************/
 data "google_container_engine_versions" "region" {
-  provider = "google"
-  zone     = "${data.google_compute_zones.available.names[0]}"
+  provider = "google-beta"
+  region   = "${var.region}"
   project  = "${var.project_id}"
-}
+}
+
+data "google_container_engine_versions" "zone" {
+  // Work around to prevent a lack of zone declaration from causing regional cluster creation from erroring out due to error
+  //
+  //     data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone.
+  //
+  zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}"
+
+  project = "${var.project_id}"
+}
diff --git a/modules/private-cluster/cluster_regional.tf b/modules/private-cluster/cluster_regional.tf
@@ -31,7 +31,7 @@ resource "google_container_cluster" "primary" {
 
   network            = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
   subnetwork         = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
-  min_master_version = "${local.kubernetes_version}"
+  min_master_version = "${local.kubernetes_version_regional}"
 
   logging_service    = "${var.logging_service}"
   monitoring_service = "${var.monitoring_service}"
@@ -102,7 +102,7 @@ resource "google_container_node_pool" "pools" {
   project            = "${var.project_id}"
   region             = "${var.region}"
   cluster            = "${var.name}"
-  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
+  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}"
   initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"
 
   autoscaling {
diff --git a/modules/private-cluster/cluster_zonal.tf b/modules/private-cluster/cluster_zonal.tf
@@ -31,7 +31,7 @@ resource "google_container_cluster" "zonal_primary" {
 
   network            = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
   subnetwork         = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
-  min_master_version = "${local.kubernetes_version}"
+  min_master_version = "${local.kubernetes_version_zonal}"
 
   logging_service    = "${var.logging_service}"
   monitoring_service = "${var.monitoring_service}"
@@ -102,7 +102,7 @@ resource "google_container_node_pool" "zonal_pools" {
   project            = "${var.project_id}"
   zone               = "${var.zones[0]}"
   cluster            = "${var.name}"
-  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
+  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}"
   initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"
 
   autoscaling {
diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf
@@ -31,10 +31,12 @@ resource "random_shuffle" "available_zones" {
 }
 
 locals {
-  kubernetes_version     = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_node_version}"
-  node_version           = "${var.node_version != "" ? var.node_version : local.kubernetes_version}"
-  custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}"
-  network_project_id     = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"
+  kubernetes_version_regional = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version}"
+  kubernetes_version_zonal    = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version}"
+  node_version_regional       = "${var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional}"
+  node_version_zonal          = "${var.node_version != "" && !var.regional ? var.node_version : local.kubernetes_version_zonal}"
+  custom_kube_dns_config      = "${length(keys(var.stub_domains)) > 0 ? true : false}"
+  network_project_id          = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"
 
   cluster_type = "${var.regional ? "regional" : "zonal"}"
 
@@ -150,6 +152,16 @@ locals {
  *****************************************/
 data "google_container_engine_versions" "region" {
   provider = "google-beta"
-  zone     = "${data.google_compute_zones.available.names[0]}"
+  region   = "${var.region}"
   project  = "${var.project_id}"
 }
+
+data "google_container_engine_versions" "zone" {
+  // Work around to prevent a lack of zone declaration from causing regional cluster creation from erroring out due to error
+  //
+  //     data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone.
+  //
+  zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}"
+
+  project = "${var.project_id}"
+}
diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf
@@ -116,7 +116,7 @@ variable "ip_range_pods" {
 }
 
 variable "ip_range_services" {
-  description = "The _name_ of the secondary subnet ip range to use for services"
+  description = "The _name_ of the secondary subnet range to use for services"
 }
 
 variable "remove_default_node_pool" {
diff --git a/variables.tf b/variables.tf
@@ -116,7 +116,7 @@ variable "ip_range_pods" {
 }
 
 variable "ip_range_services" {
-  description = "The _name_ of the secondary subnet ip range to use for services"
+  description = "The _name_ of the secondary subnet range to use for services"
 }
 
 variable "remove_default_node_pool" {
