figured out how to replace an AKS cluster

Author: dak1n1

_examples/aks/README.md

@@ -1,6 +1,6 @@
 # AKS (Azure Kubernetes Service)
 
-This example shows how to use the Terraform Kubernetes Provider and Terraform Helm Provider to configure an AKS cluster. The example builds the AKS cluster and applies the Kubernetes configurations in a single operation.
+This example shows how to use the Terraform Kubernetes Provider and Terraform Helm Provider to configure an AKS cluster. The example config in this directory builds the AKS cluster and applies the Kubernetes configurations in a single operation. This guide will also show you how to make changes to the underlying AKS cluster in such a way that Kuberntes/Helm resources are recreated after the underlying cluster is replaced.
 
 You will need the following environment variables to be set:
 
@@ -11,7 +11,7 @@ You will need the following environment variables to be set:
 
 See [AWS Provider docs](https://www.terraform.io/docs/providers/aws/index.html#configuration-reference) for more details about these variables and alternatives, like `AWS_PROFILE`.
 
-To install the EKS cluster using default values, run terraform init and apply from the directory containing this README.
+To install the AKS cluster using default values, run terraform init and apply from the directory containing this README.
 
 ```
 terraform init
@@ -20,21 +20,32 @@ terraform apply
 
 ## Kubeconfig for manual CLI access
 
-This example generates a kubeconfig file in the current working directory. However, the token in this config expires in 15 minutes. The token can be refreshed by running `terraform apply` again. Export the KUBECONFIG to manually access the cluster:
+This example generates a kubeconfig file in the current working directory, which can be used for manual CLI access to the cluster.
 
 ```
-terraform apply
 export KUBECONFIG=$(terraform output kubeconfig_path|jq -r)
 kubectl get pods -n test
 ```
 
-## Optional variables
-
-The Kubernetes version can be specified at apply time:
+However, in a real-world scenario, this config file would have to be replaced periodically as the AKS client certificates eventually expire (see the [Azure documentation](https://docs.microsoft.com/en-us/azure/aks/certificate-rotation) for the exact expiry dates). If the certificates are replaced, the AKS module will have to be targeted to pull in the new credentials before they can be passed into the Kubernetes or Helm providers.
 
 ```
-terraform apply -var=kubernetes_version=1.18
+terraform state rm module.kubernetes-config
+terraform plan
+terraform apply
+export KUBECONFIG=$(terraform output kubeconfig_path|jq -r)
+kubectl get pods -n test
 ```
 
-See https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html for currently available versions.
+This approach prevents the Kubernetes and Helm provider from using cached, invalid credentials, which would cause provider configuration errors durring the plan and apply phases. (The resources that were previously deployed will not be affected by the `state rm`).
+
+## Replacing the AKS cluster, or its authentication credentials
 
+When the cluster is initially created, the Kubernetes and Helm providers will not be initialized until authentication details are created for the cluster. However, for future operations that may involve replacing the underlying cluster (for example, changing VM sizes), the AKS cluster will have to be targeted without the Kubernetes/Helm providers, as shown below. This is done by removing the `module.kubernetes-config` from Terraform State prior to replacing cluster credentials, to avoid passing outdated credentials into the providers.
+
+This will create the new cluster and the Kubernetes resources in a single apply. If this is being applied to an existing cluster (such as in the case of credential rotation), the existing Kubernetes/Helm resources will continue running and simply undergo a credential refresh.
+
+```
+terraform state rm module.kubernetes-config
+terraform apply
+```

_examples/aks/aks-cluster/main.tf

@@ -1,7 +1,3 @@
-provider "azurerm" {
-  features {}
-}
-
 resource "azurerm_resource_group" "test" {
   name     = var.cluster_name
   location = var.location
@@ -16,7 +12,8 @@ resource "azurerm_kubernetes_cluster" "test" {
   default_node_pool {
     name       = "default"
     node_count = 1
-    vm_size    = "Standard_DS2_v2"
+    #vm_size    = "Standard_DS2_v2"
+    vm_size    = "Standard_A2_v2"
   }
 
   identity {

_examples/aks/aks-cluster/output.tf

@@ -1,7 +1,19 @@
-output "cluster_id" {
-  value = azurerm_kubernetes_cluster.test.id
+output "client_cert" {
+  value = azurerm_kubernetes_cluster.test.kube_config.0.client_certificate
+}
+
+output "client_key" {
+  value = azurerm_kubernetes_cluster.test.kube_config.0.client_key
+}
+
+output "ca_cert" {
+  value = azurerm_kubernetes_cluster.test.kube_config.0.cluster_ca_certificate
 }
 
 output "data_disk_uri" {
   value = azurerm_managed_disk.test.id
 }
+
+output "endpoint" {
+  value = azurerm_kubernetes_cluster.test.kube_config.0.host
+}

_examples/aks/kubernetes-config/main.tf

@@ -1,54 +1,10 @@
-provider "azurerm" {
-  features {}
-}
-
-# The client certificate used for authenticating into the AKS cluster will eventually expire,
-# (especially true if your clusters are created and destroyed periodically).
-# This data source fetches new authentication certificates.
-# Alternatively, use `terraform refresh` to fetch them manually.
-data "azurerm_kubernetes_cluster" "main" {
-depends_on = [var.cluster_id]
-  name                = var.cluster_name
-  resource_group_name = var.cluster_name
-}
-
-provider "kubernetes" {
-  host                   = data.azurerm_kubernetes_cluster.main.kube_config.0.host
-  client_key             = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_key)
-  client_certificate     = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_certificate)
-  cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.cluster_ca_certificate)
-}
-
 resource "kubernetes_namespace" "test" {
-depends_on = [var.cluster_id]
   metadata {
     name = "test"
   }
 }
 
-resource "kubernetes_persistent_volume" "test" {
-depends_on = [var.cluster_id]
-  metadata {
-    name = "test"
-  }
-  spec {
-    capacity = {
-      storage = "1Gi"
-    }
-    access_modes = ["ReadWriteOnce"]
-    persistent_volume_source {
-      azure_disk {
-        caching_mode = "None"
-        data_disk_uri = var.data_disk_uri
-        disk_name = "managed"
-        kind = "Managed"
-      }
-    }
-  }
-}
-
 resource "kubernetes_deployment" "test" {
-depends_on = [var.cluster_id]
   metadata {
     name = "test"
     namespace= kubernetes_namespace.test.metadata.0.name
@@ -67,6 +23,7 @@ depends_on = [var.cluster_id]
         }
       }
       spec {
+        automount_service_account_token  = false
         container {
           image = "nginx:1.19.4"
           name  = "tf-acc-test"
@@ -87,17 +44,7 @@ depends_on = [var.cluster_id]
   }
 }
 
-provider "helm" {
-  kubernetes {
-    host                   = data.azurerm_kubernetes_cluster.main.kube_config.0.host
-    client_key             = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_key)
-    client_certificate     = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_certificate)
-    cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.cluster_ca_certificate)
-  }
-}
-
 resource helm_release nginx_ingress {
-depends_on = [var.cluster_id]
   name       = "nginx-ingress-controller"
 
   repository = "https://charts.bitnami.com/bitnami"

_examples/aks/kubernetes-config/variables.tf

@@ -1,7 +1,7 @@
-variable "cluster_id" {
-  type = string
-}
-
+#variable "client_cert" {
+#  type = string
+#}
+#
 variable "cluster_name" {
   type = string
 }

_examples/aks/main.tf

@@ -15,19 +15,37 @@ terraform {
   }
 }
 
-resource "random_id" "cluster_name" {
-  byte_length = 5
+provider "kubernetes" {
+  host                   = module.aks-cluster.endpoint
+  client_key             = base64decode(module.aks-cluster.client_key)
+  client_certificate     = base64decode(module.aks-cluster.client_cert)
+  cluster_ca_certificate = base64decode(module.aks-cluster.ca_cert)
+}
+
+provider "helm" {
+  kubernetes {
+    host                   = module.aks-cluster.endpoint
+    client_key             = base64decode(module.aks-cluster.client_key)
+    client_certificate     = base64decode(module.aks-cluster.client_cert)
+    cluster_ca_certificate = base64decode(module.aks-cluster.ca_cert)
+  }
+}
+
+provider "azurerm" {
+  features {}
 }
 
 module "aks-cluster" {
+  providers               = { azurerm = azurerm }
   source                  = "./aks-cluster"
   cluster_name            = local.cluster_name
   location                = var.location
 }
 
 module "kubernetes-config" {
+  providers               = { kubernetes = kubernetes, helm = helm }
+  depends_on              = [module.aks-cluster]
   source                  = "./kubernetes-config"
-  cluster_id              = module.aks-cluster.cluster_id # creates dependency on cluster creation
   cluster_name            = local.cluster_name
   data_disk_uri           = module.aks-cluster.data_disk_uri
 }

_examples/aks/variables.tf

@@ -3,6 +3,11 @@ variable "location" {
   default = "westus2"
 }
 
+resource "random_id" "cluster_name" {
+  byte_length = 5
+}
+
 locals {
-  cluster_name = "tf-k8s-${random_id.cluster_name.hex}"
+  cluster_name                = "tf-k8s-${random_id.cluster_name.hex}"
+  cluster_credentials_updated = timestamp()
 }