Add updated examples for AKS, GKE, EKS (#1115)

Add some examples of using cloud providers with version 2 of the Kubernetes and Helm providers.

Author: dak1n1

GNUmakefile

@@ -33,6 +33,18 @@ depscheck:
 	@git diff --exit-code -- vendor || \
 		(echo; echo "Unexpected difference in vendor/ directory. Run 'go mod vendor' command or revert any go.mod/go.sum/vendor changes and commit."; exit 1)
 
+examples-lint: tools
+	@echo "==> Checking _examples dir formatting..."
+	@./scripts/fmt-examples.sh || (echo; \
+		echo "Terraform formatting errors found in _examples dir."; \
+		echo "To see the full differences, run: ./scripts/fmt-examples.sh diff"; \
+		echo "To automatically fix the formatting, run 'make examples-lint-fix' and commit the changes."; \
+		exit 1)
+
+examples-lint-fix: tools
+	@echo "==> Fixing terraform formatting of _examples dir..."
+	@./scripts/fmt-examples.sh fix
+
 fmt:
 	gofmt -w $(GOFMT_FILES)
 

_examples/aks/README.md

@@ -0,0 +1,60 @@
+# AKS (Azure Kubernetes Service)
+
+This example shows how to use the Terraform Kubernetes Provider and Terraform Helm Provider to configure an AKS cluster. The example config in this directory builds the AKS cluster and applies the Kubernetes configurations in a single operation. This guide will also show you how to make changes to the underlying AKS cluster in such a way that Kuberntes/Helm resources are recreated after the underlying cluster is replaced.
+
+You will need the following environment variables to be set:
+
+  - `ARM_SUBSCRIPTION_ID`
+  - `ARM_TENANT_ID`
+  - `ARM_CLIENT_ID`
+  - `ARM_CLIENT_SECRET`
+
+Ensure that `KUBE_CONFIG_FILE` and `KUBE_CONFIG_FILES` environment variables are NOT set, as they will interfere with the cluster build.
+
+```
+unset KUBE_CONFIG_FILE
+unset KUBE_CONFIG_FILES
+```
+
+To install the AKS cluster using default values, run terraform init and apply from the directory containing this README.
+
+```
+terraform init
+terraform apply
+```
+
+## Kubeconfig for manual CLI access
+
+This example generates a kubeconfig file in the current working directory, which can be used for manual CLI access to the cluster.
+
+```
+export KUBECONFIG=$(terraform output -raw kubeconfig_path)
+kubectl get pods -n test
+```
+
+However, in a real-world scenario, this config file would have to be replaced periodically as the AKS client certificates eventually expire (see the [Azure documentation](https://docs.microsoft.com/en-us/azure/aks/certificate-rotation) for the exact expiry dates). If the certificates (or other authentication attributes) are replaced, run a targeted `terraform apply` to save the new credentials into state.
+
+```
+terraform plan -target=module.aks-cluster
+terraform apply -target=module.aks-cluster
+```
+
+Once the targeted apply is finished, the Kubernetes and Helm providers will be available for use again. Run `terraform apply` again (without targeting) to apply any updates to Kubernetes resources.
+
+```
+terraform plan
+terraform apply
+```
+
+This approach prevents the Kubernetes and Helm providers from attempting to use cached, invalid credentials, which would cause provider configuration errors durring the plan and apply phases.
+
+## Replacing the AKS cluster and re-creating the Kubernetes / Helm resources
+
+When the cluster is initially created, the Kubernetes and Helm providers will not be initialized until authentication details are created for the cluster. However, for future operations that may involve replacing the underlying cluster (for example, changing VM sizes), the AKS cluster will have to be targeted without the Kubernetes/Helm providers, as shown below. This is done by removing the `module.kubernetes-config` from Terraform State prior to replacing cluster credentials, to avoid passing outdated credentials into the providers.
+
+This will create the new cluster and the Kubernetes resources in a single apply.
+
+```
+terraform state rm module.kubernetes-config
+terraform apply
+```

_examples/aks/aks-cluster/main.tf

@@ -0,0 +1,27 @@
+resource "azurerm_resource_group" "test" {
+  name     = var.cluster_name
+  location = var.location
+}
+
+resource "azurerm_kubernetes_cluster" "test" {
+  name                = var.cluster_name
+  location            = azurerm_resource_group.test.location
+  resource_group_name = azurerm_resource_group.test.name
+  dns_prefix          = var.cluster_name
+
+  default_node_pool {
+    name       = "default"
+    node_count = 1
+    vm_size    = "Standard_DS2_v2"
+  }
+
+  identity {
+    type = "SystemAssigned"
+  }
+}
+
+resource "local_file" "kubeconfig" {
+  content = azurerm_kubernetes_cluster.test.kube_config_raw
+  filename = "${path.root}/kubeconfig"
+}
+

_examples/aks/aks-cluster/output.tf

@@ -0,0 +1,15 @@
+output "client_cert" {
+  value = azurerm_kubernetes_cluster.test.kube_config.0.client_certificate
+}
+
+output "client_key" {
+  value = azurerm_kubernetes_cluster.test.kube_config.0.client_key
+}
+
+output "ca_cert" {
+  value = azurerm_kubernetes_cluster.test.kube_config.0.cluster_ca_certificate
+}
+
+output "endpoint" {
+  value = azurerm_kubernetes_cluster.test.kube_config.0.host
+}

_examples/aks/aks-cluster/variables.tf

@@ -0,0 +1,15 @@
+variable "kubernetes_version" {
+  default = "1.18"
+}
+
+variable "workers_count" {
+  default = "3"
+}
+
+variable "cluster_name" {
+  type = string
+}
+
+variable "location" {
+  type = string
+}

_examples/aks/kubernetes-config/main.tf

@@ -0,0 +1,56 @@
+resource "kubernetes_namespace" "test" {
+  metadata {
+    name = "test"
+  }
+}
+
+resource "kubernetes_deployment" "test" {
+  metadata {
+    name = "test"
+    namespace= kubernetes_namespace.test.metadata.0.name
+  }
+  spec {
+    replicas = 2
+    selector {
+      match_labels = {
+        app = "test"
+      }
+    }
+    template {
+      metadata {
+        labels = {
+          app  = "test"
+        }
+      }
+      spec {
+        container {
+          image = "nginx:1.19.4"
+          name  = "nginx"
+
+          resources {
+            limits = {
+              memory = "512M"
+              cpu = "1"
+            }
+            requests = {
+              memory = "256M"
+              cpu = "50m"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+resource helm_release nginx_ingress {
+  name       = "nginx-ingress-controller"
+
+  repository = "https://charts.bitnami.com/bitnami"
+  chart      = "nginx-ingress-controller"
+
+  set {
+    name  = "service.type"
+    value = "ClusterIP"
+  }
+}

_examples/aks/kubernetes-config/variables.tf

@@ -0,0 +1,3 @@
+variable "cluster_name" {
+  type = string
+}

_examples/aks/main.tf

@@ -0,0 +1,50 @@
+terraform {
+  required_providers {
+    kubernetes = {
+      source = "hashicorp/kubernetes"
+      version = ">= 2.0.0"
+    }
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "2.42"
+    }
+    helm = {
+      source  = "hashicorp/helm"
+      version = ">= 2.0.1"
+    }
+  }
+}
+
+provider "kubernetes" {
+  host                   = module.aks-cluster.endpoint
+  client_key             = base64decode(module.aks-cluster.client_key)
+  client_certificate     = base64decode(module.aks-cluster.client_cert)
+  cluster_ca_certificate = base64decode(module.aks-cluster.ca_cert)
+}
+
+provider "helm" {
+  kubernetes {
+    host                   = module.aks-cluster.endpoint
+    client_key             = base64decode(module.aks-cluster.client_key)
+    client_certificate     = base64decode(module.aks-cluster.client_cert)
+    cluster_ca_certificate = base64decode(module.aks-cluster.ca_cert)
+  }
+}
+
+provider "azurerm" {
+  features {}
+}
+
+module "aks-cluster" {
+  providers    = { azurerm = azurerm }
+  source       = "./aks-cluster"
+  cluster_name = local.cluster_name
+  location     = var.location
+}
+
+module "kubernetes-config" {
+  providers    = { kubernetes = kubernetes, helm = helm }
+  depends_on   = [module.aks-cluster]
+  source       = "./kubernetes-config"
+  cluster_name = local.cluster_name
+}

_examples/aks/outputs.tf

@@ -0,0 +1,7 @@
+output "kubeconfig_path" {
+  value = abspath("${path.root}/kubeconfig")
+}
+
+output "cluster_name" {
+  value = local.cluster_name
+}

_examples/aks/variables.tf

@@ -0,0 +1,12 @@
+variable "location" {
+  type    = string
+  default = "westus2"
+}
+
+resource "random_id" "cluster_name" {
+  byte_length = 5
+}
+
+locals {
+  cluster_name = "tf-k8s-${random_id.cluster_name.hex}"
+}