feat: enable autoscaling on a given cluster (#138)

* feat: enable autoscaling on a given cluster

* fix: pre commit failures

* fix: add lifecycle to ignore addons

* fix: add validation for cluster_autoscaler_version

* fix: update the example

* fix: update examples

* fix: get addon list from user

* fix: remove autoscaling example and update logic

* docs: update docs

* fix: doc update

Author: Aashiq-J

README.md

@@ -16,8 +16,6 @@ A module for provisioning an IBM Cloud Red Hat OpenShift cluster on VPC Gen2. Th
 - Make sure that you have a recent version of the [IBM Cloud CLI](https://cloud.ibm.com/docs/cli?topic=cli-getting-started)
 - Make sure that you have a recent version of the [IBM Cloud Kubernetes service CLI](https://cloud.ibm.com/docs/containers?topic=containers-kubernetes-service-cli)
 
-
-
 ## Usage
 ```hcl
 # Replace "master" with a GIT release version to lock into a specific release
@@ -125,7 +123,7 @@ Optionally, you need the following permissions to attach Access Management tags
 - [ Apply Taints Example](examples/apply_taints)
 - [ Existing COS](examples/existing_cos)
 - [ 2 MZR clusters in same VPC](examples/multiple_mzr_clusters)
-- [ Single Zone Cluster](examples/single_zone_cluster)
+- [ Single zone cluster example](examples/single_zone_cluster)
 - [ Standard Example With User Managed Boot Volume Encryption](examples/standard)
 <!-- END EXAMPLES HOOK -->
 
@@ -136,7 +134,9 @@ Optionally, you need the following permissions to attach Access Management tags
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.3.0 |
 | <a name="requirement_ibm"></a> [ibm](#requirement\_ibm) | >= 1.51.0 |
+| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | 2.16.1 |
 | <a name="requirement_null"></a> [null](#requirement\_null) | >= 3.2.1 |
+| <a name="requirement_time"></a> [time](#requirement\_time) | 0.9.1 |
 
 ## Modules
 
@@ -148,14 +148,17 @@ Optionally, you need the following permissions to attach Access Management tags
 
 | Name | Type |
 |------|------|
+| [ibm_container_addons.addons](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_addons) | resource |
 | [ibm_container_vpc_cluster.autoscaling_cluster](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_vpc_cluster) | resource |
 | [ibm_container_vpc_cluster.cluster](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_vpc_cluster) | resource |
 | [ibm_container_vpc_worker_pool.autoscaling_pool](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_vpc_worker_pool) | resource |
 | [ibm_container_vpc_worker_pool.pool](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_vpc_worker_pool) | resource |
 | [ibm_resource_tag.cluster_access_tag](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/resource_tag) | resource |
 | [ibm_resource_tag.cos_access_tag](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/resource_tag) | resource |
+| [kubernetes_config_map_v1_data.set_autoscaling](https://registry.terraform.io/providers/hashicorp/kubernetes/2.16.1/docs/resources/config_map_v1_data) | resource |
 | [null_resource.confirm_network_healthy](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [null_resource.reset_api_key](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
+| [time_sleep.wait_operators](https://registry.terraform.io/providers/hashicorp/time/0.9.1/docs/resources/sleep) | resource |
 | [ibm_container_cluster_config.cluster_config](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/data-sources/container_cluster_config) | data source |
 | [ibm_container_cluster_versions.cluster_versions](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/data-sources/container_cluster_versions) | data source |
 
@@ -164,6 +167,7 @@ Optionally, you need the following permissions to attach Access Management tags
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_access_tags"></a> [access\_tags](#input\_access\_tags) | A list of access tags to apply to the resources created by the module, see https://cloud.ibm.com/docs/account?topic=account-access-tags-tutorial for more details | `list(string)` | `[]` | no |
+| <a name="input_addons"></a> [addons](#input\_addons) | List of all addons supported by the ocp cluster. | <pre>object({<br>    alb-oauth-proxy           = optional(string)<br>    debug-tool                = optional(string)<br>    image-key-synchronizer    = optional(string)<br>    istio                     = optional(string)<br>    openshift-data-foundation = optional(string)<br>    static-route              = optional(string)<br>    cluster-autoscaler        = optional(string)<br>    vpc-block-csi-driver      = optional(string)<br>  })</pre> | `null` | no |
 | <a name="input_cluster_name"></a> [cluster\_name](#input\_cluster\_name) | The name that will be assigned to the provisioned cluster | `string` | n/a | yes |
 | <a name="input_cluster_ready_when"></a> [cluster\_ready\_when](#input\_cluster\_ready\_when) | The cluster is ready when one of the following: MasterNodeReady (not recommended), OneWorkerNodeReady, Normal, IngressReady | `string` | `"IngressReady"` | no |
 | <a name="input_cos_name"></a> [cos\_name](#input\_cos\_name) | Name of the COS instance to provision. New instance only provisioned if `use_existing_cos = false`. Default: `<cluster_name>_cos` | `string` | `null` | no |
@@ -182,7 +186,7 @@ Optionally, you need the following permissions to attach Access Management tags
 | <a name="input_verify_worker_network_readiness"></a> [verify\_worker\_network\_readiness](#input\_verify\_worker\_network\_readiness) | By setting this to true, a script will run kubectl commands to verify that all worker nodes can communicate successfully with the master. If the runtime does not have access to the kube cluster to run kubectl commands, this should be set to false. | `bool` | `true` | no |
 | <a name="input_vpc_id"></a> [vpc\_id](#input\_vpc\_id) | Id of the VPC instance where this cluster will be provisioned | `string` | n/a | yes |
 | <a name="input_vpc_subnets"></a> [vpc\_subnets](#input\_vpc\_subnets) | Metadata that describes the VPC's subnets. Obtain this information from the VPC where this cluster will be created | <pre>map(list(object({<br>    id         = string<br>    zone       = string<br>    cidr_block = string<br>  })))</pre> | n/a | yes |
-| <a name="input_worker_pools"></a> [worker\_pools](#input\_worker\_pools) | List of worker pools | <pre>list(object({<br>    subnet_prefix = optional(string)<br>    vpc_subnets = optional(list(object({<br>      id         = string<br>      zone       = string<br>      cidr_block = string<br>    })))<br>    pool_name         = string<br>    machine_type      = string<br>    workers_per_zone  = number<br>    resource_group_id = optional(string)<br>    labels            = optional(map(string))<br>    boot_volume_encryption_kms_config = optional(object({<br>      crk             = string<br>      kms_instance_id = string<br>      kms_account_id  = optional(string)<br>    }))<br>  }))</pre> | n/a | yes |
+| <a name="input_worker_pools"></a> [worker\_pools](#input\_worker\_pools) | List of worker pools | <pre>list(object({<br>    subnet_prefix = optional(string)<br>    vpc_subnets = optional(list(object({<br>      id         = string<br>      zone       = string<br>      cidr_block = string<br>    })))<br>    pool_name         = string<br>    machine_type      = string<br>    workers_per_zone  = number<br>    resource_group_id = optional(string)<br>    labels            = optional(map(string))<br>    minSize           = optional(number)<br>    maxSize           = optional(number)<br>    enableAutoscaling = optional(bool)<br>    boot_volume_encryption_kms_config = optional(object({<br>      crk             = string<br>      kms_instance_id = string<br>      kms_account_id  = optional(string)<br>    }))<br>  }))</pre> | n/a | yes |
 | <a name="input_worker_pools_taints"></a> [worker\_pools\_taints](#input\_worker\_pools\_taints) | Optional, Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | `null` | no |
 
 ## Outputs

examples/single_zone_cluster/README.md

@@ -1,3 +1,3 @@
-# Single Zone Cluster
+# Single zone cluster example
 
-This is an example of creating a single zone VPC Gen2 OpenShift cluster
+This example creates a VPC with a Red Hat OpenShift cluster in a single zone. The example includes autoscaling for the cluster.

examples/single_zone_cluster/main.tf

@@ -29,6 +29,9 @@ module "vpc" {
 # Base OCP
 ###############################################################################
 locals {
+  addons = {
+    "cluster-autoscaler" = "1.0.8"
+  }
 
   cluster_vpc_subnets = {
     default = module.vpc.subnet_detail_map.zone-1
@@ -39,7 +42,26 @@ locals {
       pool_name        = "default" # ibm_container_vpc_cluster automatically names default pool "default" (See https://github.com/IBM-Cloud/terraform-provider-ibm/issues/2849)
       machine_type     = "bx2.4x16"
       workers_per_zone = 2
-  }]
+    },
+    {
+      subnet_prefix     = "default"
+      pool_name         = "logging"
+      machine_type      = "bx2.4x16"
+      workers_per_zone  = 2
+      minSize           = 1
+      maxSize           = 6
+      enableAutoscaling = true
+    },
+    {
+      subnet_prefix     = "default"
+      pool_name         = "sample"
+      machine_type      = "bx2.4x16"
+      workers_per_zone  = 4
+      minSize           = 1
+      maxSize           = 6
+      enableAutoscaling = true
+    }
+  ]
 }
 
 module "ocp_base" {
@@ -54,6 +76,13 @@ module "ocp_base" {
   ocp_version          = var.ocp_version
   tags                 = var.resource_tags
   worker_pools         = local.sz_pool
+  addons               = local.addons
+}
+
+data "ibm_container_cluster_config" "cluster_config" {
+  cluster_name_id   = module.ocp_base.cluster_id
+  resource_group_id = module.ocp_base.resource_group_id
+
 }
 
 ##############################################################################

examples/single_zone_cluster/provider.tf

@@ -9,3 +9,9 @@ provider "ibm" {
 }
 
 ##############################################################################
+
+provider "kubernetes" {
+  host                   = data.ibm_container_cluster_config.cluster_config.host
+  token                  = data.ibm_container_cluster_config.cluster_config.token
+  cluster_ca_certificate = data.ibm_container_cluster_config.cluster_config.ca_certificate
+}

examples/single_zone_cluster/version.tf

@@ -6,6 +6,10 @@ terraform {
       source  = "ibm-cloud/ibm"
       version = "1.51.0"
     }
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = ">= 2.16.1"
+    }
   }
 }
 

main.tf

@@ -24,6 +24,9 @@ locals {
   # tflint-ignore: terraform_unused_declarations
   validate_check = regex("^${local.validate_msg}$", (!local.validate_condition ? local.validate_msg : ""))
 
+  addons_list = var.addons != null ? { for k, v in var.addons : k => v if v != null } : {}
+  addons      = lookup(local.addons_list, "vpc-block-csi-driver", null) == null ? merge(local.addons_list, { vpc-block-csi-driver = "5.0" }) : local.addons_list
+
   delete_timeout = "2h"
   create_timeout = "3h"
   update_timeout = "3h"
@@ -352,3 +355,51 @@ resource "null_resource" "confirm_network_healthy" {
     }
   }
 }
+
+
+resource "ibm_container_addons" "addons" {
+  cluster           = local.cluster_id
+  resource_group_id = var.resource_group_id
+
+  dynamic "addons" {
+    for_each = local.addons
+    content {
+      name    = addons.key
+      version = addons.value
+    }
+  }
+}
+
+resource "time_sleep" "wait_operators" {
+  depends_on      = [ibm_container_addons.addons]
+  create_duration = "5s"
+}
+
+locals {
+  worker_pool_config = [
+    for worker in var.worker_pools :
+    {
+      name    = worker.pool_name
+      minSize = worker.minSize
+      maxSize = worker.maxSize
+      enabled = worker.enableAutoscaling
+    } if worker.enableAutoscaling != null && worker.minSize != null && worker.maxSize != null
+  ]
+
+}
+
+resource "kubernetes_config_map_v1_data" "set_autoscaling" {
+  count      = !(var.disable_public_endpoint) && lookup(local.addons_list, "cluster-autoscaler", null) != null ? 1 : 0
+  depends_on = [time_sleep.wait_operators]
+
+  metadata {
+    name      = "iks-ca-configmap"
+    namespace = "kube-system"
+  }
+
+  data = {
+    "workerPoolsConfig.json" = jsonencode(local.worker_pool_config)
+  }
+
+  force = true
+}