feat: Flexible worker group definition

Signed-off-by: Devon Crouse <[email protected]>

Author: devoncrouse

.gitignore

@@ -13,6 +13,6 @@ provider.tf
 generated/**
 
 # visual code
- **/.vscode/*
+**/.vscode/*
 
 .terraform.lock.hcl

locals.tf

@@ -1,29 +1,23 @@
-# Copyright 2017, 2021 Oracle Corporation and/or affiliates.
+# Copyright 2017, 2023 Oracle Corporation and/or affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
 
 locals {
-  tenancy_id     = coalesce(var.tenancy_id, var.tenancy_ocid)
-  compartment_id = coalesce(
-    var.compartment_id, var.compartment_ocid,
-    var.tenancy_id, var.tenancy_ocid,
-  )
-  user_id = var.user_id != "" ? var.user_id : var.current_user_ocid
+  worker_image_id = (length(var.worker_group_image_id) > 0 ? var.worker_group_image_id
+  : var.node_pool_image_id != "none" ? var.node_pool_image_id : "")
+  worker_image_type = (length(var.worker_group_image_type) > 0 ? var.worker_group_image_type
+  : var.node_pool_image_type != "none" ? var.node_pool_image_type : "")
 
-  api_private_key = (
-    var.api_private_key != ""
-    ? try(base64decode(var.api_private_key), var.api_private_key)
-    : var.api_private_key_path != ""
-      ? file(var.api_private_key_path)
-      : null)
-
-  bastion_public_ip                      = var.create_bastion_host == true ? module.bastion[0].bastion_public_ip : var.bastion_public_ip != "" ? var.bastion_public_ip: ""
-  operator_private_ip                    = var.create_operator == true ? module.operator[0].operator_private_ip : var.operator_private_ip !="" ? var.operator_private_ip: ""
+  bastion_public_ip                      = var.create_bastion_host == true ? module.bastion[0].bastion_public_ip : var.bastion_public_ip != "" ? var.bastion_public_ip : ""
+  operator_private_ip                    = var.create_operator == true ? module.operator[0].operator_private_ip : var.operator_private_ip != "" ? var.operator_private_ip : ""
   operator_instance_principal_group_name = var.create_operator == true ? module.operator[0].operator_instance_principal_group_name : ""
 
-  vcn_id       = var.create_vcn == true ? module.vcn[0].vcn_id : coalesce(var.vcn_id, try(data.oci_core_vcns.vcns[0].virtual_networks[0].id,""))
-  ig_route_id  = var.create_vcn == true ? module.vcn[0].ig_route_id : coalesce(var.ig_route_table_id, try(data.oci_core_route_tables.ig[0].route_tables[0].id,""))
-  nat_route_id = var.create_vcn == true ? module.vcn[0].nat_route_id : coalesce(var.nat_route_table_id, try(data.oci_core_route_tables.nat[0].route_tables[0].id,""))
+  vcn_id       = var.create_vcn == true ? module.vcn[0].vcn_id : coalesce(var.vcn_id, try(data.oci_core_vcns.vcns[0].virtual_networks[0].id, ""))
+  ig_route_id  = var.create_vcn == true ? module.vcn[0].ig_route_id : coalesce(var.ig_route_table_id, try(data.oci_core_route_tables.ig[0].route_tables[0].id, ""))
+  nat_route_id = var.create_vcn == true ? module.vcn[0].nat_route_id : coalesce(var.nat_route_table_id, try(data.oci_core_route_tables.nat[0].route_tables[0].id, ""))
 
-  ssh_key_arg                            = var.ssh_private_key_path == "none" ? "" : " -i ${var.ssh_private_key_path}"
   validate_drg_input = var.create_drg && (var.drg_id != null) ? tobool("[ERROR]: create_drg variable can not be true if drg_id is provided.]") : true
+
+  worker_group_primary_subnet_id = coalesce(
+    var.worker_group_primary_subnet_id,
+  lookup(module.network.subnet_ids, "workers", ""))
 }

main.tf

@@ -398,7 +398,7 @@ module "extensions" {
   vpa_version          = var.vpa_version
 
   #Gatekeeper
-  enable_gatekeeper   = var.enable_gatekeeper
+  enable_gatekeeper  = var.enable_gatekeeper
   gatekeeper_version = var.gatekeeper_version
 
   # service account
@@ -407,8 +407,9 @@ module "extensions" {
   service_account_namespace            = var.service_account_namespace
   service_account_cluster_role_binding = var.service_account_cluster_role_binding
 
-  #check worker nodes are active
-  check_node_active = var.check_node_active
+  # check worker nodes are active
+  check_node_active   = var.check_node_active
+  expected_node_count = module.workergroup.expected_node_count + module.oke.expected_node_count
 
   # oke upgrade
   upgrade_nodepool        = var.upgrade_nodepool
@@ -421,7 +422,8 @@ module "extensions" {
     module.bastion,
     module.network,
     module.operator,
-    module.oke
+    module.oke,
+    module.workergroup
   ]
 
   providers = {

modules/bastionsvc/variables.tf

@@ -8,7 +8,7 @@ variable "label_prefix" {}
 
 # bastion service parameters
 variable "bastion_service_access" {
-  type        = list(string)
+  type = list(string)
 }
 
 variable "bastion_service_name" {}

modules/bastionsvc/versions.tf

@@ -3,7 +3,7 @@ terraform {
     oci = {
       source = "oracle/oci"
       # pass oci home region provider explicitly for identity operations
-      version               = ">= 4.67.3"
+      version = ">= 4.67.3"
     }
   }
   required_version = ">= 1.0.0"

modules/extensions/activeworker.tf

@@ -1,9 +1,9 @@
-# Copyright 2017, 2021 Oracle Corporation and/or affiliates.
+# Copyright 2017, 2022 Oracle Corporation and/or affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
 
 resource "null_resource" "check_worker_active" {
   triggers = {
-    node_pools = length(data.oci_containerengine_node_pools.all_node_pools.node_pools)
+    expected_node_count = var.expected_node_count
   }
 
   connection {

modules/extensions/iam.tf

@@ -20,8 +20,8 @@ locals {
 resource "random_id" "dynamic_group_suffix" {
   keepers = {
     # Generate a new suffix only when variables are changed
-    label_prefix         = local.dynamic_group_prefix
-    tenancy_id           = var.tenancy_id
+    label_prefix = local.dynamic_group_prefix
+    tenancy_id   = var.tenancy_id
   }
 
   byte_length = 8
@@ -32,21 +32,21 @@ resource "oci_identity_policy" "operator_use_dynamic_group_policy" {
   provider       = oci.home
   compartment_id = random_id.dynamic_group_suffix.keepers.tenancy_id
   description    = "policy to allow operator host to manage dynamic group"
-  name           = join("-", compact([
+  name = join("-", compact([
     random_id.dynamic_group_suffix.keepers.label_prefix,
     "operator-instance-principal-dynamic-group",
     random_id.dynamic_group_suffix.hex
   ]))
-  statements     = ["Allow dynamic-group ${var.operator_dynamic_group} to use dynamic-groups in tenancy"]
-  count          = (local.create_operator_dynamic_group_policy == true) ? 1 : 0
+  statements = ["Allow dynamic-group ${var.operator_dynamic_group} to use dynamic-groups in tenancy"]
+  count      = (local.create_operator_dynamic_group_policy == true) ? 1 : 0
 }
 
 # 30s delay to allow policies to take effect globally
 resource "time_sleep" "wait_30_seconds" {
   depends_on = [oci_identity_policy.operator_use_dynamic_group_policy]
 
   create_duration = "30s"
-  count          = (local.create_operator_dynamic_group_policy == true) ? 1 : 0
+  count           = (local.create_operator_dynamic_group_policy == true) ? 1 : 0
 }
 
 resource "null_resource" "update_dynamic_group" {
@@ -76,5 +76,5 @@ resource "null_resource" "update_dynamic_group" {
     ]
   }
 
-  count = (local.create_operator_dynamic_group_policy && var.bastion_state == "RUNNING" ) ? 1 : 0
+  count = (local.create_operator_dynamic_group_policy && var.bastion_state == "RUNNING") ? 1 : 0
 }

modules/extensions/k8stools.tf

@@ -21,7 +21,7 @@ resource "null_resource" "install_k8stools_on_operator" {
 
   provisioner "remote-exec" {
     inline = [
-      "cloud-init status --wait",
+      "cloud-init status --wait &> /dev/null",
       "if [ -f \"$HOME/install_kubectx.sh\" ]; then bash \"$HOME/install_kubectx.sh\"; rm -f \"$HOME/install_kubectx.sh\";fi",
     ]
   }

modules/extensions/locals.tf

@@ -1,23 +1,13 @@
-# Copyright 2017, 2021 Oracle Corporation and/or affiliates.
+# Copyright 2017, 2022 Oracle Corporation and/or affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
 
 locals {
   ssh_private_key = (
     var.ssh_private_key != ""
     ? try(base64decode(var.ssh_private_key), var.ssh_private_key)
     : var.ssh_private_key_path != "none"
-      ? file(var.ssh_private_key_path)
-      : null)
-
-  node_pools_size_list = [
-    for node_pool in data.oci_containerengine_node_pools.all_node_pools.node_pools :
-    node_pool.node_config_details[0].size
-  ]
-
-  # workaround for summing a list of numbers: https://github.com/hashicorp/terraform/issues/17239
-  total_nodes = length(flatten([
-    for nodes in local.node_pools_size_list : range(nodes)
-  ]))
+    ? file(var.ssh_private_key_path)
+  : null)
 
   service_account_cluster_role_binding_name = var.service_account_cluster_role_binding == "" ? "${var.service_account_name}-crb" : var.service_account_cluster_role_binding
 

modules/extensions/scripts/check_worker_active.template.sh

@@ -1,27 +1,39 @@
-#!/bin/bash
-# Copyright 2017, 2021 Oracle Corporation and/or affiliates.
+#!/usr/bin/env bash
+# Copyright 2017, 2022 Oracle Corporation and/or affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
+# shellcheck disable=SC1083,SC2309,SC2154,SC2157,SC2034 # Ignore templated/escaped/unused file variables
+export ALL_FILE=~/all_node.active ONE_FILE=~/one_node.active
 
-rm -f all_node.active
-rm -f one_node.active
-
-while [ ! -f $HOME/*node.active ]
-do
-  echo 'sleeping for 30s'
-  sleep 30
-  if [ ${check_node_active} == all ]; then
-    echo 'checking if all worker nodes are active'
-    active_workers=`(kubectl get nodes | awk 'NR>1 {print $2}' | wc -l)`
-    echo $active_workers 'active worker nodes found out of ${total_nodes}'
-    if [ $active_workers -eq ${total_nodes} ]; then
-      touch all_node.active
+function clean_node_active() {
+  rm -f "$${ALL_FILE}" "$${ONE_FILE}"
+}
+
+function get_actual_node_count() {
+  (kubectl get --no-headers nodes | grep -v NotReady | awk '{print $1}' | wc -l) 2>/dev/null || echo '0'
+}
+
+function wait_for_active() {
+  clean_node_active
+
+  while true; do
+    local actual_node_count
+    actual_node_count=$(get_actual_node_count)
+    if [[ $${actual_node_count} -ge ${expected_node_count} ]]; then touch all_node.active; fi
+    if [[ $${actual_node_count} -ge 1 ]]; then touch one_node.active; fi
+
+    if [[ -f "$${ONE_FILE}" ]] && [[ "${check_node_active}" == 'one' ]]; then
+      echo "Ready with $${actual_node_count} node(s)"
+      break
     fi
-  else
-    echo 'checking if 1 active worker node'
-    active_workers=`(kubectl get nodes | awk 'NR>1 {print $2}' | wc -l)`
-    if [ $active_workers -ge 1 ]; then
-      echo '1 active worker node found'
-      touch one_node.active
+
+    if [[ -f "$${ALL_FILE}" ]] && [[ "${check_node_active}" == 'all' ]]; then
+      echo "Ready with $${actual_node_count} node(s)"
+      break
     fi
-  fi
-done
+
+    echo "$(date): Waiting for ${check_node_active} of ${expected_node_count} node(s) to become ready ($${actual_node_count} found)"
+    sleep 30
+  done
+}
+
+if [[ ${expected_node_count} -ge 1 ]]; then time wait_for_active; fi