Upgrade cilium extension to 1.16

Author: robo-cap

docs/src/guide/cluster_addons.md

@@ -1,6 +1,6 @@
 # Cluster Add-ons
 
-With this module to manage both essential and optional add-ons on enhanced OKE clusters. 
+With this module to manage both essential and optional add-ons on **enhanced** OKE clusters. 
 
 This module provides the option to remove [Essential addons](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengintroducingclusteraddons.htm#contengintroducingclusteraddons__section-essential-addons) and to manage, both essential & [optional addons](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengintroducingclusteraddons.htm#contengintroducingclusteraddons__section-optional-addons).
 

docs/src/guide/extensions_networking.md

@@ -23,6 +23,91 @@
 {{#include ../../../examples/extensions/vars-extensions-cilium.auto.tfvars:4:}}
 ```
 
+Cillium is a eBPF based CNI for Kubernetes that can be configured on OKE clusters.
+
+The OKE cluster should be initially configured to run **flannel**. 
+
+On **enhanced** clusters we can use the cluster-addons module to remove flannel extension and kube-proxy (Optional) at cluster creation. 
+
+
+```
+cluster_addons_to_remove = {
+  Flannel = {
+    remove_k8s_resources = true
+  },
+  KubeProxy = {
+    remove_k8s_resources = true
+  }
+}
+```
+
+If you want to use cilium as [kube-proxy replacement](https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/), you can use the following helm_values:
+
+```
+cilium_helm_values      = {
+  kubeProxyReplacement = true
+}
+```
+
+For the basic clusters you can add the following label to the worker nodes to prevent flannel pods from being scheduled:
+
+```
+oci.oraclecloud.com/custom-k8s-networking=true
+```
+
+If you want to override and of the default values(listed below) you can use the `cilium_helm_values` variable:
+
+```
+"annotateK8sNode": true
+"cluster":
+  "id": 1
+  "name": "oke-${var.state_id}"
+"clustermesh":
+  "apiserver":
+    "kvstoremesh":
+      "enabled": false
+  "useAPIServer": false
+"cni":
+  "exclusive": true
+  "install": true
+"hubble":
+  "metrics":
+    "dashboards":
+      "enabled": false
+  "relay":
+    "enabled": true
+  "ui":
+    "enabled": true
+"installNoConntrackIptablesRules": false
+"ipam":
+  "mode": "kubernetes"
+"k8s":
+  "requireIPv4PodCIDR": true
+"k8sServiceHost": "${var.cluster_private_endpoint}"
+"k8sServicePort": "6443"
+"kubeProxyReplacement": false
+"operator":
+  "prometheus":
+    "enabled": false
+"pmtuDiscovery":
+  "enabled": true
+"rollOutCiliumPods": true
+"tunnelProtocol": "vxlan"
+```
+
+
+**Notes:**
+1. Tested with OKE version `v1.29.1` and the worker nodes running: `Oracle-Linux-8.9-2024.05.29-0-OKE-1.29.1-707`.
+
+2. In case the `hubble-relay` and `hubble-ui` pods fail to start, run the following commands:
+
+```
+kubectl delete pod --namespace kube-system -l k8s-app=kube-dns
+kubectl delete pod --namespace kube-system -l k8s-app=hubble-relay
+kubectl delete pod --namespace kube-system -l k8s-app=hubble-ui
+kubectl delete pod --namespace kube-system -l k8s-app=kube-dns-autoscaler
+```
+
 ### References
 * [cilium.io](https://cilium.io)
 

examples/extensions/vars-extensions-cilium.auto.tfvars

@@ -1,9 +1,9 @@
-# Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates.
+# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
 
 cilium_install           = true
 cilium_reapply           = false
-cilium_namespace         = "network"
-cilium_helm_version      = "45.2.0"
+cilium_namespace         = "kube-system"
+cilium_helm_version      = "1.16.3"
 cilium_helm_values       = {}
 cilium_helm_values_files = []

module-extensions.tf

@@ -1,6 +1,13 @@
 # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
 
+locals {
+  cluster_private_endpoint = ( var.create_cluster ? 
+    coalesce(split(":", lookup(one(module.cluster[*].endpoints), "private_endpoint", ""))...) : 
+    coalesce(split(":", lookup(local.existing_cluster_endpoints, "private_endpoint", ""))...)
+  )
+}
+
 module "extensions" {
   source   = "./modules/extensions"
   depends_on = [ module.network ]
@@ -12,6 +19,7 @@ module "extensions" {
   kubernetes_version  = var.kubernetes_version
   expected_node_count = local.worker_count_expected
   worker_pools        = one(module.workers[*].worker_pools)
+  cluster_private_endpoint = local.cluster_private_endpoint
 
   # Bastion/operator connection
   ssh_private_key = sensitive(local.ssh_private_key)

module-workers.tf

@@ -58,7 +58,7 @@ module "workers" {
   image_type                 = var.worker_image_type
   kubeproxy_mode             = var.kubeproxy_mode
   max_pods_per_node          = var.max_pods_per_node
-  node_labels                = var.worker_node_labels
+  node_labels                = alltrue([var.cluster_type == "basic", var.cilium_install == true]) ? merge(var.worker_node_labels, {"oci.oraclecloud.com/custom-k8s-networking" = true}) : var.worker_node_labels
   node_metadata              = var.worker_node_metadata
   agent_config               = var.agent_config
   platform_config            = var.platform_config

modules/extensions/cilium.tf

@@ -2,45 +2,47 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
 
 locals {
-  cilium_helm_crds_file       = join("/", [local.yaml_manifest_path, "cilium.crds.yaml"])
-  cilium_helm_manifest_file   = join("/", [local.yaml_manifest_path, "cilium.manifest.yaml"])
-  cilium_helm_values_file     = join("/", [local.yaml_manifest_path, "cilium.values.yaml"])
-  cilium_net_attach_def_file  = join("/", [local.yaml_manifest_path, "cilium.net_attach_def.yaml"])
-  cilium_veth_config_map_file = join("/", [local.yaml_manifest_path, "cilium.cni_config_map.yaml"])
-
-  cilium_helm_crds     = one(data.helm_template.cilium[*].crds)
-  cilium_helm_manifest = one(data.helm_template.cilium[*].manifest)
+  cilium_helm_crds_file            = join("/", [local.yaml_manifest_path, "cilium.crds.yaml"])
+  cilium_helm_manifest_file        = join("/", [local.yaml_manifest_path, "cilium.manifest.yaml"])
+  cilium_helm_values_file          = join("/", [local.yaml_manifest_path, "cilium.values.yaml"])
+  cilium_helm_values_override_file = join("/", [local.yaml_manifest_path, "cilium.values-override.yaml"])
+  cilium_net_attach_def_file       = join("/", [local.yaml_manifest_path, "cilium.net_attach_def.yaml"])
+  cilium_veth_config_map_file      = join("/", [local.yaml_manifest_path, "cilium.cni_config_map.yaml"])
+
+  cilium_helm_crds            = one(data.helm_template.cilium[*].crds)
+  cilium_helm_values_override = one(data.helm_template.cilium[*].values)
+  
+  cilium_helm_repository      = "https://helm.cilium.io"
 
   cilium_vxlan_cni = {
     install      = true
-    chainingMode = "none"
     exclusive    = true # !var.multus_install
   }
 
-  # TODO Support Flannel w/ generic-veth & tunnel disabled
-  cilium_tunnel = "vxlan" # var.cni_type == "flannel" ? "disabled" : "vxlan"
-  cilium_flannel_cni = {
-    install      = true
-    chainingMode = "generic-veth"
-    configMap    = "cni-configuration"
-    customConf   = var.cni_type == "flannel"
-    exclusive    = !var.multus_install
-  }
-
   cilium_helm_values = {
     annotateK8sNode                 = true
-    cluster                         = { name = "oke-${var.state_id}" }
-    clustermesh                     = { useAPIServer = true }
+    cluster                         = {
+      name = "oke-${var.state_id}"
+      id   = 1
+    }
+    clustermesh                     = { 
+      useAPIServer = false 
+      apiserver = {
+        kvstoremesh = {
+          enabled = false
+        }
+      }
+    }
     cni                             = local.cilium_vxlan_cni
-    containerRuntime                = { integration = "crio" }
-    installIptablesRules            = true
     installNoConntrackIptablesRules = false
     ipam                            = { mode = "kubernetes" }
-    ipv4NativeRoutingCIDR           = element(var.vcn_cidrs, 0)
-    kubeProxyReplacement            = "disabled"
+    kubeProxyReplacement            = false
+    k8sServiceHost                  = var.cluster_private_endpoint
+    k8sServicePort                  = "6443"
     pmtuDiscovery                   = { enabled = true }
-    tunnel                          = local.cilium_tunnel
-
+    rollOutCiliumPods               = true
+    tunnelProtocol                  = local.cilium_tunnel
+    
     hubble = {
       metrics = {
         dashboards = { enabled = var.prometheus_install }
@@ -52,19 +54,9 @@ locals {
 
     k8s = {
       requireIPv4PodCIDR   = true # wait for Kubernetes to provide the PodCIDR (ipam kubernetes)
-      enableIPv4Masquerade = true # var.cni_type != "flannel"  # masquerade IPv4 traffic leaving the node from endpoints
     }
 
     # Prometheus metrics
-    metrics = {
-      dashboards = { enabled = var.prometheus_install }
-      #   # serviceMonitor = { enabled = var.prometheus_enabled }
-    }
-
-    prometheus = {
-      enabled = var.prometheus_install
-      # serviceMonitor = { enabled = var.prometheus_enabled }
-    }
 
     operator = {
       prometheus = {
@@ -74,6 +66,17 @@ locals {
     }
   }
 
+ # TODO Support Flannel w/ generic-veth & tunnel disabled
+  cilium_tunnel = "vxlan" # var.cni_type == "flannel" ? "disabled" : "vxlan"
+
+  cilium_flannel_cni = {
+    install      = true
+    chainingMode = "generic-veth"
+    configMap    = "cni-configuration"
+    customConf   = var.cni_type == "flannel"
+    exclusive    = !var.multus_install
+  }
+  
   cilium_net_attach_def_conf = {
     cniVersion = "0.3.1"
     name       = "cilium"
@@ -126,15 +129,16 @@ locals {
     data = { "cni-config" = jsonencode(local.cilium_veth_conf) }
   }
 
-  cilium_net_attach_def_yaml  = yamlencode(local.cilium_net_attach_def)
-  cilium_veth_config_map_yaml = yamlencode(local.cilium_veth_config_map)
-  cilium_helm_values_yaml     = yamlencode(local.cilium_helm_values)
+  cilium_net_attach_def_yaml       = yamlencode(local.cilium_net_attach_def)
+  cilium_veth_config_map_yaml      = yamlencode(local.cilium_veth_config_map)
+  cilium_helm_values_yaml          = yamlencode(merge(local.cilium_helm_values, var.cilium_helm_values))
+  cilium_helm_values_override_yaml = local.cilium_helm_values_override != null ? join("\n", local.cilium_helm_values_override) : ""
 }
 
 data "helm_template" "cilium" {
   count        = var.cilium_install ? 1 : 0
   chart        = "cilium"
-  repository   = "https://helm.cilium.io"
+  repository   = local.cilium_helm_repository
   version      = var.cilium_helm_version
   kube_version = var.kubernetes_version
 
@@ -165,7 +169,7 @@ resource "null_resource" "cilium" {
   triggers = {
     helm_version = var.cilium_helm_version
     crds_md5     = try(md5(join("\n", local.cilium_helm_crds)), null)
-    manifest_md5 = try(md5(local.cilium_helm_manifest), null)
+    manifest_md5 = try(md5(local.cilium_helm_values_override_yaml), null)
     reapply      = var.cilium_reapply ? uuid() : null
   }
 
@@ -190,24 +194,19 @@ resource "null_resource" "cilium" {
   }
 
   provisioner "file" {
-    content     = local.cilium_helm_manifest
-    destination = local.cilium_helm_manifest_file
-  }
-
-  provisioner "file" {
-    content     = local.cilium_helm_values_yaml
-    destination = local.cilium_helm_values_file
+    content     = local.cilium_helm_values_override_yaml
+    destination = local.cilium_helm_values_override_file
   }
 
-  provisioner "file" {
-    content     = local.cilium_net_attach_def_yaml
-    destination = local.cilium_net_attach_def_file
-  }
+  # provisioner "file" {
+  #   content     = local.cilium_net_attach_def_yaml
+  #   destination = local.cilium_net_attach_def_file
+  # }
 
-  provisioner "file" {
-    content     = local.cilium_veth_config_map_yaml
-    destination = local.cilium_veth_config_map_file
-  }
+  # provisioner "file" {
+  #   content     = local.cilium_veth_config_map_yaml
+  #   destination = local.cilium_veth_config_map_file
+  # }
 
   provisioner "remote-exec" {
     inline = [for c in compact([
@@ -219,7 +218,7 @@ resource "null_resource" "cilium" {
       format(local.kubectl_apply_server_ns_file, var.cilium_namespace, local.cilium_helm_crds_file),
 
       # Install full manifest
-      format(local.kubectl_apply_ns_file, var.cilium_namespace, local.cilium_helm_manifest_file),
+      format(local.helm_upgrade_install, "cilium", "cilium", local.cilium_helm_repository, var.cilium_helm_version, var.cilium_namespace, local.cilium_helm_values_override_file),
 
       # Install Network Attachment Definition when Multus is enabled
       # var.multus_install ? format(local.kubectl_apply_file, local.cilium_net_attach_def_file) : null,

modules/extensions/locals.tf

@@ -4,11 +4,13 @@
 locals {
   yaml_manifest_path           = "/home/${var.operator_user}/yaml"
   kubectl                      = "set -o pipefail; kubectl"
+  helm                         = "set -o pipefail; helm"
   kubectl_apply_ns_file        = "${local.kubectl} apply -n %s -f %s"
   kubectl_apply_file           = "${local.kubectl} apply -f %s"
   kubectl_apply_server_file    = "${local.kubectl} apply --force-conflicts=true --server-side -f %s"
   kubectl_apply_server_ns_file = "${local.kubectl} apply -n %s --force-conflicts=true --server-side -f %s"
   kubectl_create_missing_ns    = "${local.kubectl} create ns %s --dry-run=client -o yaml | kubectl apply -f -"
   selector_linux               = { "kubernetes.io/os" = "linux" }
   output_log                   = "bash -c \"%s | tee >(systemd-cat -t %s -p info)\""
+  helm_upgrade_install         = "${local.helm} upgrade --install %s %s --repo %s --version %s --namespace %s --create-namespace --skip-crds -f %s"
 }

modules/extensions/variables.tf

@@ -7,6 +7,7 @@ variable "state_id" { type = string }
 variable "worker_pools" { type = any }
 variable "kubernetes_version" { type = string }
 variable "expected_node_count" { type = number }
+variable "cluster_private_endpoint" { type = string }
 
 # Connection
 variable "bastion_host" { type = string }
@@ -25,7 +26,7 @@ variable "cilium_install" { type = bool }
 variable "cilium_reapply" { type = bool }
 variable "cilium_namespace" { type = string }
 variable "cilium_helm_version" { type = string }
-variable "cilium_helm_values" { type = map(string) }
+variable "cilium_helm_values" { type = any }
 variable "cilium_helm_values_files" { type = list(string) }
 
 # CNI: Multus

modules/workers/locals.tf

@@ -157,7 +157,6 @@ locals {
         {
           "oke.oraclecloud.com/tf.module"          = "terraform-oci-oke"
           "oke.oraclecloud.com/tf.state_id"        = var.state_id
-          # "oke.oraclecloud.com/tf.workspace"       = terraform.workspace
           "oke.oraclecloud.com/pool.name"          = pool_name
           "oke.oraclecloud.com/pool.mode"          = pool.mode
           "oke.oraclecloud.com/cluster_autoscaler" = pool.allow_autoscaler ? "allowed" : "disabled"

variables-extensions.tf

@@ -22,15 +22,15 @@ variable "cilium_namespace" {
 }
 
 variable "cilium_helm_version" {
-  default     = "1.14.4"
+  default     = "1.16.3"
   description = "Version of the Helm chart to install. List available releases using `helm search repo [keyword] --versions`."
   type        = string
 }
 
 variable "cilium_helm_values" {
   default     = {}
   description = "Map of individual Helm chart values. See https://registry.terraform.io/providers/hashicorp/helm/latest/docs/data-sources/template."
-  type        = map(string)
+  type        = any
 }
 
 variable "cilium_helm_values_files" {