Mainline Cilium support for Kubespray (#454)

Signed-off-by: Rareș Cosma <[email protected]>

Author: rarescosma

bin/apply.bash

@@ -32,7 +32,11 @@ if [ -z "${CK8S_KUBESPRAY_NO_VENV+x}" ]; then
 fi
 
 log_info "Running kubespray"
-ansible-playbook -i "${config[inventory_file]}" -i "${here}/node-labels-and-taints-inventory.bash" cluster.yml -b "${@}"
+ansible-playbook \
+  -i "${config[inventory_file]}" \
+  -i "${here}/node-labels-and-taints-inventory.bash" \
+  -i "${here}/../dynamic-config/cilium-values-inventory.yaml" \
+  cluster.yml -b "${@}"
 
 log_info "Kubespray done"
 

bin/run-playbook.bash

@@ -34,7 +34,11 @@ if [ -z "${CK8S_KUBESPRAY_NO_VENV+x}" ]; then
 fi
 
 log_info "Running kubespray"
-ansible-playbook -i "${config[inventory_file]}" -i "${here}/node-labels-and-taints-inventory.bash" "-e serial=1" "playbooks/${playbook}" "${@}"
+ansible-playbook \
+  -i "${config[inventory_file]}" \
+  -i "${here}/node-labels-and-taints-inventory.bash" \
+  -i "${here}/../dynamic-config/cilium-values-inventory.yaml" \
+  "-e serial=1" "playbooks/${playbook}" "${@}"
 
 popd
 

config/common/group_vars/k8s_cluster/ck8s-cilium.yaml

@@ -0,0 +1,51 @@
+ck8s_cilium:
+  # policyEnforcementMode: Determine whether an endpoint accepts traffic from a source or not
+  #
+  # Has three options:
+  # - default: endpoints have unrestricted network access until selected by policy
+  # - always: policy enforcement is enabled on all endpoints even if no rules select specific endpoints.
+  # - never: All traffic is allowed from any source (on ingress) or destination (on egress)
+  policyEnforcementMode: default
+
+  # policyAuditMode: when true, no network policy is enforced.
+  #
+  # This feature helps to validate the impact of host policies before enforcing them.
+  policyAuditMode: false
+
+  # enableNodeSelectability: when true, allows Kubernetes nodes to be selected
+  # by cidr block selectors in classic NetworkPolicies.
+  #
+  # Without this, access to the Kubernets API Server pods might be restricted.
+  #
+  # See: https://github.com/cilium/cilium/issues/20550
+  enableNodeSelectability: true
+
+  operator:
+    unmanagedPodWatcher:
+      restart: true
+    monitoring:
+      enabled: true
+      installServiceMonitor: false
+
+  wireguard:
+    enabled: true
+    strictMode: false
+
+  hubble:
+    enabled: true
+    monitoring:
+      installServiceMonitor: false
+    metrics:
+      - drop:labelsContext=traffic_direction,source_pod,source_namespace,source_ip,destination_pod,destination_namespace,destination_ip
+      - flow:labelsContext=traffic_direction,source_pod,source_namespace,source_ip,destination_pod,destination_namespace,destination_ip
+      - dns
+      - tcp
+      - icmp
+      - httpV2
+
+  prometheus:
+    enabled: true
+    installServiceMonitor: false
+
+  envoy:
+    enabled: false

docs/cilium.md

@@ -0,0 +1,40 @@
+# Cilium as network plugin
+
+## Creating a Cilium cluster
+
+To create a new cluster that uses `cilium` as its network plugin, set the following keys in `group_vars/all/k8s_cluster/ck8s-k8s-cluster.yaml`:
+
+```yaml
+# Set cilium as the network plugin
+kube_network_plugin: cilium
+
+# Pin the cilium chart version
+cilium_version: "1.17.5"
+
+# Tell cilium to store identities in CRDs rather than try talking to etcd directly
+cilium_identity_allocation_mode: "crd"
+
+# Enable hubble (needed for some of the metrics)
+cilium_enable_hubble: true
+cilium_hubble_install: true
+cilium_hubble_tls_generate: true
+
+# See https://github.com/kubernetes-sigs/kubespray/issues/12276
+kube_owner: root
+```
+
+## Configuring Cilium
+
+Cilium configuration itself lives in `group_vars/all/k8s_cluster/ck8s-cilium.yaml`.
+
+It closely mimics the [Cilium configuration block that from CAPI](https://github.com/elastisys/ck8s-cluster-api/blob/e2ce0c947c773efa1e1b8e78fcdc2c1f50f484d5/config/base-values.yaml#L119-L156).
+
+For new clusters the file should be copied automatically to the configuration directory during the `init` step.
+
+We don't yet have a migration plan for existing clusters to switch CNIs, but should you desire to experiment you'll need to copy the file manually.
+
+## Development tips
+
+After a successful initial run of Kubespray, if you just want to test out configuration changes you can speed up subsequent runs by a huge factor by skipping certain tags:
+
+    ./bin/ck8s-kubespray apply sc --skip-tags "bootstrap-os,preinstall,container-engine,download,node"

dynamic-config/cilium-values-inventory.yaml

@@ -0,0 +1,48 @@
+all:
+  vars:
+    cilium_extra_values: "{{ cilium_extra_values_template | from_yaml }}"
+    cilium_extra_values_template: |
+      {% if ck8s_cilium | default(false) %}
+      policyEnforcementMode: {{ ck8s_cilium.policyEnforcementMode }}
+
+      policyAuditMode: {{ ck8s_cilium.policyAuditMode }}
+
+      operator:
+        unmanagedPodWatcher:
+          restart: {{ ck8s_cilium.operator.unmanagedPodWatcher.restart }}
+        prometheus:
+          enabled: {{ ck8s_cilium.operator.monitoring.enabled }}
+          serviceMonitor:
+            enabled: {{ ck8s_cilium.operator.monitoring.installServiceMonitor }}
+
+      encryption:
+        enabled: {{ ck8s_cilium.wireguard.enabled }}
+        type: wireguard
+        strictMode:
+          enabled: {{ ck8s_cilium.wireguard.strictMode }}
+
+      hubble:
+        enabled: {{ ck8s_cilium.hubble.enabled }}
+        metrics:
+          enabled: {{ ck8s_cilium.hubble.metrics }}
+          serviceMonitor:
+            enabled: {{ ck8s_cilium.hubble.monitoring.installServiceMonitor }}
+
+      prometheus:
+        enabled: {{ ck8s_cilium.prometheus.enabled }}
+        serviceMonitor:
+          enabled: {{ ck8s_cilium.prometheus.installServiceMonitor }}
+          trustCRDsExist: true
+
+      envoy:
+        enabled: {{ ck8s_cilium.envoy.enabled }}
+
+      {% if ck8s_cilium.enableNodeSelectability | default(false) %}
+      policyCIDRMatchMode: nodes
+      {% endif %}
+
+      annotateK8sNode: true
+
+      {% else %}
+      {}
+      {% endif %}

migration/v2.28/prepare/30-copy-ck8s-cilium-config.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+HERE="$(dirname "$(readlink -f "${0}")")"
+ROOT="$(readlink -f "${HERE}/../../../")"
+
+# shellcheck source=scripts/migration/lib.sh
+source "${ROOT}/scripts/migration/lib.sh"
+
+cilium_path="group_vars/k8s_cluster/ck8s-cilium.yaml"
+
+src_file="${ROOT}/config/common/${cilium_path}"
+
+if [[ "${CK8S_CLUSTER}" =~ ^(sc|both)$ ]]; then
+  dest_file="${CK8S_CONFIG_PATH}/sc-config/${cilium_path}"
+  if [[ ! -f "${dest_file}" ]]; then
+    log_info "Copying ck8s-cilium config to SC [src=${src_file} dest=${dest_file}]"
+    cp "${src_file}" "${dest_file}"
+  fi
+fi
+
+if [[ "${CK8S_CLUSTER}" =~ ^(wc|both)$ ]]; then
+  dest_file="${CK8S_CONFIG_PATH}/wc-config/${cilium_path}"
+  if [[ ! -f "${dest_file}" ]]; then
+    log_info "Copying ck8s-cilium config to WC [src=${src_file} dest=${dest_file}]"
+    cp "${src_file}" "${dest_file}"
+  fi
+fi