feat: add certMount.mode=perNode for live cert rotation (#20)

disentangle-network · privsim · web-flow · commit c08d5c24d449 · 2026-03-07T21:58:51.000-07:00
Add certMount configuration to the nebula DaemonSet supporting two modes: "shared" (default, unchanged init-container pattern) and "perNode" (direct Secret mount enabling live cert rotation via K8s Secret volume auto-updates). perNode mode is designed for fleet overlay topology where each cluster has one overlay identity. Secret name derived from release fullname, not K8s node names. Closes #19 Co-authored-by: privsim <excaliberswake@pm.me>
diff --git a/helm/disentangle/templates/nebula-daemonset.yaml b/helm/disentangle/templates/nebula-daemonset.yaml
@@ -1,4 +1,5 @@
 {{- if .Values.nebula.enabled }}
+{{- $certMode := .Values.nebula.certMount.mode | default "shared" }}
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -19,6 +20,7 @@ spec:
     spec:
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
+      {{- if eq $certMode "shared" }}
       initContainers:
         - name: cert-selector
           image: "{{ .Values.nebula.initImage.repository }}:{{ .Values.nebula.initImage.tag }}"
@@ -35,6 +37,7 @@ spec:
               readOnly: true
             - name: selected-certs
               mountPath: /selected
+      {{- end }}
       containers:
         - name: nebula
           image: "{{ .Values.nebula.image.repository }}:{{ .Values.nebula.image.tag }}"
@@ -67,13 +70,19 @@ spec:
         - name: nebula-config
           configMap:
             name: {{ include "disentangle.fullname" . }}-nebula-config
+        {{- if eq $certMode "shared" }}
         - name: all-certs
           secret:
             secretName: {{ .Values.nebula.certSecretName }}
         - name: selected-certs
           emptyDir:
             medium: Memory
             sizeLimit: 1Mi
+        {{- else if eq $certMode "perNode" }}
+        - name: selected-certs
+          secret:
+            secretName: {{ .Values.nebula.certMount.secretName | default (printf "%s%s" (.Values.nebula.certMount.secretPrefix | default "nebula-cert-") (include "disentangle.fullname" .)) }}
+        {{- end }}
         - name: tun
           hostPath:
             path: /dev/net/tun
diff --git a/helm/disentangle/tests/nebula_daemonset_test.yaml b/helm/disentangle/tests/nebula_daemonset_test.yaml
@@ -206,3 +206,127 @@ tests:
       - lengthEqual:
           path: spec.template.spec.volumes
           count: 4
+
+  # --- certMount.mode=shared (explicit) tests ---
+
+  - it: should behave identically when certMount.mode is explicitly shared
+    set:
+      nebula.enabled: true
+      nebula.certMount.mode: "shared"
+    asserts:
+      - lengthEqual:
+          path: spec.template.spec.initContainers
+          count: 1
+      - lengthEqual:
+          path: spec.template.spec.volumes
+          count: 4
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: selected-certs
+            emptyDir:
+              medium: Memory
+              sizeLimit: 1Mi
+
+  # --- certMount.mode=perNode tests ---
+
+  - it: perNode mode should not have initContainers
+    set:
+      nebula.enabled: true
+      nebula.certMount.mode: "perNode"
+    asserts:
+      - isNull:
+          path: spec.template.spec.initContainers
+
+  - it: perNode mode should have three volumes (no all-certs)
+    set:
+      nebula.enabled: true
+      nebula.certMount.mode: "perNode"
+    asserts:
+      - lengthEqual:
+          path: spec.template.spec.volumes
+          count: 3
+
+  - it: perNode mode selected-certs volume should be a Secret (not emptyDir)
+    set:
+      nebula.enabled: true
+      nebula.certMount.mode: "perNode"
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: selected-certs
+            secret:
+              secretName: nebula-cert-test-release-disentangle
+
+  - it: perNode mode should derive secretName from prefix + fullname by default
+    set:
+      nebula.enabled: true
+      nebula.certMount.mode: "perNode"
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: selected-certs
+            secret:
+              secretName: nebula-cert-test-release-disentangle
+
+  - it: perNode mode should use custom secretName when set
+    set:
+      nebula.enabled: true
+      nebula.certMount.mode: "perNode"
+      nebula.certMount.secretName: "my-custom-nebula-secret"
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: selected-certs
+            secret:
+              secretName: my-custom-nebula-secret
+
+  - it: perNode mode should use custom secretPrefix when secretName is empty
+    set:
+      nebula.enabled: true
+      nebula.certMount.mode: "perNode"
+      nebula.certMount.secretPrefix: "overlay-cert-"
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: selected-certs
+            secret:
+              secretName: overlay-cert-test-release-disentangle
+
+  - it: perNode mode should preserve the same volumeMounts on the main container
+    set:
+      nebula.enabled: true
+      nebula.certMount.mode: "perNode"
+    asserts:
+      - lengthEqual:
+          path: spec.template.spec.containers[0].volumeMounts
+          count: 3
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: selected-certs
+            mountPath: /etc/nebula/certs
+            readOnly: true
+
+  - it: perNode mode should still have nebula-config and tun volumes
+    set:
+      nebula.enabled: true
+      nebula.certMount.mode: "perNode"
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: nebula-config
+            configMap:
+              name: test-release-disentangle-nebula-config
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: tun
+            hostPath:
+              path: /dev/net/tun
+              type: CharDevice
diff --git a/helm/disentangle/values.schema.json b/helm/disentangle/values.schema.json
@@ -554,6 +554,29 @@
           "description": "Name of the Kubernetes Secret containing Nebula certificates (created by 'launch mesh add', SOPS-encrypted).",
           "default": "nebula-certs"
         },
+        "certMount": {
+          "type": "object",
+          "description": "Certificate mount mode for the Nebula sidecar.",
+          "additionalProperties": false,
+          "properties": {
+            "mode": {
+              "type": "string",
+              "description": "Certificate mount mode. 'shared' uses init-container to select from shared Secret. 'perNode' mounts a dedicated Secret directly (enables live cert rotation).",
+              "enum": ["shared", "perNode"],
+              "default": "shared"
+            },
+            "secretName": {
+              "type": "string",
+              "description": "Exact Secret name for perNode mode. If empty, derived from secretPrefix + release fullname.",
+              "default": ""
+            },
+            "secretPrefix": {
+              "type": "string",
+              "description": "Prefix for deriving Secret name in perNode mode when secretName is empty.",
+              "default": "nebula-cert-"
+            }
+          }
+        },
         "staticHostMap": {
           "type": "object",
           "description": "Maps VPN IPs to real addresses for lighthouse discovery. Keys are VPN IPs, values are real host:port addresses.",
diff --git a/helm/disentangle/values.yaml b/helm/disentangle/values.yaml
@@ -127,6 +127,22 @@ nebula:
   # Certificate secret name (created by launch mesh add, SOPS-encrypted)
   # Expected keys: ca.crt, <nodeName>.crt, <nodeName>.key for each K8s node
   certSecretName: "nebula-certs"
+  # Certificate mount mode:
+  #   "shared"  - (default) init-container copies per-node certs from a single
+  #               shared Secret (certSecretName) into an emptyDir. Requires pod
+  #               restart to pick up rotated certs.
+  #   "perNode" - each release mounts its own Secret directly at /etc/nebula/certs.
+  #               Secret name = certMount.secretName or secretPrefix + fullname.
+  #               Enables live cert rotation without pod restart.
+  #               Designed for fleet overlay topology (one overlay identity per cluster).
+  certMount:
+    mode: "shared"
+    # Only used when mode=perNode. Override to set the exact Secret name.
+    # Expected Secret keys: ca.crt, host.crt, host.key
+    secretName: ""
+    # Only used when mode=perNode and secretName is empty. Prefix concatenated
+    # with the Helm release fullname to derive the Secret name.
+    secretPrefix: "nebula-cert-"
   # Static host map: maps VPN IPs to real addresses for lighthouse discovery
   staticHostMap: {}
     # "10.42.0.1": "lighthouse.disentangle.network:4242"
