Kubernetes: properly reuse tls certificates ⚠️

charts/Makefile

@@ -4,9 +4,16 @@ REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
 include ${REPO_BASE_DIR}/scripts/common.Makefile
 include $(REPO_CONFIG_LOCATION)
 
+#
+# Vars
+#
+
 export CONFIG_DIR := $(shell dirname $(REPO_CONFIG_LOCATION))
 CHART_DIRS := $(wildcard $(REPO_BASE_DIR)/charts/*/)
 
+HELMFILE_EXTRA_ARGS ?=
+HELMFILE := helmfile $(HELMFILE_EXTRA_ARGS)
+
 .PHONY: .check-helmfile-installed
 .check-helmfile-installed: ## Checks if helmfile is installed
 	@if ! command -v helmfile >/dev/null 2>&1; then \
@@ -22,27 +29,27 @@ simcore-charts/helmfile.yaml: ## Copies the simcore helmfile to the charts direc
 .PHONY: helmfile-lint
 helmfile-lint: .check-helmfile-installed helmfile.yaml ## Lints the helmfile
 	set -a; source $(REPO_CONFIG_LOCATION); set +a; \
-	helmfile lint
+	$(HELMFILE) lint
 
 .PHONY: helmfile-apply
 helmfile-apply: .check-helmfile-installed helmfile.yaml ## Applies the helmfile configuration
 	set -a; source $(REPO_CONFIG_LOCATION); set +a; \
-	helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml apply
+	$(HELMFILE) -f $(REPO_BASE_DIR)/charts/helmfile.yaml apply
 
 .PHONY: helmfile-sync
 helmfile-sync: .check-helmfile-installed helmfile.yaml ## Syncs the helmfile configuration (use `helmfile-apply` to deploy the app)
 	set -a; source $(REPO_CONFIG_LOCATION); set +a; \
-	helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml sync
+	$(HELMFILE) -f $(REPO_BASE_DIR)/charts/helmfile.yaml sync
 
 .PHONY: helmfile-diff
 helmfile-diff: .check-helmfile-installed helmfile.yaml ## Shows the differences that would be applied by helmfile
 	@set -a; source $(REPO_CONFIG_LOCATION); set +a; \
-	helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml diff
+	$(HELMFILE) -f $(REPO_BASE_DIR)/charts/helmfile.yaml diff
 
 .PHONY: helmfile-delete
 helmfile-delete: .check-helmfile-installed helmfile.yaml ## Deletes the helmfile configuration
 	@set -a; source $(REPO_CONFIG_LOCATION); set +a; \
-	helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml delete
+	$(HELMFILE) -f $(REPO_BASE_DIR)/charts/helmfile.yaml delete
 
 .PHONY: up
 up: helmfile-apply ## Start the stack

charts/adminer/values.yaml.gotmpl

@@ -52,12 +52,8 @@ ingress:
   className: ""
   annotations:
     namespace: {{ .Release.Namespace }}
-    cert-manager.io/cluster-issuer: "cert-issuer"
+    traefik.ingress.kubernetes.io/router.tls: "true"
     traefik.ingress.kubernetes.io/router.entrypoints: websecure
-  tls:
-    - hosts:
-        - {{ requiredEnv "K8S_MONITORING_FQDN" }}
-      secretName: monitoring-tls
   hosts:
     - host: {{ requiredEnv "K8S_MONITORING_FQDN" }}
       paths:

charts/cert-manager/README.md

@@ -3,3 +3,9 @@
 Read more https://cert-manager.io/docs/installation/best-practice/#network-requirements
 
 Be aware that this might have an affect on cert manager webhook application that is called during installation of the cert manager helm chart. If network policy is misconfigured, this will affect installation (e.g. `certissuers` might be missing as they are installed via helm hooks that apparently require cert manager webhook to be reachable)
+
+## Extract certificate from secret
+
+```bash
+kubectl -n <namespace> get secret <secret-tls> -o jsonpath="{.data['tls\.crt']}" | base64 -d | openssl x509 -text -noout | head
+```

charts/cert-manager/templates/certificates.yaml

@@ -0,0 +1,25 @@
+{{- range .Values.certificates -}}
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ .certName }}
+spec:
+  # https://github.com/emberstack/kubernetes-reflector?tab=readme-ov-file#cert-manager-support
+  secretTemplate:
+    annotations:
+      reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "traefik"
+      reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "traefik"
+  duration: 2160h # 90 days
+  renewBefore: 720h # 30 days
+  secretName: {{ .secretName }}
+  dnsNames:
+{{- range .dnsNames }}
+    - "{{ . }}"
+{{- end }}
+  issuerRef:
+    name: cert-issuer
+    kind: ClusterIssuer
+---
+{{- end -}}

charts/cert-manager/values.common.yaml.gotmpl

@@ -1,3 +1,9 @@
+certificates:
+  - certName: {{ requiredEnv "MACHINE_FQDN" | replace "." "-" }}
+    secretName: {{ requiredEnv "MACHINE_FQDN" | replace "." "-" }}-tls
+    dnsNames:
+      - {{ requiredEnv "K8S_MONITORING_FQDN" }}
+
 cert-manager:
   crds:
     enabled: true

charts/cert-manager/values.selfsigned.yaml.gotmpl

@@ -23,12 +23,6 @@ cert-manager:
         "helm.sh/hook": post-install,post-upgrade
         "helm.sh/hook-weight": "1"
     spec:
-      secretTemplate:
-        annotations:
-          reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
-          reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: ""  # Control destination namespaces: emptystring means all
-          reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true" # Auto create reflection for matching namespaces
-          reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "" # Control auto-reflection namespaces
       isCA: true
       commonName: local-ca
       subject:

charts/longhorn/values.yaml.gotmpl

@@ -58,11 +58,9 @@ ingress:
     className: ""
     annotations:
       namespace: {{ .Release.Namespace }}
-      cert-manager.io/cluster-issuer: "cert-issuer"
+      traefik.ingress.kubernetes.io/router.tls: "true"
       traefik.ingress.kubernetes.io/router.entrypoints: websecure
       traefik.ingress.kubernetes.io/router.middlewares: traefik-traefik-basic-auth@kubernetescrd,traefik-longhorn-strip-prefix@kubernetescrd  # namespace + middleware name
-    tls: true
-    tlsSecret: monitoring-tls
     host: {{ requiredEnv "K8S_MONITORING_FQDN" }}
     path: /longhorn
     pathType: Prefix

charts/portainer/values.yaml.gotmpl

@@ -41,13 +41,9 @@ portainer:
     className: ""
     annotations:
       namespace: {{ .Release.Namespace }}
-      cert-manager.io/cluster-issuer: "cert-issuer"
+      traefik.ingress.kubernetes.io/router.tls: "true"
       traefik.ingress.kubernetes.io/router.entrypoints: websecure
       traefik.ingress.kubernetes.io/router.middlewares: traefik-traefik-basic-auth@kubernetescrd,traefik-portainer-strip-prefix@kubernetescrd  # namespace + middleware name
-    tls:
-      - hosts:
-          - {{ requiredEnv "K8S_MONITORING_FQDN" }}
-        secretName: monitoring-tls
     hosts:
       - host: {{ requiredEnv "K8S_MONITORING_FQDN" }}
         paths:

charts/reflector/namespace.yaml

@@ -0,0 +1,15 @@
+# namespace with defined pod security standard
+# inspired from https://aro-labs.com/pod-security-standards/
+# official doc: https://kubernetes.io/docs/concepts/security/pod-security-standards/
+#
+# Warning: if pod / container does not meet enforced standards, it will not be deployed (silently)
+# execute `kubectl -n <namespace> events` to see errors (e.g.)
+#   Error creating: pods "xyz" is forbidden: violates PodSecurity "baseline:latest": privileged
+#   container "xyz" must not set securityContext.privileged to true
+#
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: reflector
+  labels:
+    pod-security.kubernetes.io/enforce: baseline

charts/reflector/values.yaml.gotmpl

@@ -0,0 +1,4 @@
+configuration:
+  watcher:
+    # https://github.com/emberstack/kubernetes-reflector/issues/560#issuecomment-3415122791
+    timeout: 30  # seconds

charts/traefik/values.common.yaml.gotmpl

@@ -20,6 +20,9 @@ service:
 ports:
   web:
     nodePort: 32080
+    redirectTo:
+       port: websecure
+       permanent: true
   websecure:
     nodePort: 32443
 
@@ -35,3 +38,129 @@ affinity:  # https://github.com/traefik/traefik-helm-chart/blob/v28.2.0/traefik/
             app.kubernetes.io/name: '{{`{{ template "traefik.name" . }}`}}'
             app.kubernetes.io/instance: '{{ .Release.Name }}'
         topologyKey: kubernetes.io/hostname
+
+tlsStore:
+  default:
+    certificates:
+      # generated by cert manager and copied by reflector
+      - secretName: {{ requiredEnv "MACHINE_FQDN" | replace "." "-" }}-tls
+
+extraObjects:
+
+- apiVersion: v1
+  kind: Service
+  metadata:
+    name: traefik-api
+    namespace: {{.Release.Namespace}}
+  spec:
+    type: ClusterIP
+    selector:
+      app.kubernetes.io/name: traefik
+      app.kubernetes.io/instance: {{.Release.Namespace}}-traefik
+    ports:
+    - port: 8080
+      name: traefik
+      targetPort: 9000
+      protocol: TCP
+
+- apiVersion: v1
+  kind: Secret
+  metadata:
+    name: traefik-authorized-users
+    namespace: {{.Release.Namespace}}
+  data:
+    users: |2
+      {{ requiredEnv "TRAEFIK_K8S_AUTHORIZED_USER" }}
+
+- apiVersion: traefik.io/v1alpha1
+  kind: Middleware
+  metadata:
+    name: traefik-basic-auth
+    namespace: {{.Release.Namespace}}
+  spec:
+    basicAuth:
+      secret: traefik-authorized-users  # https://doc.traefik.io/traefik/middlewares/http/basicauth/#users
+
+- apiVersion: traefik.io/v1alpha1
+  kind: Middleware
+  metadata:
+    name: portainer-strip-prefix
+    namespace: {{.Release.Namespace}}
+  spec:
+    stripPrefix:
+      prefixes:
+      - /portainer
+
+- apiVersion: traefik.io/v1alpha1
+  kind: Middleware
+  metadata:
+    name: longhorn-strip-prefix
+    namespace: {{.Release.Namespace}}
+  spec:
+    stripPrefix:
+      prefixes:
+      - /longhorn
+
+  # a (href) links do not work properly without trailing slash
+- apiVersion: traefik.io/v1alpha1
+  kind: Middleware
+  metadata:
+    name: logs-append-slash
+    namespace: {{ .Release.Namespace }}
+  spec:
+    redirectRegex:
+      regex: "^(https?://[^/]+/logs)$"
+      replacement: "${1}/"
+
+- apiVersion: traefik.io/v1alpha1
+  kind: Middleware
+  metadata:
+    name: logs-strip-prefix
+    namespace: {{.Release.Namespace}}
+  spec:
+    stripPrefix:
+      prefixes:
+      - /logs
+
+- apiVersion: traefik.io/v1alpha1
+  kind: Middleware
+  metadata:
+    name: internal-ipallowlist
+  spec:
+    ipAllowList:
+      sourceRange:
+        - 10.0.0.0/8
+        - 172.16.0.0/12
+        - 192.168.0.0/16
+
+- apiVersion: networking.k8s.io/v1
+  kind: Ingress
+  metadata:
+    name: traefik-dashboard
+    namespace: {{.Release.Namespace}}
+    annotations:
+      traefik.ingress.kubernetes.io/router.entrypoints: websecure
+      traefik.ingress.kubernetes.io/router.middlewares: {{.Release.Namespace}}-traefik-basic-auth@kubernetescrd # namespace + middleware name
+      traefik.ingress.kubernetes.io/router.tls: "true"
+  spec:
+    rules:
+    - host: {{ requiredEnv "K8S_MONITORING_FQDN" }}
+      http:
+        paths:
+        - path: /dashboard
+          pathType: Prefix
+          backend:
+            service:
+              name: traefik-api
+              port:
+                name: traefik
+    - host: {{ requiredEnv "K8S_MONITORING_FQDN" }}
+      http:
+        paths:
+        - path: /api
+          pathType: Prefix
+          backend:
+            service:
+              name: traefik-api
+              port:
+                name: traefik