Skip to content

Commit 239508b

Browse files
YuryHrytsukYuryHrytsuk
authored
Kubernetes: properly reuse tls certificates ⚠️ (#1234)
* Kubernetes: properly reuse tls certificates Traefik does not properly work when ingress'es in multiple namespaces use the same tls certificate. See more in traefik/traefik#12116. This works around the problem by manually defining certificates and uploading them to TLSStore. Ingress'es use TLSStore under the hood. Implementation detail: 1) we generate certificates by explicitly defining certificate resource in cert-manager 2) we copy generated secrets (containing certitificates) to traefik namespace via reflector 3) traefik explicitly defines TLSStore that references secrets (containing certificates) Bonus: - Add HELMFILE_EXTRA_ARGS variable to Makefile to pass options to helmfile CLI if necessary Related issue/s - closes #1228 Related PR/s - configuration ... * traefik: delete ingress tls config * Update certificate values * Customize reflector chart & polish certs template * Merge traefik values + explicit cert dates
1 parent 566d3bf commit 239508b

File tree

14 files changed

+201
-247
lines changed

14 files changed

+201
-247
lines changed

‎charts/Makefile‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,16 @@ REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
44
include ${REPO_BASE_DIR}/scripts/common.Makefile
55
include $(REPO_CONFIG_LOCATION)
66

7+
#
8+
# Vars
9+
#
10+
711
export CONFIG_DIR := $(shell dirname $(REPO_CONFIG_LOCATION))
812
CHART_DIRS := $(wildcard $(REPO_BASE_DIR)/charts/*/)
913

14+
HELMFILE_EXTRA_ARGS ?=
15+
HELMFILE := helmfile $(HELMFILE_EXTRA_ARGS)
16+
1017
.PHONY: .check-helmfile-installed
1118
.check-helmfile-installed: ## Checks if helmfile is installed
1219
@if ! command -v helmfile >/dev/null 2>&1; then \
@@ -22,27 +29,27 @@ simcore-charts/helmfile.yaml: ## Copies the simcore helmfile to the charts direc
2229
.PHONY: helmfile-lint
2330
helmfile-lint: .check-helmfile-installed helmfile.yaml ## Lints the helmfile
2431
set -a; source $(REPO_CONFIG_LOCATION); set +a; \
25-
helmfile lint
32+
$(HELMFILE) lint
2633

2734
.PHONY: helmfile-apply
2835
helmfile-apply: .check-helmfile-installed helmfile.yaml ## Applies the helmfile configuration
2936
set -a; source $(REPO_CONFIG_LOCATION); set +a; \
30-
helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml apply
37+
$(HELMFILE) -f $(REPO_BASE_DIR)/charts/helmfile.yaml apply
3138

3239
.PHONY: helmfile-sync
3340
helmfile-sync: .check-helmfile-installed helmfile.yaml ## Syncs the helmfile configuration (use `helmfile-apply` to deploy the app)
3441
set -a; source $(REPO_CONFIG_LOCATION); set +a; \
35-
helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml sync
42+
$(HELMFILE) -f $(REPO_BASE_DIR)/charts/helmfile.yaml sync
3643

3744
.PHONY: helmfile-diff
3845
helmfile-diff: .check-helmfile-installed helmfile.yaml ## Shows the differences that would be applied by helmfile
3946
@set -a; source $(REPO_CONFIG_LOCATION); set +a; \
40-
helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml diff
47+
$(HELMFILE) -f $(REPO_BASE_DIR)/charts/helmfile.yaml diff
4148

4249
.PHONY: helmfile-delete
4350
helmfile-delete: .check-helmfile-installed helmfile.yaml ## Deletes the helmfile configuration
4451
@set -a; source $(REPO_CONFIG_LOCATION); set +a; \
45-
helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml delete
52+
$(HELMFILE) -f $(REPO_BASE_DIR)/charts/helmfile.yaml delete
4653

4754
.PHONY: up
4855
up: helmfile-apply ## Start the stack

‎charts/adminer/values.yaml.gotmpl‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,8 @@ ingress:
5252
className: ""
5353
annotations:
5454
namespace: {{ .Release.Namespace }}
55-
cert-manager.io/cluster-issuer: "cert-issuer"
55+
traefik.ingress.kubernetes.io/router.tls: "true"
5656
traefik.ingress.kubernetes.io/router.entrypoints: websecure
57-
tls:
58-
- hosts:
59-
- {{ requiredEnv "K8S_MONITORING_FQDN" }}
60-
secretName: monitoring-tls
6157
hosts:
6258
- host: {{ requiredEnv "K8S_MONITORING_FQDN" }}
6359
paths:

‎charts/cert-manager/README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,9 @@
33
Read more https://cert-manager.io/docs/installation/best-practice/#network-requirements
44

55
Be aware that this might have an affect on cert manager webhook application that is called during installation of the cert manager helm chart. If network policy is misconfigured, this will affect installation (e.g. `certissuers` might be missing as they are installed via helm hooks that apparently require cert manager webhook to be reachable)
6+
7+
## Extract certificate from secret
8+
9+
```bash
10+
kubectl -n <namespace> get secret <secret-tls> -o jsonpath="{.data['tls\.crt']}" | base64 -d | openssl x509 -text -noout | head
11+
```

‎charts/cert-manager/templates/certificates.yaml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{{- range .Values.certificates -}}
2+
apiVersion: cert-manager.io/v1
3+
kind: Certificate
4+
metadata:
5+
name: {{ .certName }}
6+
spec:
7+
# https://github.com/emberstack/kubernetes-reflector?tab=readme-ov-file#cert-manager-support
8+
secretTemplate:
9+
annotations:
10+
reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
11+
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "traefik"
12+
reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
13+
reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "traefik"
14+
duration: 2160h # 90 days
15+
renewBefore: 720h # 30 days
16+
secretName: {{ .secretName }}
17+
dnsNames:
18+
{{- range .dnsNames }}
19+
- "{{ . }}"
20+
{{- end }}
21+
issuerRef:
22+
name: cert-issuer
23+
kind: ClusterIssuer
24+
---
25+
{{- end -}}

‎charts/cert-manager/values.common.yaml.gotmpl‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
certificates:
2+
- certName: {{ requiredEnv "MACHINE_FQDN" | replace "." "-" }}
3+
secretName: {{ requiredEnv "MACHINE_FQDN" | replace "." "-" }}-tls
4+
dnsNames:
5+
- {{ requiredEnv "K8S_MONITORING_FQDN" }}
6+
17
cert-manager:
28
crds:
39
enabled: true

‎charts/cert-manager/values.selfsigned.yaml.gotmpl‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,6 @@ cert-manager:
2323
"helm.sh/hook": post-install,post-upgrade
2424
"helm.sh/hook-weight": "1"
2525
spec:
26-
secretTemplate:
27-
annotations:
28-
reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
29-
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "" # Control destination namespaces: emptystring means all
30-
reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true" # Auto create reflection for matching namespaces
31-
reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "" # Control auto-reflection namespaces
3226
isCA: true
3327
commonName: local-ca
3428
subject:

‎charts/longhorn/values.yaml.gotmpl‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,9 @@ ingress:
5858
className: ""
5959
annotations:
6060
namespace: {{ .Release.Namespace }}
61-
cert-manager.io/cluster-issuer: "cert-issuer"
61+
traefik.ingress.kubernetes.io/router.tls: "true"
6262
traefik.ingress.kubernetes.io/router.entrypoints: websecure
6363
traefik.ingress.kubernetes.io/router.middlewares: traefik-traefik-basic-auth@kubernetescrd,traefik-longhorn-strip-prefix@kubernetescrd # namespace + middleware name
64-
tls: true
65-
tlsSecret: monitoring-tls
6664
host: {{ requiredEnv "K8S_MONITORING_FQDN" }}
6765
path: /longhorn
6866
pathType: Prefix

‎charts/portainer/values.yaml.gotmpl‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,9 @@ portainer:
4141
className: ""
4242
annotations:
4343
namespace: {{ .Release.Namespace }}
44-
cert-manager.io/cluster-issuer: "cert-issuer"
44+
traefik.ingress.kubernetes.io/router.tls: "true"
4545
traefik.ingress.kubernetes.io/router.entrypoints: websecure
4646
traefik.ingress.kubernetes.io/router.middlewares: traefik-traefik-basic-auth@kubernetescrd,traefik-portainer-strip-prefix@kubernetescrd # namespace + middleware name
47-
tls:
48-
- hosts:
49-
- {{ requiredEnv "K8S_MONITORING_FQDN" }}
50-
secretName: monitoring-tls
5147
hosts:
5248
- host: {{ requiredEnv "K8S_MONITORING_FQDN" }}
5349
paths:

‎charts/reflector/namespace.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# namespace with defined pod security standard
2+
# inspired from https://aro-labs.com/pod-security-standards/
3+
# official doc: https://kubernetes.io/docs/concepts/security/pod-security-standards/
4+
#
5+
# Warning: if pod / container does not meet enforced standards, it will not be deployed (silently)
6+
# execute `kubectl -n <namespace> events` to see errors (e.g.)
7+
# Error creating: pods "xyz" is forbidden: violates PodSecurity "baseline:latest": privileged
8+
# container "xyz" must not set securityContext.privileged to true
9+
#
10+
apiVersion: v1
11+
kind: Namespace
12+
metadata:
13+
name: reflector
14+
labels:
15+
pod-security.kubernetes.io/enforce: baseline

‎charts/reflector/values.yaml.gotmpl‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
configuration:
2+
watcher:
3+
# https://github.com/emberstack/kubernetes-reflector/issues/560#issuecomment-3415122791
4+
timeout: 30 # seconds

0 commit comments

Comments
 (0)