Skip to content

Commit 8def09d

Browse files
YuryHrytsukYuryHrytsuk
committed
Kubernetes: properly reuse tls certificates
Traefik does not properly work when ingress'es in multiple namespaces use the same tls certificate. See more in traefik/traefik#12116. This works around the problem by manually defining certificates and uploading them to TLSStore. Ingress'es use TLSStore under the hood. Implementation detail: 1) we generate certificates by explicitly defining certificate resource in cert-manager 2) we copy generated secrets (containing certitificates) to traefik namespace via reflector 3) traefik explicitly defines TLSStore that references secrets (containing certificates) Bonus: - Add HELMFILE_EXTRA_ARGS variable to Makefile to pass options to helmfile CLI if necessary Related issue/s - closes #1228 Related PR/s - configuration ...
1 parent 472cb6c commit 8def09d

File tree

12 files changed

+57
-120
lines changed

12 files changed

+57
-120
lines changed

‎charts/Makefile‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,16 @@ REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
44
include ${REPO_BASE_DIR}/scripts/common.Makefile
55
include $(REPO_CONFIG_LOCATION)
66

7+
#
8+
# Vars
9+
#
10+
711
export CONFIG_DIR := $(shell dirname $(REPO_CONFIG_LOCATION))
812
CHART_DIRS := $(wildcard $(REPO_BASE_DIR)/charts/*/)
913

14+
HELMFILE_EXTRA_ARGS ?=
15+
HELMFILE := helmfile $(HELMFILE_EXTRA_ARGS)
16+
1017
.PHONY: .check-helmfile-installed
1118
.check-helmfile-installed: ## Checks if helmfile is installed
1219
@if ! command -v helmfile >/dev/null 2>&1; then \
@@ -22,27 +29,27 @@ simcore-charts/helmfile.yaml: ## Copies the simcore helmfile to the charts direc
2229
.PHONY: helmfile-lint
2330
helmfile-lint: .check-helmfile-installed helmfile.yaml ## Lints the helmfile
2431
set -a; source $(REPO_CONFIG_LOCATION); set +a; \
25-
helmfile lint
32+
$(HELMFILE) lint
2633

2734
.PHONY: helmfile-apply
2835
helmfile-apply: .check-helmfile-installed helmfile.yaml ## Applies the helmfile configuration
2936
set -a; source $(REPO_CONFIG_LOCATION); set +a; \
30-
helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml apply
37+
$(HELMFILE) -f $(REPO_BASE_DIR)/charts/helmfile.yaml apply
3138

3239
.PHONY: helmfile-sync
3340
helmfile-sync: .check-helmfile-installed helmfile.yaml ## Syncs the helmfile configuration (use `helmfile-apply` to deploy the app)
3441
set -a; source $(REPO_CONFIG_LOCATION); set +a; \
35-
helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml sync
42+
$(HELMFILE) -f $(REPO_BASE_DIR)/charts/helmfile.yaml sync
3643

3744
.PHONY: helmfile-diff
3845
helmfile-diff: .check-helmfile-installed helmfile.yaml ## Shows the differences that would be applied by helmfile
3946
@set -a; source $(REPO_CONFIG_LOCATION); set +a; \
40-
helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml diff
47+
$(HELMFILE) -f $(REPO_BASE_DIR)/charts/helmfile.yaml diff
4148

4249
.PHONY: helmfile-delete
4350
helmfile-delete: .check-helmfile-installed helmfile.yaml ## Deletes the helmfile configuration
4451
@set -a; source $(REPO_CONFIG_LOCATION); set +a; \
45-
helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml delete
52+
$(HELMFILE) -f $(REPO_BASE_DIR)/charts/helmfile.yaml delete
4653

4754
.PHONY: up
4855
up: helmfile-apply ## Start the stack

‎charts/adminer/values.yaml.gotmpl‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,8 @@ ingress:
5252
className: ""
5353
annotations:
5454
namespace: {{ .Release.Namespace }}
55-
cert-manager.io/cluster-issuer: "cert-issuer"
55+
traefik.ingress.kubernetes.io/router.tls: "true"
5656
traefik.ingress.kubernetes.io/router.entrypoints: websecure
57-
tls:
58-
- hosts:
59-
- {{ requiredEnv "K8S_MONITORING_FQDN" }}
60-
secretName: monitoring-tls
6157
hosts:
6258
- host: {{ requiredEnv "K8S_MONITORING_FQDN" }}
6359
paths:

‎charts/cert-manager/README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,9 @@
33
Read more https://cert-manager.io/docs/installation/best-practice/#network-requirements
44

55
Be aware that this might have an affect on cert manager webhook application that is called during installation of the cert manager helm chart. If network policy is misconfigured, this will affect installation (e.g. `certissuers` might be missing as they are installed via helm hooks that apparently require cert manager webhook to be reachable)
6+
7+
## Extract certificate from secret
8+
9+
```bash
10+
kubectl -n <namespace> get secret <secret-tls> -o jsonpath="{.data['tls\.crt']}" | base64 -d | openssl x509 -text -noout | head
11+
```

‎charts/cert-manager/templates/certificates.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{{- range .Values.certificates -}}
2+
{{- $fqdnNoDots := replace "." "-" .fqdn -}}
3+
apiVersion: cert-manager.io/v1
4+
kind: Certificate
5+
metadata:
6+
name: {{ $fqdnNoDots }}
7+
spec:
8+
secretTemplate:
9+
annotations:
10+
reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
11+
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "traefik"
12+
reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
13+
reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "traefik"
14+
secretName: {{ $fqdnNoDots }}-tls
15+
dnsNames:
16+
{{- range .dnsNames }}
17+
- "{{ . }}"
18+
{{- end }}
19+
issuerRef:
20+
name: cert-issuer
21+
kind: ClusterIssuer
22+
---
23+
{{- end -}}

‎charts/cert-manager/values.common.yaml.gotmpl‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
certificates:
2+
- fqdn: {{ requiredEnv "MACHINE_FQDN" }}
3+
dnsNames:
4+
- {{ requiredEnv "K8S_MONITORING_FQDN" }}
5+
16
cert-manager:
27
crds:
38
enabled: true

‎charts/cert-manager/values.selfsigned.yaml.gotmpl‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,6 @@ cert-manager:
2323
"helm.sh/hook": post-install,post-upgrade
2424
"helm.sh/hook-weight": "1"
2525
spec:
26-
secretTemplate:
27-
annotations:
28-
reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
29-
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "" # Control destination namespaces: emptystring means all
30-
reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true" # Auto create reflection for matching namespaces
31-
reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "" # Control auto-reflection namespaces
3226
isCA: true
3327
commonName: local-ca
3428
subject:

‎charts/longhorn/values.yaml.gotmpl‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,9 @@ ingress:
5858
className: ""
5959
annotations:
6060
namespace: {{ .Release.Namespace }}
61-
cert-manager.io/cluster-issuer: "cert-issuer"
61+
traefik.ingress.kubernetes.io/router.tls: "true"
6262
traefik.ingress.kubernetes.io/router.entrypoints: websecure
6363
traefik.ingress.kubernetes.io/router.middlewares: traefik-traefik-basic-auth@kubernetescrd,traefik-longhorn-strip-prefix@kubernetescrd # namespace + middleware name
64-
tls: true
65-
tlsSecret: monitoring-tls
6664
host: {{ requiredEnv "K8S_MONITORING_FQDN" }}
6765
path: /longhorn
6866
pathType: Prefix

‎charts/portainer/values.yaml.gotmpl‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,9 @@ portainer:
4141
className: ""
4242
annotations:
4343
namespace: {{ .Release.Namespace }}
44-
cert-manager.io/cluster-issuer: "cert-issuer"
44+
traefik.ingress.kubernetes.io/router.tls: "true"
4545
traefik.ingress.kubernetes.io/router.entrypoints: websecure
4646
traefik.ingress.kubernetes.io/router.middlewares: traefik-traefik-basic-auth@kubernetescrd,traefik-portainer-strip-prefix@kubernetescrd # namespace + middleware name
47-
tls:
48-
- hosts:
49-
- {{ requiredEnv "K8S_MONITORING_FQDN" }}
50-
secretName: monitoring-tls
5147
hosts:
5248
- host: {{ requiredEnv "K8S_MONITORING_FQDN" }}
5349
paths:

‎charts/traefik/values.common.yaml.gotmpl‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,9 @@ affinity: # https://github.com/traefik/traefik-helm-chart/blob/v28.2.0/traefik/
3535
app.kubernetes.io/name: '{{`{{ template "traefik.name" . }}`}}'
3636
app.kubernetes.io/instance: '{{ .Release.Name }}'
3737
topologyKey: kubernetes.io/hostname
38+
39+
tlsStore:
40+
default:
41+
certificates:
42+
# generated by cert manager and copied by reflector
43+
- secretName: {{ requiredEnv "MACHINE_FQDN" | replace "." "-" }}-tls

‎charts/traefik/values.insecure.yaml.gotmpl‎

Lines changed: 0 additions & 89 deletions
This file was deleted.

0 commit comments

Comments
 (0)