Skip to content

Commit 7d2b4cc

Browse files
mikhail-klimkomikhail-klimko
authored
feat(app-proxy): add leader election (#621)
1 parent 20f9724 commit 7d2b4cc

File tree

10 files changed

+336
-22
lines changed

10 files changed

+336
-22
lines changed

‎charts/gitops-runtime/templates/_components/cap-app-proxy/_deployment.yaml

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{{- define "cap-app-proxy.resources.deployment" }}
2+
{{- $cfCommonTplSemver := printf "cf-common-%s" (index .Subcharts "cf-common").Chart.Version }}
23
apiVersion: apps/v1
34
kind: Deployment
45
metadata:
@@ -86,6 +87,60 @@ spec:
8687
- mountPath: /app/config/all
8788
name: all-certs
8889
readOnly: true
90+
{{- if gt (int .Values.replicaCount) 1 }}
91+
- name: leader-elector
92+
image: {{ include (printf "%s.image.name" $cfCommonTplSemver ) (dict "image" (index .Values "leader-elector" "image") "context" .) }}
93+
imagePullPolicy: {{ index .Values "leader-elector" "image" "pullPolicy" | default "IfNotPresent" }}
94+
command:
95+
- leader-elector
96+
args:
97+
- --id=$(POD_NAME)
98+
- --lease-name=$(LEASE_NAME)
99+
- --namespace=$(NAMESPACE)
100+
- --lease-duration=$(LEASE_DURATION)
101+
- --lease-renew-duration=$(LEASE_RENEW_DURATION)
102+
env:
103+
- name: NAMESPACE
104+
valueFrom:
105+
fieldRef:
106+
fieldPath: metadata.namespace
107+
- name: POD_NAME
108+
valueFrom:
109+
fieldRef:
110+
fieldPath: metadata.name
111+
- name: LEASE_NAME
112+
value: cap-app-proxy
113+
- name: LEASE_DURATION
114+
value: 10s
115+
- name: LEASE_RENEW_DURATION
116+
value: 5s
117+
{{- with index .Values "leader-elector" "containerSecurityContext" }}
118+
securityContext:
119+
{{- toYaml . | nindent 10 }}
120+
{{- end }}
121+
livenessProbe:
122+
httpGet:
123+
path: /healthz
124+
port: 4040
125+
initialDelaySeconds: {{ index .Values "leader-elector" "livenessProbe" "initialDelaySeconds" }}
126+
periodSeconds: {{ index .Values "leader-elector" "livenessProbe" "periodSeconds" }}
127+
timeoutSeconds: {{ index .Values "leader-elector" "livenessProbe" "timeoutSeconds" }}
128+
successThreshold: {{ index .Values "leader-elector" "livenessProbe" "successThreshold" }}
129+
failureThreshold: {{ index .Values "leader-elector" "livenessProbe" "failureThreshold" }}
130+
readinessProbe:
131+
httpGet:
132+
path: /readyz
133+
port: 4040
134+
initialDelaySeconds: {{ index .Values "leader-elector" "readinessProbe" "initialDelaySeconds" }}
135+
periodSeconds: {{ index .Values "leader-elector" "readinessProbe" "periodSeconds" }}
136+
timeoutSeconds: {{ index .Values "leader-elector" "readinessProbe" "timeoutSeconds" }}
137+
successThreshold: {{ index .Values "leader-elector" "readinessProbe" "successThreshold" }}
138+
failureThreshold: {{ index .Values "leader-elector" "readinessProbe" "failureThreshold" }}
139+
{{- with index .Values "leader-elector" "resources" }}
140+
resources:
141+
{{- toYaml . | nindent 10 }}
142+
{{- end }}
143+
{{- end }}
89144
{{- with .Values.nodeSelector | default .Values.global.nodeSelector }}
90145
nodeSelector:
91146
{{- toYaml . | nindent 8 }}
@@ -98,6 +153,9 @@ spec:
98153
tolerations:
99154
{{- toYaml . | nindent 6 }}
100155
{{- end }}
156+
{{- with .Values.topologySpreadConstraints }}
157+
topologySpreadConstraints: {{- include (printf "%s.tplrender" $cfCommonTplSemver ) (dict "Values" . "context" .) | nindent 8 }}
158+
{{- end }}
101159
volumes:
102160
{{- with .Values.extraVolumes }}
103161
{{- toYaml . | nindent 6 }}

‎charts/gitops-runtime/templates/_components/cap-app-proxy/_rbac.yaml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,25 @@ rules:
5252
- update
5353
- delete
5454
- patch
55+
- apiGroups:
56+
- coordination.k8s.io
57+
resources:
58+
- leases
59+
verbs:
60+
- get
61+
- list
62+
- watch
63+
- create
64+
- update
65+
- patch
66+
- delete
67+
- apiGroups:
68+
- ""
69+
resources:
70+
- events
71+
verbs:
72+
- create
73+
- patch
5574
---
5675
apiVersion: rbac.authorization.k8s.io/v1
5776
kind: RoleBinding

‎charts/gitops-runtime/templates/_components/cap-app-proxy/environment-variables/_main-container.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,12 @@ IRW_JIRA_ENRICHMENT_TASK_IMAGE:
211211
key: enrichmentJiraEnrichmentImage
212212
optional: true
213213
NODE_EXTRA_CA_CERTS: /app/config/all/all.cer
214+
{{- if gt (int .Values.replicaCount) 1 }}
215+
LEADER_ID:
216+
valueFrom:
217+
fieldRef:
218+
fieldPath: metadata.name
219+
{{- end }}
214220
{{ include "codefresh-gitops-runtime.get-proxy-env-vars" . }}
215221
{{- end -}}
216222

‎charts/gitops-runtime/templates/_components/gitops-operator/_deployment.yaml

Lines changed: 25 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -99,27 +99,30 @@ spec:
9999
{{- with .Values.affinity }}
100100
affinity: {{ toYaml . | nindent 8 }}
101101
{{- end }}
102+
{{- with .Values.topologySpreadConstraints }}
103+
topologySpreadConstraints: {{- include (printf "%s.tplrender" $cfCommonTplSemver ) (dict "Values" . "context" .) | nindent 8 }}
104+
{{- end }}
102105
volumes:
103-
- name: tls-certs
104-
configMap:
105-
name: argocd-tls-certs-cm
106-
- name: argocd-repo-server-tls
107-
secret:
108-
secretName: argocd-repo-server-tls
109-
optional: true
110-
items:
111-
- key: tls.crt
112-
path: tls.crt
113-
- key: tls.key
114-
path: tls.key
115-
- key: ca.crt
116-
path: ca.crt
117-
{{- if .Values.global.codefresh.tls.caCerts.secretKeyRef }}
118-
- name: codefresh-tls
119-
secret:
120-
secretName: {{ .Values.global.codefresh.tls.caCerts.secretKeyRef.name }}
121-
{{- end }}
122-
{{- with .Values.extraVolumes }}
123-
{{- toYaml . | nindent 6 }}
124-
{{- end }}
106+
- name: tls-certs
107+
configMap:
108+
name: argocd-tls-certs-cm
109+
- name: argocd-repo-server-tls
110+
secret:
111+
secretName: argocd-repo-server-tls
112+
optional: true
113+
items:
114+
- key: tls.crt
115+
path: tls.crt
116+
- key: tls.key
117+
path: tls.key
118+
- key: ca.crt
119+
path: ca.crt
120+
{{- if .Values.global.codefresh.tls.caCerts.secretKeyRef }}
121+
- name: codefresh-tls
122+
secret:
123+
secretName: {{ .Values.global.codefresh.tls.caCerts.secretKeyRef.name }}
124+
{{- end }}
125+
{{- with .Values.extraVolumes }}
126+
{{- toYaml . | nindent 6 }}
127+
{{- end }}
125128
{{- end }}

‎charts/gitops-runtime/templates/_components/gitops-operator/_pdb.yaml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{{- define "gitops-operator.resources.pdb" }}
2+
apiVersion: policy/v1
3+
kind: PodDisruptionBudget
4+
metadata:
5+
name: {{ include "gitops-operator.fullname" . }}
6+
labels:
7+
{{- include "gitops-operator.labels" . | nindent 4 }}
8+
spec:
9+
{{- if or .Values.pdb.minAvailable .Values.pdb.maxUnavailable }}
10+
{{- with .Values.pdb.minAvailable }}
11+
minAvailable: {{ . }}
12+
{{- end }}
13+
{{- with .Values.pdb.maxUnavailable }}
14+
maxUnavailable: {{ . }}
15+
{{- end }}
16+
{{- else }}
17+
{{- fail (printf "ERROR: pdb.minAvailable or pdb.maxUnavailable is required!" ) }}
18+
{{- end }}
19+
selector:
20+
matchLabels: {{- include "gitops-operator.selectorLabels" . | nindent 6 }}
21+
{{- end }}

‎charts/gitops-runtime/templates/_components/internal-router/_deployment.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{{- define "internal-router.resources.deployment" }}
2+
{{- $cfCommonTplSemver := printf "cf-common-%s" (index .Subcharts "cf-common").Chart.Version }}
23
apiVersion: apps/v1
34
kind: Deployment
45
metadata:
@@ -82,4 +83,7 @@ spec:
8283
tolerations:
8384
{{- toYaml . | nindent 8 }}
8485
{{- end }}
86+
{{- with .Values.topologySpreadConstraints }}
87+
topologySpreadConstraints: {{- include (printf "%s.tplrender" $cfCommonTplSemver ) (dict "Values" . "context" .) | nindent 8 }}
88+
{{- end }}
8589
{{- end }}

‎charts/gitops-runtime/templates/gitops-operator/pdb.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{{- $gitopsOperatorContext := deepCopy . }}
2+
{{- $_ := set $gitopsOperatorContext "Values" (deepCopy (get .Values "gitops-operator")) }}
3+
{{- $_ := set $gitopsOperatorContext.Values "global" (deepCopy (get .Values "global")) }}
4+
5+
{{- if $gitopsOperatorContext.Values.pdb.enabled }}
6+
{{- include "gitops-operator.resources.pdb" $gitopsOperatorContext }}
7+
{{- end }}

‎charts/gitops-runtime/tests/app-proxy-misc_test.yaml

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
# yaml-language-server: $schema=https://raw.githubusercontent.com/helm-unittest/helm-unittest/main/schema/helm-testsuite.json
12
suite: misc tests on app-proxy templates generation
23
templates:
34
- app-proxy/deployment.yaml
@@ -266,3 +267,31 @@ tests:
266267
content:
267268
name: HTTPS_PROXY
268269
value: proxys.example.com
270+
271+
- it: app-proxy should have leader-elector container if replicaCount > 1
272+
template: 'app-proxy/deployment.yaml'
273+
values:
274+
- ./values/mandatory-values.yaml
275+
set:
276+
app-proxy.replicaCount: 2
277+
asserts:
278+
- lengthEqual:
279+
path: spec.template.spec.containers
280+
count: 2
281+
- equal:
282+
path: spec.template.spec.containers[1].name
283+
value: leader-elector
284+
285+
- it: app-proxy should not have leader-elector container if replicaCount = 1
286+
template: 'app-proxy/deployment.yaml'
287+
values:
288+
- ./values/mandatory-values.yaml
289+
set:
290+
app-proxy.replicaCount: 1
291+
asserts:
292+
- lengthEqual:
293+
path: spec.template.spec.containers
294+
count: 1
295+
- equal:
296+
path: spec.template.spec.containers[0].name
297+
value: cap-app-proxy

‎charts/gitops-runtime/values-ha.yaml

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
global:
2+
topologySpreadConstraints:
3+
- maxSkew: 1
4+
topologyKey: kubernetes.io/hostname
5+
whenUnsatisfiable: DoNotSchedule
6+
7+
app-proxy:
8+
# -- Set to 1 until https://codefresh-io.atlassian.net/browse/CR-29338 is resolved
9+
replicaCount: 1
10+
pdb:
11+
enabled: true
12+
minAvailable: 1
13+
topologySpreadConstraints:
14+
- maxSkew: 1
15+
topologyKey: kubernetes.io/hostname
16+
whenUnsatisfiable: DoNotSchedule
17+
labelSelector:
18+
matchLabels:
19+
app: cap-app-proxy
20+
21+
gitops-operator:
22+
replicaCount: 2
23+
pdb:
24+
enabled: true
25+
minAvailable: 1
26+
topologySpreadConstraints:
27+
- maxSkew: 1
28+
topologyKey: kubernetes.io/hostname
29+
whenUnsatisfiable: DoNotSchedule
30+
labelSelector:
31+
matchLabels:
32+
app: gitops-operator
33+
34+
internal-router:
35+
replicaCount: 2
36+
pdb:
37+
enabled: true
38+
minAvailable: 1
39+
topologySpreadConstraints:
40+
- maxSkew: 1
41+
topologyKey: kubernetes.io/hostname
42+
whenUnsatisfiable: DoNotSchedule
43+
labelSelector:
44+
matchLabels:
45+
app: internal-router
46+
47+
cf-argocd-extras:
48+
sourcesServer:
49+
hpa:
50+
enabled: true
51+
minReplicas: 2
52+
pdb:
53+
enabled: true
54+
minAvailable: 1
55+
topologySpreadConstraints:
56+
- maxSkew: 1
57+
topologyKey: kubernetes.io/hostname
58+
whenUnsatisfiable: DoNotSchedule
59+
labelSelector:
60+
matchLabels:
61+
app.kubernetes.io/component: sources-server
62+
eventReporter:
63+
topologySpreadConstraints:
64+
- maxSkew: 1
65+
topologyKey: kubernetes.io/hostname
66+
whenUnsatisfiable: DoNotSchedule
67+
labelSelector:
68+
matchLabels:
69+
app.kubernetes.io/component: event-reporter
70+
71+
argo-cd:
72+
redis-ha:
73+
enabled: true
74+
75+
controller:
76+
replicas: 1
77+
78+
server:
79+
autoscaling:
80+
enabled: true
81+
minReplicas: 2
82+
pdb:
83+
enabled: true
84+
minAvailable: 1
85+
86+
repoServer:
87+
autoscaling:
88+
enabled: true
89+
minReplicas: 2
90+
pdb:
91+
enabled: true
92+
minAvailable: 1
93+
94+
applicationSet:
95+
replicas: 2
96+
97+
argo-workflows:
98+
controller:
99+
replicas: 2
100+
pdb:
101+
enabled: true
102+
minAvailable: 1
103+
server:
104+
autoscaling:
105+
enabled: true
106+
minReplicas: 2
107+
pdb:
108+
enabled: true
109+
minAvailable: 1
110+
111+
event-reporters:
112+
workflow:
113+
sensor:
114+
replicas: 2
115+
affinity:
116+
podAntiAffinity:
117+
requiredDuringSchedulingIgnoredDuringExecution:
118+
- labelSelector:
119+
matchExpressions:
120+
- key: sensor-name
121+
operator: In
122+
values:
123+
- workflow-reporter
124+
topologyKey: "kubernetes.io/hostname"
125+
rollout:
126+
sensor:
127+
replicas: 2
128+
affinity:
129+
podAntiAffinity:
130+
requiredDuringSchedulingIgnoredDuringExecution:
131+
- labelSelector:
132+
matchExpressions:
133+
- key: sensor-name
134+
operator: In
135+
values:
136+
- rollout-reporter
137+
topologyKey: "kubernetes.io/hostname"

0 commit comments

Comments
 (0)