Skip to content

Commit d55e668

Browse files
cognifloydcognifloyd
authored
Merge pull request #323: add an st2canary pod that validates st2.packs.volumes
2 parents ef56670 + b69f1bf commit d55e668

File tree

10 files changed

+276
-17
lines changed

10 files changed

+276
-17
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
* BREAKING: Use the standardized labels recommended in the Helm docs. You can use `migrations/v1.0/standardize-labels.sh` to prepare an existing cluster before running `helm update`. (#351) (by @cognifloyd)
66
* Drop support for `networking.k8s.io/v1beta1` which was removed in kubernetes v1.22 (EOL 2022-10-28) (#353) (by @cognifloyd)
77
* Reduce duplication in label tests (#354) (by @cognifloyd)
8+
* Add `st2canary` job as a Helm Hook that runs before install/upgrade to ensure `st2.packs.volumes` is configured correctly (if `st2.packs.volumes.enabled`). (#323) (by @cognifloyd)
89

910
## v0.110.0
1011
* Switch st2 to `v3.8` as a new default stable version (#347)

‎templates/_helpers.tpl‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ Usage: "{{ include "stackstorm-ha.labels" (list $ "st2servicename") }}"
1717
{{ include "stackstorm-ha.selectorLabels" . }}
1818
{{- if list "st2web" "ingress" | has $name }}
1919
app.kubernetes.io/component: frontend
20-
{{- else if eq $name "st2tests" }}
20+
{{- else if list "st2canary" "st2tests" | has $name }}
2121
app.kubernetes.io/component: tests
2222
{{- else }}
2323
app.kubernetes.io/component: backend
@@ -222,6 +222,7 @@ consolidate pack-configs-volumes definitions
222222
{{- define "stackstorm-ha.pack-configs-volume-mount" -}}
223223
- name: st2-pack-configs-vol
224224
mountPath: /opt/stackstorm/configs/
225+
readOnly: false
225226
{{- if and .Values.st2.packs.volumes.enabled .Values.st2.packs.volumes.configs .Values.st2.packs.configs }}
226227
- name: st2-pack-configs-from-helm-vol
227228
mountPath: /opt/stackstorm/configs-helm/
@@ -248,8 +249,10 @@ For custom st2packs-Container reduce duplicity by defining it here once
248249
{{- if .Values.st2.packs.volumes.enabled }}
249250
- name: st2-packs-vol
250251
mountPath: /opt/stackstorm/packs
252+
readOnly: false
251253
- name: st2-virtualenvs-vol
252254
mountPath: /opt/stackstorm/virtualenvs
255+
readOnly: false
253256
{{- else if .Values.st2.packs.images }}
254257
- name: st2-packs-vol
255258
mountPath: /opt/stackstorm/packs
@@ -266,8 +269,10 @@ define this here as well to simplify comparison with packs-volume-mounts
266269
{{- if or .Values.st2.packs.images .Values.st2.packs.volumes.enabled }}
267270
- name: st2-packs-vol
268271
mountPath: /opt/stackstorm/packs
272+
readOnly: false
269273
- name: st2-virtualenvs-vol
270274
mountPath: /opt/stackstorm/virtualenvs
275+
readOnly: false
271276
{{- end }}
272277
{{- end -}}
273278

‎templates/jobs.yaml‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -484,6 +484,118 @@ spec:
484484
{{- end }}
485485

486486
{{- end }}
487+
{{- if $.Values.st2.packs.volumes.enabled }}
488+
---
489+
apiVersion: batch/v1
490+
kind: Job
491+
metadata:
492+
name: {{ $.Release.Name }}-job-ensure-packs-volumes-are-writable
493+
labels: {{- include "stackstorm-ha.labels" (list $ "st2canary") | nindent 4 }}
494+
annotations:
495+
helm.sh/hook: pre-install, pre-upgrade, pre-rollback
496+
helm.sh/hook-weight: "-5" # fairly high priority
497+
helm.sh/hook-delete-policy: hook-succeeded
498+
{{- if $.Values.jobs.annotations }}
499+
{{- toYaml $.Values.jobs.annotations | nindent 4 }}
500+
{{- end }}
501+
spec:
502+
template:
503+
metadata:
504+
name: job-st2canary-for-writable-packs-volumes
505+
labels: {{- include "stackstorm-ha.labels" (list $ "st2canary") | nindent 8 }}
506+
annotations:
507+
{{- if $.Values.jobs.annotations }}
508+
{{- toYaml $.Values.jobs.annotations | nindent 8 }}
509+
{{- end }}
510+
spec:
511+
imagePullSecrets:
512+
{{- if $.Values.image.pullSecret }}
513+
- name: {{ $.Values.image.pullSecret }}
514+
{{- end }}
515+
initContainers: []
516+
containers:
517+
- name: st2canary-for-writable-packs-volumes
518+
image: '{{ template "stackstorm-ha.imageRepository" $ }}/st2actionrunner:{{ tpl $.Values.image.tag $ }}'
519+
imagePullPolicy: {{ $.Values.image.pullPolicy }}
520+
{{- with $.Values.securityContext }}
521+
securityContext: {{- toYaml . | nindent 10 }}
522+
{{- end }}
523+
# TODO: maybe use kubectl to assert the volumes have RWX mode
524+
# If volume is a persistentVolumeClaim, then:
525+
# the PVC must only have ReadWriteMany in spec.accessModes
526+
# If volume is something else, then validating through metadata is iffy.
527+
# azureFile, cephfs, csi, glusterfs, nfs, pvc, quobyte, need at least:
528+
# readOnly: false
529+
# ephemeral volumes could also work, ... but that config is even deeper.
530+
command:
531+
- 'sh'
532+
# -e => exit on failure
533+
# -E => trap ERR is inherited in subfunctions
534+
- '-eEc'
535+
- |
536+
cat << 'INTRO'
537+
Testing write permissions for packs volumes.
538+
If this passes, the pod will automatically be deleted.
539+
If this fails, inspect the pod for errors in kubernetes,
540+
and then delete this st2canary pod manually.
541+
INTRO
542+
543+
function __handle_error__(){
544+
cat <<- 'FAIL'
545+
ERROR: One or more volumes in st2.packs.volumes (from helm values) does not meet
546+
StackStorm's shared volumes requirements!
547+
see: https://github.com/StackStorm/stackstorm-k8s#method-2-shared-volumes
548+
549+
HINT: The volumes defined in st2.packs.volumes must use ReadWriteMany (RWX) access mode
550+
so StackStorm can dynamically install packs from any of the st2actionrunner pods
551+
and have those file changes available in all of the other StackStorm pods.
552+
see: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes
553+
FAIL
554+
}
555+
trap __handle_error__ ERR
556+
557+
for volume in packs virtualenvs {{ if $.Values.st2.packs.volumes.configs }}configs{{ end }}; do
558+
echo Testing write permissions on ${volume} volume...
559+
touch /opt/stackstorm/${volume}/.write-test
560+
rm /opt/stackstorm/${volume}/.write-test
561+
echo
562+
done
563+
echo DONE
564+
volumeMounts:
565+
{{- include "stackstorm-ha.packs-volume-mounts" $ | nindent 8 }}
566+
{{/* do not include the pack-configs-volume-mount helper here */}}
567+
- name: st2-pack-configs-vol
568+
mountPath: /opt/stackstorm/configs/
569+
readOnly: false
570+
# TODO: Find out default resource limits for this specific job (#5)
571+
#resources:
572+
volumes:
573+
{{- include "stackstorm-ha.packs-volumes" $ | nindent 8 }}
574+
{{- if $.Values.st2.packs.volumes.configs }}
575+
{{/* do not include the pack-configs-volume helper here */}}
576+
- name: st2-pack-configs-vol
577+
{{- toYaml $.Values.st2.packs.volumes.configs | nindent 10 }}
578+
{{- end }}
579+
# st2canary job does not support extra_volumes. Let us know if you need this.
580+
{{- if $.Values.dnsPolicy }}
581+
dnsPolicy: {{ $.Values.dnsPolicy }}
582+
{{- end }}
583+
{{- with $.Values.dnsConfig }}
584+
dnsConfig: {{- toYaml . | nindent 8 }}
585+
{{- end }}
586+
{{- with $.Values.podSecurityContext }}
587+
securityContext: {{- toYaml . | nindent 8 }}
588+
{{- end }}
589+
{{- with $.Values.jobs.nodeSelector }}
590+
nodeSelector: {{- toYaml . | nindent 8 }}
591+
{{- end }}
592+
{{- with $.Values.jobs.affinity }}
593+
affinity: {{- toYaml . | nindent 8 }}
594+
{{- end }}
595+
{{- with $.Values.jobs.tolerations }}
596+
tolerations: {{- toYaml . | nindent 8 }}
597+
{{- end }}
598+
{{- end }}
487599
{{- range .Values.jobs.extra_hooks -}}
488600
{{- $name := print "extra-helm-hook" (include "stackstorm-ha.hyphenPrefix" (required "You must name each entry in jobs.extra_hooks." .name)) }}
489601
{{- if not ($.Values.jobs.skip | has $name) }}

‎tests/unit/custom_annotations_test.yaml‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,24 @@ tests:
150150
st2:
151151
rbac:
152152
enabled: true # enable rbac job
153+
packs:
154+
sensors: [] # ensure only 1 sensor
155+
images: []
156+
volumes:
157+
enabled: true
158+
packs:
159+
nfs:
160+
server: "10.12.34.56"
161+
path: /var/nfsshare/packs
162+
virtualenvs:
163+
nfs:
164+
server: "10.12.34.56"
165+
path: /var/nfsshare/virtualenvs
166+
configs:
167+
nfs:
168+
server: "10.12.34.56"
169+
path: /var/nfsshare/configs
170+
configs: {} # has one core.yaml config file by default (dicts get merged)
153171
jobs:
154172
annotations:
155173
foo: bar
@@ -161,11 +179,12 @@ tests:
161179
command: ["st2", "run", "--tail", "custom_pack.warn_about_upgrade"]
162180
asserts:
163181
- hasDocuments:
164-
count: 5
182+
count: 6
165183
# job-st2-apply-rbac-defintions
166184
# job-st2-apikey-load
167185
# job-st2-key-load
168186
# job-st2-register-content
187+
# job-st2canary-for-writable-packs-volumes
169188
# extra_hooks job
170189

171190
# job annotations

‎tests/unit/dns_test.yaml‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,24 @@ tests:
2525
- jobs.yaml
2626
set:
2727
st2:
28-
packs: { sensors: [] } # ensure only 1 sensor
28+
packs:
29+
sensors: [] # ensure only 1 sensor
30+
images: []
31+
volumes: &volumes_enabled
32+
enabled: true # for st2canary volumes job
33+
packs:
34+
nfs:
35+
server: "10.12.34.56"
36+
path: /var/nfsshare/packs
37+
virtualenvs:
38+
nfs:
39+
server: "10.12.34.56"
40+
path: /var/nfsshare/virtualenvs
41+
configs:
42+
nfs:
43+
server: "10.12.34.56"
44+
path: /var/nfsshare/configs
45+
configs: {} # has one core.yaml config file by default (dicts get merged)
2946
rbac: { enabled: true } # enable rbac job
3047
jobs:
3148
extra_hooks: &jobs_extra_hooks
@@ -56,7 +73,11 @@ tests:
5673
value: "2"
5774
- name: edns0
5875
st2:
59-
packs: { sensors: [] } # ensure only 1 sensor
76+
packs:
77+
sensors: [] # ensure only 1 sensor
78+
images: []
79+
volumes: *volumes_enabled
80+
configs: {} # has one core.yaml config file by default (dicts get merged)
6081
rbac: { enabled: true } # enable rbac job
6182
jobs:
6283
extra_hooks: *jobs_extra_hooks

‎tests/unit/labels_test.yaml‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,24 @@ tests:
115115
st2:
116116
rbac:
117117
enabled: true # enable rbac job
118+
packs:
119+
sensors: [] # ensure only 1 sensor
120+
images: [] # no extra packs to load
121+
volumes:
122+
enabled: true
123+
packs:
124+
nfs:
125+
server: "10.12.34.56"
126+
path: /var/nfsshare/packs
127+
virtualenvs:
128+
nfs:
129+
server: "10.12.34.56"
130+
path: /var/nfsshare/virtualenvs
131+
configs:
132+
nfs:
133+
server: "10.12.34.56"
134+
path: /var/nfsshare/configs
135+
configs: {} # has one core.yaml config file by default (dicts get merged)
118136
jobs:
119137
extra_hooks:
120138
- name: upgrade-warning
@@ -123,11 +141,12 @@ tests:
123141
command: ["st2", "run", "--tail", "custom_pack.warn_about_upgrade"]
124142
asserts:
125143
- hasDocuments:
126-
count: 5
144+
count: 6
127145
# job-st2-apply-rbac-defintions
128146
# job-st2-apikey-load
129147
# job-st2-key-load
130148
# job-st2-register-content
149+
# job-st2canary-for-writable-packs-volumes
131150
# extra_hooks job
132151

133152
# unlike deployments, jobs should not have selector.matchLabels
@@ -143,8 +162,12 @@ tests:
143162
- equal: *metadata_labels_instance
144163
- equal: *spec_template_metadata_labels_instance
145164

146-
- matchRegex: *regex_metadata_labels_component_backend_or_frontend
147-
- matchRegex: *regex_spec_template_metadata_labels_component_backend_or_frontend
165+
- matchRegex:
166+
path: metadata.labels.[app.kubernetes.io/component]
167+
pattern: ^(backend|tests)$
168+
- matchRegex:
169+
path: spec.template.metadata.labels.[app.kubernetes.io/component]
170+
pattern: ^(backend|tests)$
148171

149172
- equal: *metadata_labels_part_of
150173
- equal: *spec_template_metadata_labels_part_of

0 commit comments

Comments
 (0)