Merge branch 'master' into alternate_registry

Author: cognifloyd

.circleci/config.yml

@@ -27,7 +27,7 @@ jobs:
       - kubernetes/install
       - minikube/minikube-install:
           # https://github.com/kubernetes/minikube/releases
-          version: v1.28.0
+          version: v1.29.0
       - run:
           name: Install Helm v3
           command: curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
@@ -63,9 +63,9 @@ workflows:
             parameters:
               # https://kubernetes.io/releases
               kubernetes-version:
+                - "v1.26.1"
                 - "v1.25.5"
                 - "v1.24.9"
-                - "v1.23.15"
   # Run periodic nightly Helm tests to ensure there are no regressions
   e2e-nightly:
     jobs:
@@ -74,9 +74,9 @@ workflows:
             parameters:
               # https://kubernetes.io/releases
               kubernetes-version:
+                - "v1.26.1"
                 - "v1.25.5"
                 - "v1.24.9"
-                - "v1.23.15"
     triggers:
       - schedule:
           cron: "0 1 * * *"

.github/workflows/e2e.yaml

@@ -28,7 +28,7 @@ jobs:
         # https://github.com/StackStorm/stackstorm-k8s/issues/342
         # https://github.com/k3s-io/k3s/releases
         k3s-channel:
-          - "v1.26.0+k3s1"
+          - "v1.26.1+k3s1"
     steps:
       - name: Checkout source
         uses: actions/checkout@v3

CHANGELOG.md

@@ -1,9 +1,11 @@
 # Changelog
 
 ## In Development
+* Shift K3s and K8s versions forward. (by @mamercad)
 * BREAKING: Use the standardized labels recommended in the Helm docs. You can use `migrations/v1.0/standardize-labels.sh` to prepare an existing cluster before running `helm update`. (#351) (by @cognifloyd)
 * Drop support for `networking.k8s.io/v1beta1` which was removed in kubernetes v1.22 (EOL 2022-10-28) (#353) (by @cognifloyd)
 * Reduce duplication in label tests (#354) (by @cognifloyd)
+* Add `st2canary` job as a Helm Hook that runs before install/upgrade to ensure `st2.packs.volumes` is configured correctly (if `st2.packs.volumes.enabled`). (#323) (by @cognifloyd)
 
 ## v0.110.0
 * Switch st2 to `v3.8` as a new default stable version (#347)

templates/_helpers.tpl

@@ -17,7 +17,7 @@ Usage: "{{ include "stackstorm-ha.labels" (list $ "st2servicename") }}"
 {{ include "stackstorm-ha.selectorLabels" . }}
 {{- if list "st2web" "ingress" | has $name }}
 app.kubernetes.io/component: frontend
-{{- else if eq $name "st2tests" }}
+{{- else if list "st2canary" "st2tests" | has $name }}
 app.kubernetes.io/component: tests
 {{- else }}
 app.kubernetes.io/component: backend
@@ -234,6 +234,7 @@ consolidate pack-configs-volumes definitions
 {{- define "stackstorm-ha.pack-configs-volume-mount" -}}
 - name: st2-pack-configs-vol
   mountPath: /opt/stackstorm/configs/
+  readOnly: false
   {{- if and .Values.st2.packs.volumes.enabled .Values.st2.packs.volumes.configs .Values.st2.packs.configs }}
 - name: st2-pack-configs-from-helm-vol
   mountPath: /opt/stackstorm/configs-helm/
@@ -260,8 +261,10 @@ For custom st2packs-Container reduce duplicity by defining it here once
   {{- if .Values.st2.packs.volumes.enabled }}
 - name: st2-packs-vol
   mountPath: /opt/stackstorm/packs
+  readOnly: false
 - name: st2-virtualenvs-vol
   mountPath: /opt/stackstorm/virtualenvs
+  readOnly: false
   {{- else if .Values.st2.packs.images }}
 - name: st2-packs-vol
   mountPath: /opt/stackstorm/packs
@@ -278,8 +281,10 @@ define this here as well to simplify comparison with packs-volume-mounts
   {{- if or .Values.st2.packs.images .Values.st2.packs.volumes.enabled }}
 - name: st2-packs-vol
   mountPath: /opt/stackstorm/packs
+  readOnly: false
 - name: st2-virtualenvs-vol
   mountPath: /opt/stackstorm/virtualenvs
+  readOnly: false
   {{- end }}
 {{- end -}}
 

templates/jobs.yaml

@@ -484,6 +484,118 @@ spec:
     {{- end }}
 
 {{- end }}
+{{- if $.Values.st2.packs.volumes.enabled }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ $.Release.Name }}-job-ensure-packs-volumes-are-writable
+  labels: {{- include "stackstorm-ha.labels" (list $ "st2canary") | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-install, pre-upgrade, pre-rollback
+    helm.sh/hook-weight: "-5" # fairly high priority
+    helm.sh/hook-delete-policy: hook-succeeded
+  {{- if $.Values.jobs.annotations }}
+    {{- toYaml $.Values.jobs.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  template:
+    metadata:
+      name: job-st2canary-for-writable-packs-volumes
+      labels: {{- include "stackstorm-ha.labels" (list $ "st2canary") | nindent 8 }}
+      annotations:
+      {{- if $.Values.jobs.annotations }}
+        {{- toYaml $.Values.jobs.annotations | nindent 8 }}
+      {{- end }}
+    spec:
+      imagePullSecrets:
+      {{- if $.Values.image.pullSecret }}
+      - name: {{ $.Values.image.pullSecret }}
+      {{- end }}
+      initContainers: []
+      containers:
+      - name: st2canary-for-writable-packs-volumes
+        image: '{{ template "stackstorm-ha.imageRepository" $ }}/st2actionrunner:{{ tpl $.Values.image.tag $ }}'
+        imagePullPolicy: {{ $.Values.image.pullPolicy }}
+        {{- with $.Values.securityContext }}
+        securityContext: {{- toYaml . | nindent 10 }}
+        {{- end }}
+        # TODO: maybe use kubectl to assert the volumes have RWX mode
+        # If volume is a persistentVolumeClaim, then:
+        #   the PVC must only have ReadWriteMany in spec.accessModes
+        # If volume is something else, then validating through metadata is iffy.
+        #   azureFile, cephfs, csi, glusterfs, nfs, pvc, quobyte, need at least:
+        #     readOnly: false
+        #   ephemeral volumes could also work, ... but that config is even deeper.
+        command:
+          - 'sh'
+          # -e => exit on failure
+          # -E => trap ERR is inherited in subfunctions
+          - '-eEc'
+          - |
+            cat << 'INTRO'
+            Testing write permissions for packs volumes.
+            If this passes, the pod will automatically be deleted.
+            If this fails, inspect the pod for errors in kubernetes,
+            and then delete this st2canary pod manually.
+            INTRO
+
+            function __handle_error__(){
+              cat <<- 'FAIL'
+              ERROR: One or more volumes in st2.packs.volumes (from helm values) does not meet
+              StackStorm's shared volumes requirements!
+              see: https://github.com/StackStorm/stackstorm-k8s#method-2-shared-volumes
+
+              HINT: The volumes defined in st2.packs.volumes must use ReadWriteMany (RWX) access mode
+              so StackStorm can dynamically install packs from any of the st2actionrunner pods
+              and have those file changes available in all of the other StackStorm pods.
+              see: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes
+              FAIL
+            }
+            trap __handle_error__ ERR
+
+            for volume in packs virtualenvs {{ if $.Values.st2.packs.volumes.configs }}configs{{ end }}; do
+              echo Testing write permissions on ${volume} volume...
+              touch /opt/stackstorm/${volume}/.write-test
+              rm /opt/stackstorm/${volume}/.write-test
+              echo
+            done
+            echo DONE
+        volumeMounts:
+        {{- include "stackstorm-ha.packs-volume-mounts" $ | nindent 8 }}
+        {{/* do not include the pack-configs-volume-mount helper here */}}
+        - name: st2-pack-configs-vol
+          mountPath: /opt/stackstorm/configs/
+          readOnly: false
+        # TODO: Find out default resource limits for this specific job (#5)
+        #resources:
+      volumes:
+        {{- include "stackstorm-ha.packs-volumes" $ | nindent 8 }}
+        {{- if $.Values.st2.packs.volumes.configs }}
+          {{/* do not include the pack-configs-volume helper here */}}
+        - name: st2-pack-configs-vol
+          {{- toYaml $.Values.st2.packs.volumes.configs | nindent 10 }}
+        {{- end }}
+        # st2canary job does not support extra_volumes. Let us know if you need this.
+    {{- if $.Values.dnsPolicy }}
+      dnsPolicy: {{ $.Values.dnsPolicy }}
+    {{- end }}
+    {{- with $.Values.dnsConfig }}
+      dnsConfig: {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with $.Values.podSecurityContext }}
+      securityContext: {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with $.Values.jobs.nodeSelector }}
+      nodeSelector: {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with $.Values.jobs.affinity }}
+      affinity: {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with $.Values.jobs.tolerations }}
+      tolerations: {{- toYaml . | nindent 8 }}
+    {{- end }}
+{{- end }}
 {{- range .Values.jobs.extra_hooks -}}
   {{- $name := print "extra-helm-hook" (include "stackstorm-ha.hyphenPrefix" (required "You must name each entry in jobs.extra_hooks." .name)) }}
   {{- if not ($.Values.jobs.skip | has $name) }}

tests/unit/custom_annotations_test.yaml

@@ -150,6 +150,24 @@ tests:
       st2:
         rbac:
           enabled: true # enable rbac job
+        packs:
+          sensors: [] # ensure only 1 sensor
+          images: []
+          volumes:
+            enabled: true
+            packs:
+              nfs:
+                server: "10.12.34.56"
+                path: /var/nfsshare/packs
+            virtualenvs:
+              nfs:
+                server: "10.12.34.56"
+                path: /var/nfsshare/virtualenvs
+            configs:
+              nfs:
+                server: "10.12.34.56"
+                path: /var/nfsshare/configs
+          configs: {} # has one core.yaml config file by default (dicts get merged)
       jobs:
         annotations:
           foo: bar
@@ -161,11 +179,12 @@ tests:
             command: ["st2", "run", "--tail", "custom_pack.warn_about_upgrade"]
     asserts:
       - hasDocuments:
-          count: 5
+          count: 6
           # job-st2-apply-rbac-defintions
           # job-st2-apikey-load
           # job-st2-key-load
           # job-st2-register-content
+          # job-st2canary-for-writable-packs-volumes
           # extra_hooks job
 
       # job annotations

tests/unit/dns_test.yaml

@@ -25,7 +25,24 @@ tests:
       - jobs.yaml
     set:
       st2:
-        packs: { sensors: [] } # ensure only 1 sensor
+        packs:
+          sensors: [] # ensure only 1 sensor
+          images: []
+          volumes: &volumes_enabled
+            enabled: true # for st2canary volumes job
+            packs:
+              nfs:
+                server: "10.12.34.56"
+                path: /var/nfsshare/packs
+            virtualenvs:
+              nfs:
+                server: "10.12.34.56"
+                path: /var/nfsshare/virtualenvs
+            configs:
+              nfs:
+                server: "10.12.34.56"
+                path: /var/nfsshare/configs
+          configs: {} # has one core.yaml config file by default (dicts get merged)
         rbac: { enabled: true } # enable rbac job
       jobs:
         extra_hooks: &jobs_extra_hooks
@@ -56,7 +73,11 @@ tests:
             value: "2"
           - name: edns0
       st2:
-        packs: { sensors: [] } # ensure only 1 sensor
+        packs:
+          sensors: [] # ensure only 1 sensor
+          images: []
+          volumes: *volumes_enabled
+          configs: {} # has one core.yaml config file by default (dicts get merged)
         rbac: { enabled: true } # enable rbac job
       jobs:
         extra_hooks: *jobs_extra_hooks

tests/unit/labels_test.yaml

@@ -115,6 +115,24 @@ tests:
       st2:
         rbac:
           enabled: true # enable rbac job
+        packs:
+          sensors: [] # ensure only 1 sensor
+          images: [] # no extra packs to load
+          volumes:
+            enabled: true
+            packs:
+              nfs:
+                server: "10.12.34.56"
+                path: /var/nfsshare/packs
+            virtualenvs:
+              nfs:
+                server: "10.12.34.56"
+                path: /var/nfsshare/virtualenvs
+            configs:
+              nfs:
+                server: "10.12.34.56"
+                path: /var/nfsshare/configs
+          configs: {} # has one core.yaml config file by default (dicts get merged)
       jobs:
         extra_hooks:
           - name: upgrade-warning
@@ -123,11 +141,12 @@ tests:
             command: ["st2", "run", "--tail", "custom_pack.warn_about_upgrade"]
     asserts:
       - hasDocuments:
-          count: 5
+          count: 6
           # job-st2-apply-rbac-defintions
           # job-st2-apikey-load
           # job-st2-key-load
           # job-st2-register-content
+          # job-st2canary-for-writable-packs-volumes
           # extra_hooks job
 
       # unlike deployments, jobs should not have selector.matchLabels
@@ -143,8 +162,12 @@ tests:
       - equal: *metadata_labels_instance
       - equal: *spec_template_metadata_labels_instance
 
-      - matchRegex: *regex_metadata_labels_component_backend_or_frontend
-      - matchRegex: *regex_spec_template_metadata_labels_component_backend_or_frontend
+      - matchRegex:
+          path: metadata.labels.[app.kubernetes.io/component]
+          pattern: ^(backend|tests)$
+      - matchRegex:
+          path: spec.template.metadata.labels.[app.kubernetes.io/component]
+          pattern: ^(backend|tests)$
 
       - equal: *metadata_labels_part_of
       - equal: *spec_template_metadata_labels_part_of