Merge pull request #218 from cognifloyd/dynamic-st2sensors

Allow partitioning sensors with hash ranges across multiple deployments instead of one-sensor-per-pod

Author: cognifloyd

CHANGELOG.md

@@ -5,6 +5,7 @@
 * Prefix template helpers with chart name and format helper comments as template comments. (#272) (by @cognifloyd)
 * New feature: Add `extra_volumes` to all python-based st2 deployments. This can facilitate changing log levels by loading logging conf file(s) from a custom ConfigMap. (#276) (by @cognifloyd)
 * Initialize basic unittest infrastructure using `helm-unittest`. Added tests for labels, custom annotations, SecurityContext, pullSecrets, pullPolicy, Resources, nodeSelector, tolerations, affinity, dnsPolicy, dnsConfig, ServiceAccount attach, postStartScript, and both sensor-modes. (#284, #288)
+* Allow partitioning sensors using the hash_range strategy instead of one sensor per pod. (#218) (by @cognifloyd)
 
 ## v0.80.0
 * Switch st2 to `v3.6` as a new default stable version (#274)

README.md

@@ -123,6 +123,11 @@ which in our case is `Redis`.
 st2sensorcontainer manages StackStorm sensors: It starts, stops and restarts them as subprocesses.
 By default, deployment is configured with `1` replica containing all the sensors.
 
+You can increase the number of `st2sensorcontainer` pods by increasing the number of deployments.
+The replicas count is still only `1` per deployment, but the sensors are distributed between the deployments using
+[Sensor Hash Range Partitioning](https://docs.stackstorm.com/reference/sensor_partitioning.html#hash).
+The hash ranges are calculated automatically based on the number of deployments.
+
 st2sensorcontainer also supports a more Docker-friendly single-sensor-per-container mode as a way of
 [Sensor Partitioning](https://docs.stackstorm.com/latest/reference/sensor_partitioning.html). This
 distributes the computing load between many pods and relies on K8s failover/reschedule mechanisms,

templates/deployments.yaml

@@ -1164,13 +1164,29 @@ spec:
       tolerations: {{- toYaml . | nindent 8 }}
     {{- end }}
 
+{{- $sensor_deployments_count := max 1 ($.Values.st2sensorcontainer.deployments|int) | int }}
 {{- $one_sensor_per_pod := not ($.Values.st2.packs.sensors | empty) }}
-{{- range ($one_sensor_per_pod | ternary ($.Values.st2.packs.sensors) (until 1)) }}
-  {{- $sensor := omit $.Values.st2sensorcontainer "name" "ref" "postStartScript" }}
+{{- $some_sensors_per_pod := gt $sensor_deployments_count 1 }}
+{{- if and $one_sensor_per_pod $some_sensors_per_pod }}
+  {{- fail "The sensor values are ambiguous. To use one-sensor-per-pod, use `st2.packs.sensors`. To use multiple-sensors-per-pod, use `st2sensorcontainer.deployments > 1`. Do not use both!" }}
+{{- end }}
+{{- range ($one_sensor_per_pod | ternary ($.Values.st2.packs.sensors) (until $sensor_deployments_count)) }}
+  {{- $sensor := omit $.Values.st2sensorcontainer "name" "ref" "deployments" "postStartScript" }}
   {{- if $one_sensor_per_pod }}
+    {{/* . is an entry in st2.packs.sensors */}}
     {{- range $key, $val := . }}
       {{- $_ := set $sensor $key $val }}
     {{- end }}
+  {{- else if $some_sensors_per_pod }}
+    {{/* . is an index number of deployments (starting from 0) */}}
+    {{- $max_hash := 4294967296 }}
+    {{- $sensor_range_size := div (add1 $max_hash) $sensor_deployments_count }}
+    {{- $sensor_range_start := mul . $sensor_range_size }}
+    {{- $sensor_range_end := add $sensor_range_start $sensor_range_size -1 }}
+    {{/* deal with integer math rounding errors by moving the last range_end to the max_hash */}}
+    {{- $sensor_range_end := eq (add1 .) $sensor_deployments_count | ternary $max_hash $sensor_range_end }}
+    {{- $_ := set $sensor "hash_ranges" (print $sensor_range_start ".." $sensor_range_end) }}
+    {{- $_ := set $sensor "name" (. | toString) }}
   {{- end }}
   {{- $name := print "st2sensorcontainer" (include "stackstorm-ha.hyphenPrefix" $sensor.name) }}
 ---
@@ -1196,8 +1212,8 @@ spec:
   # https://docs.stackstorm.com/reference/ha.html#st2sensorcontainer
   # It is possible to run st2sensorcontainer(s) in one of these modes:
   #   (1) run all sensors in one pod (1 deployment with 1 pod, the default); or
-  #   (2) run one sensor per pod using st2.packs.sensors.
-  # It is possible to run st2sensorcontainer in HA mode by running one process on each compute instance.
+  #   (2) run multiple sensors per pod (2+ deployments with 1 pod each) using hash range partitions; or
+  #   (3) run one sensor per pod using st2.packs.sensors.
   # Each sensor node needs to be provided with proper partition information to share work with other sensor nodes
   # so that the same sensor does not run on different nodes. See: https://docs.stackstorm.com/reference/sensor_partitioning.html
   replicas: 1
@@ -1219,6 +1235,9 @@ spec:
         {{- end }}
         {{- if $one_sensor_per_pod }}
         stackstorm/sensor-mode: one-sensor-per-pod
+        {{- else if $some_sensors_per_pod }}
+        stackstorm/sensor-mode: multiple-sensors-per-pod
+        stackstorm/sensor-hash-ranges: {{ $sensor.hash_ranges | quote }}
         {{- else }}
         stackstorm/sensor-mode: all-sensors-in-one-pod
         {{- end }}
@@ -1234,6 +1253,23 @@ spec:
         {{- include "stackstorm-ha.packs-pullSecrets" $ | nindent 6 }}
       {{- end }}
       initContainers:
+      {{- if $some_sensors_per_pod }}
+      - name: {{ $name }}-init-config
+        image: busybox:1.28
+        volumeMounts:
+        - name: st2-sensor-config-vol
+          mountPath: /tmp/st2
+        command:
+          - 'sh'
+          - '-ec'
+          - |
+              cat <<EOT > /tmp/st2/st2.sensorcontainer.conf
+              [sensorcontainer]
+              sensor_node_name = {{ $name }}
+              partition_provider = name:hash, hash_ranges:{{ $sensor.hash_ranges }}
+              single_sensor_mode = False
+              EOT
+      {{- end }}
       {{- include "stackstorm-ha.init-containers-wait-for-db" $ | nindent 6 }}
       {{- include "stackstorm-ha.init-containers-wait-for-mq" $ | nindent 6 }}
       {{- if and $.Values.st2.packs.images (not $.Values.st2.packs.volumes.enabled) }}
@@ -1255,14 +1291,19 @@ spec:
         livenessProbe:
           {{- toYaml . | nindent 10 }}
         {{- end }}
-        {{- if $one_sensor_per_pod }}
+        {{- if or $one_sensor_per_pod $some_sensors_per_pod }}{{/* ie: when there is more than one pod of sensors */}}
         command:
           - /opt/stackstorm/st2/bin/st2sensorcontainer
           - --config-file=/etc/st2/st2.conf
           - --config-file=/etc/st2/st2.docker.conf
           - --config-file=/etc/st2/st2.user.conf
+          {{- if $one_sensor_per_pod }}{{/* only in st2.packs.sensors[] */}}
           - --single-sensor-mode
           - --sensor-ref={{ required "You must define `ref` for everything in st2.packs.sensors. This assigns each sensor to a pod." $sensor.ref }}
+          {{- else if $some_sensors_per_pod }}
+          # injected by {{ $name }}-init-config
+          - --config-file=/etc/st2/st2.sensorcontainer.conf
+          {{- end }}
         {{- end }}
         {{- if $sensor.env }}
         env: {{- include "stackstorm-ha.customEnv" . | nindent 8 }}
@@ -1277,6 +1318,12 @@ spec:
         volumeMounts:
         {{- include "stackstorm-ha.st2-config-volume-mounts" $ | nindent 8 }}
         {{- include "stackstorm-ha.packs-volume-mounts" $ | nindent 8 }}
+        {{- if $some_sensors_per_pod }}
+        - name: st2-sensor-config-vol
+          mountPath: /etc/st2/st2.sensorcontainer.conf
+          subPath: st2.sensorcontainer.conf
+          readOnly: true
+        {{- end }}
         {{- if ne "disable" (default "" $.Values.st2.datastore_crypto_key) }}
         - name: st2-encryption-key-vol
           mountPath: /etc/st2/keys
@@ -1311,6 +1358,11 @@ spec:
         {{- end }}
         {{- include "stackstorm-ha.st2-config-volume" $ | nindent 8 }}
         {{- include "stackstorm-ha.packs-volumes" $ | nindent 8 }}
+        {{- if $some_sensors_per_pod }}
+        - name: st2-sensor-config-vol
+          emptyDir: # This is for a tiny file
+            medium: Memory
+        {{- end }}
         {{- range $sensor.extra_volumes }}
         - name: {{ required "Each volume must have a 'name' in $sensor.extra_volumes" .name }}
           {{- tpl (required "Each volume must have a 'volume' definition in $sensor.extra_volumes" .volume | toYaml) $ | nindent 10 }}

tests/unit/st2sensors_test.yaml

@@ -247,3 +247,154 @@ tests:
     asserts:
       - failedTemplate:
           errorMessage: "You must define `ref` for everything in st2.packs.sensors. This assigns each sensor to a pod."
+
+  - it: stackstorm/sensor-mode = multiple-sensors-per-pod
+    template: deployments.yaml
+    release:
+      name: multi-sensor-release
+    set:
+      st2:
+        packs: { sensors: [] }
+      image:
+        tag: globaldefault
+      st2sensorcontainer:
+        image:
+          tag: sensordefault
+        deployments: 3
+        securityContext: *override_security_context
+      securityContext: *global_security_context
+    asserts:
+      - hasDocuments:
+          count: 15 # all but st2chatops
+
+      - isKind:
+          of: Deployment
+        documentIndex: *first_sensor_doc
+      - isKind:
+          of: Deployment
+        documentIndex: *second_sensor_doc
+      - isKind:
+          of: Deployment
+        documentIndex: *third_sensor_doc
+
+      - equal:
+          path: metadata.name
+          value: multi-sensor-release-st2sensorcontainer-0
+        documentIndex: *first_sensor_doc
+      - equal:
+          path: metadata.name
+          value: multi-sensor-release-st2sensorcontainer-1
+        documentIndex: *second_sensor_doc
+      - equal:
+          path: metadata.name
+          value: multi-sensor-release-st2sensorcontainer-2
+        documentIndex: *third_sensor_doc
+
+      - equal:
+          path: metadata.labels.app
+          value: st2sensorcontainer-0
+        documentIndex: *first_sensor_doc
+      - equal:
+          path: metadata.labels.app
+          value: st2sensorcontainer-1
+        documentIndex: *second_sensor_doc
+      - equal:
+          path: metadata.labels.app
+          value: st2sensorcontainer-2
+        documentIndex: *third_sensor_doc
+
+      - equal:
+          path: spec.template.metadata.labels.app
+          value: st2sensorcontainer-0
+        documentIndex: *first_sensor_doc
+      - equal:
+          path: spec.template.metadata.labels.app
+          value: st2sensorcontainer-1
+        documentIndex: *second_sensor_doc
+      - equal:
+          path: spec.template.metadata.labels.app
+          value: st2sensorcontainer-2
+        documentIndex: *third_sensor_doc
+
+      - equal:
+          path: spec.selector.matchLabels.app
+          value: st2sensorcontainer-0
+        documentIndex: *first_sensor_doc
+      - equal:
+          path: spec.selector.matchLabels.app
+          value: st2sensorcontainer-1
+        documentIndex: *second_sensor_doc
+      - equal:
+          path: spec.selector.matchLabels.app
+          value: st2sensorcontainer-2
+        documentIndex: *third_sensor_doc
+
+      - equal: *oneReplica
+        documentIndex: *first_sensor_doc
+      - equal: *oneReplica
+        documentIndex: *second_sensor_doc
+      - equal: *oneReplica
+        documentIndex: *third_sensor_doc
+
+      - equal: &multiSensorAnnotation
+          path: spec.template.metadata.annotations.stackstorm/sensor-mode
+          value: multiple-sensors-per-pod
+        documentIndex: *first_sensor_doc
+      - equal: *multiSensorAnnotation
+        documentIndex: *second_sensor_doc
+      - equal: *multiSensorAnnotation
+        documentIndex: *third_sensor_doc
+
+      - notContains: &notSingleSensorMode
+          path: spec.template.spec.containers[0].command
+          content: '--single-sensor-mode'
+        documentIndex: *first_sensor_doc
+      - notContains: *notSingleSensorMode
+        documentIndex: *second_sensor_doc
+      - notContains: *notSingleSensorMode
+        documentIndex: *third_sensor_doc
+
+      - contains: &sensorConf
+          path: spec.template.spec.containers[0].command
+          content: '--config-file=/etc/st2/st2.sensorcontainer.conf'
+        documentIndex: *first_sensor_doc
+      - contains: *sensorConf
+        documentIndex: *second_sensor_doc
+      - contains: *sensorConf
+        documentIndex: *third_sensor_doc
+
+      # st2sensorcontainer values apply to all sensor containers replacing globals
+
+      - matchRegex: &multiSensorTag
+          path: spec.template.spec.containers[0].image
+          pattern: '.*:sensordefault$'
+        documentIndex: *first_sensor_doc
+      - matchRegex: *multiSensorTag
+        documentIndex: *second_sensor_doc
+      - matchRegex: *multiSensorTag
+        documentIndex: *third_sensor_doc
+
+      - equal: &multiSensorSecurityContext
+          path: spec.template.spec.containers[0].securityContext
+          value: *override_security_context
+        documentIndex: *first_sensor_doc
+      - equal: *multiSensorSecurityContext
+        documentIndex: *second_sensor_doc
+      - equal: *multiSensorSecurityContext
+        documentIndex: *third_sensor_doc
+
+  - it: ambiguous stackstorm/sensor-mode fails
+    template: deployments.yaml
+    release:
+      name: missing-sensor-ref-release
+    set:
+      st2:
+        packs:
+          sensors:
+            - name: foo
+              ref: some_pack.foo_sensor
+      st2sensorcontainer:
+        deployments: 2
+    asserts:
+      - failedTemplate:
+          errorMessage: "The sensor values are ambiguous. To use one-sensor-per-pod, use `st2.packs.sensors`. To use multiple-sensors-per-pod, use `st2sensorcontainer.deployments > 1`. Do not use both!"

values.yaml

@@ -152,10 +152,11 @@ st2:
     # https://docs.stackstorm.com/reference/ha.html#st2sensorcontainer
     # It is possible to run st2sensorcontainer(s) in one of these modes:
     #   (1) run all sensors in one pod (1 deployment with 1 pod, the default); or
-    #   (2) run one sensor per pod using st2.packs.sensors (here).
+    #   (2) run multiple sensors per pod (2+ deployments with 1 pod each) using hash range partitions; or
+    #   (3) run one sensor per pod using st2.packs.sensors (here).
     # Each sensor node needs to be provided with proper partition information to share work with other sensor
     # nodes so that the same sensor does not run on different nodes.
-    # When this is empty (the default), the chart adds one pod to run all sensors.
+    # By default, this is empty and st2sensorcontainer.deployments (below) is 1 so that there is one pod to run all sensors.
     sensors: []
       # This is a list of sensor pods (stackstorm/sensor-mode=one-sensor-per-pod).
       # Each entry should have `name` (the pod name) and `ref` (which sensor to run in the pod).
@@ -655,10 +656,21 @@ st2actionrunner:
 # https://docs.stackstorm.com/reference/ha.html#st2sensorcontainer
 # It is possible to run st2sensorcontainer(s) in one of these modes:
 #   (1) run all sensors in one pod (1 deployment with 1 pod, the default); or
-#   (2) run one sensor per pod using st2.packs.sensors (see above).
-# To use the default mode (all sensors in one pod), st2.packs.sensors must be empty.
+#   (2) run multiple sensor per pod (2+ deployments with 1 pod each) using hash range partitions; or
+#   (3) run one sensor per pod using st2.packs.sensors (see above).
+# To use the deployments (modes 1 and 2 in this list), st2.packs.sensors must be empty.
 # For one-sensor-per-pod, define defaults here and add config for each sensor to st2.packs.sensors (above).
 st2sensorcontainer:
+  # "deployments" is analogous to "replicas" for other services. Each deployment has 1 replica.
+  # The deployments count cannot be greater than 1 unless st2.packs.sensors is empty.
+  # If deployments is 1 (the default), then StackStorm puts all sensors in one pod.
+  # If deployments is 2+, StackStorm is configured to use sensor hash ranges to ensure that
+  # each sensor runs on only one pod. Each hash range is assigned to a separate
+  # deployment to ease maintenance of the sensor deployments and pods.
+  # See: https://docs.stackstorm.com/reference/sensor_partitioning.html#hash
+  deployments: 1
+
+  # values below this point are also used as defaults in st2.packs.sensors (if used)
   resources:
     requests:
       memory: "100Mi"