Merge pull request #51 from StackStorm/feat/partitionsensors

Allow st2sensorcontainer to be partitioned

Author: warrenvw

CHANGELOG.md

@@ -1,7 +1,10 @@
 # Changelog
 
 ## In Development
+
+## v0.10.0
 * Bump versions of all dependencies (#50)
+* Allow st2sensorcontainer to be partitioned (#51)
 * Replace single-node `etcd` coordination backend with 3-node etcd HA cluster, deployed as a Helm dependency (#52)
 * Fixed improper job load order for enterprise edition failing due to missing RBAC roles & assignments (#53)
 

README.md

@@ -112,12 +112,27 @@ In an HA deployment there must be a minimum of `2` replicas of st2notifier runni
 which in our case is `etcd`.
 
 ### [st2sensorcontainer](https://docs.stackstorm.com/reference/ha.html#st2sensorcontainer)
-st2sensorcontainer manages StackStorm sensors: starts, stops and restarts them as a subprocesses.
-At the moment K8s configuration consists of Deployment with hardcoded `1` replica.
-Future plans are to re-work this setup and benefit from Docker-friendly [single-sensor-per-container mode #4179](https://github.com/StackStorm/st2/pull/4179)
-(since st2 `v2.9`) as a way of [Sensor Partitioning](https://docs.stackstorm.com/latest/reference/sensor_partitioning.html), distributing the computing load
-between many pods and relying on K8s failover/reschedule mechanisms, instead of running everything on `1` single instance of st2sensorcontainer.
+st2sensorcontainer manages StackStorm sensors: It starts, stops and restarts them as subprocesses.
+By default, deployment is configured with `1` replica containing all the sensors.
 
+st2sensorcontainer also supports a more Docker-friendly single-sensor-per-container mode as a way of
+[Sensor Partitioning](https://docs.stackstorm.com/latest/reference/sensor_partitioning.html). This
+distributes the computing load between many pods and relies on K8s failover/reschedule mechanisms,
+instead of running everything on a single instance of st2sensorcontainer. The sensor(s) must be
+deployed as part of the custom packs image.
+
+As an example, override the default Helm values as follows:
+
+```
+st2:
+  packs:
+    sensors:
+      - name: github
+        ref: githubwebhook.GitHubWebhookSensor
+      - name: circleci
+        ref: circle_ci.CircleCIWebhookSensor
+```
+	
 ### [st2actionrunner](https://docs.stackstorm.com/reference/ha.html#st2actionrunner)
 Stackstorm workers that actually execute actions.
 `5` replicas for K8s Deployment are configured by default to increase StackStorm ability to execute actions without excessive queuing.

templates/_helpers.tpl

@@ -27,3 +27,8 @@ stackstorm
 {{- define "enterpriseSuffix" -}}
 {{ if required "Missing context '.Values.enterprise.enabled'!" .Values.enterprise.enabled }}-enterprise{{ end }}
 {{- end -}}
+
+# Generate '-' prefix only when the variable is defined
+{{- define "hyphenPrefix" -}}
+{{ if . }}-{{ . }}{{end}}
+{{- end -}}

templates/deployments.yaml

@@ -780,56 +780,56 @@ spec:
 {{ toYaml . | indent 8 }}
     {{- end }}
 
+{{- range .Values.st2.packs.sensors }}
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ .Release.Name }}-st2sensorcontainer{{ template "enterpriseSuffix" . }}
+  name: {{ $.Release.Name }}-st2sensorcontainer{{ template "hyphenPrefix" .name }}{{ template "enterpriseSuffix" $ }}
   labels:
-    app: st2sensorcontainer
+    app: st2sensorcontainer{{ template "hyphenPrefix" .name }}
     tier: backend
     vendor: stackstorm
-    support: {{ template "supportMethod" . }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
+    support: {{ template "supportMethod" $ }}
+    chart: {{ $.Chart.Name }}-{{ $.Chart.Version }}
+    release: {{ $.Release.Name }}
+    heritage: {{ $.Release.Service }}
 spec:
   selector:
     matchLabels:
-      app: st2sensorcontainer
-      support: {{ template "supportMethod" . }}
-      release: {{ .Release.Name }}
+      app: st2sensorcontainer{{ template "hyphenPrefix" .name }}
+      support: {{ template "supportMethod" $ }}
+      release: {{ $.Release.Name }}
   # https://docs.stackstorm.com/reference/ha.html#st2sensorcontainer
   # It is possible to run st2sensorcontainer in HA mode by running one process on each compute instance. Each sensor node needs to be
   # provided with proper partition information to share work with other sensor nodes so that the same sensor does not run on different nodes.
   # See Partitioning Sensors for information on how to partition sensors.
-  # TODO: Re-work to use single-sensor-per-container mode instead of running 1 node. Proper implementation is possible with Helm templating (#4)
   replicas: 1
   template:
     metadata:
       labels:
-        app: st2sensorcontainer
+        app: st2sensorcontainer{{ template "hyphenPrefix" .name }}
         tier: backend
         vendor: stackstorm
-        support: {{ template "supportMethod" . }}
-        chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-        release: {{ .Release.Name }}
-        heritage: {{ .Release.Service }}
+        support: {{ template "supportMethod" $ }}
+        chart: {{ $.Chart.Name }}-{{ $.Chart.Version }}
+        release: {{ $.Release.Name }}
+        heritage: {{ $.Release.Service }}
       annotations:
-        checksum/config: {{ include (print $.Template.BasePath "/configmaps_st2-conf.yaml") . | sha256sum }}
-        checksum/packs: {{ include (print $.Template.BasePath "/configmaps_packs.yaml") . | sha256sum }}
+        checksum/config: {{ include (print $.Template.BasePath "/configmaps_st2-conf.yaml") $ | sha256sum }}
+        checksum/packs: {{ include (print $.Template.BasePath "/configmaps_packs.yaml") $ | sha256sum }}
     spec:
-      {{- if .Values.enterprise.enabled }}
+      {{- if $.Values.enterprise.enabled }}
       imagePullSecrets:
-      - name: {{ .Release.Name }}-st2-license
+      - name: {{ $.Release.Name }}-st2-license
       {{- end }}
-      {{- if .Values.st2.packs.image.repository }}
+      {{- if $.Values.st2.packs.image.repository }}
       initContainers:
       # Merge packs and virtualenvs from st2sensorcontainer with those from the st2.packs image
       # Custom packs
       - name: st2-custom-packs
-        image: "{{ .Values.st2.packs.image.repository }}/{{ .Values.st2.packs.image.name }}:{{ .Values.st2.packs.image.tag }}"
-        imagePullPolicy: {{ .Values.st2.packs.image.pullPolicy | quote }}
+        image: "{{ $.Values.st2.packs.image.repository }}/{{ $.Values.st2.packs.image.name }}:{{ $.Values.st2.packs.image.tag }}"
+        imagePullPolicy: {{ $.Values.st2.packs.image.pullPolicy | quote }}
         volumeMounts:
         - name: st2-packs-vol
           mountPath: /opt/stackstorm/packs-shared
@@ -843,8 +843,8 @@ spec:
             /bin/cp -aR /opt/stackstorm/virtualenvs/. /opt/stackstorm/virtualenvs-shared
       # System packs
       - name: st2-system-packs
-        image: "{{ template "imageRepository" . }}/st2actionrunner{{ template "enterpriseSuffix" . }}:{{ .Chart.AppVersion }}"
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        image: "{{ template "imageRepository" $ }}/st2actionrunner{{ template "enterpriseSuffix" $ }}:{{ $.Chart.AppVersion }}"
+        imagePullPolicy: {{ $.Values.image.pullPolicy }}
         volumeMounts:
         - name: st2-packs-vol
           mountPath: /opt/stackstorm/packs-shared
@@ -856,23 +856,38 @@ spec:
             /bin/cp -aR /opt/stackstorm/virtualenvs/. /opt/stackstorm/virtualenvs-shared
       {{- end }}
       containers:
-      - name: st2sensorcontainer{{ template "enterpriseSuffix" . }}
-        image: "{{ template "imageRepository" . }}/st2sensorcontainer{{ template "enterpriseSuffix" . }}:{{ .Chart.AppVersion }}"
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
-        # TODO: Add liveness/readiness probes (#3)
-        #livenessProbe:
-        #readinessProbe:
+      - name: st2sensorcontainer{{ template "hyphenPrefix" .name }}{{ template "enterpriseSuffix" $ }}
+        image: "{{ template "imageRepository" $ }}/st2sensorcontainer{{ template "enterpriseSuffix" $ }}:{{ $.Chart.AppVersion }}"
+        imagePullPolicy: {{ $.Values.image.pullPolicy }}
+        {{- with .readinessProbe }}
+        # Probe to check if app is running. Failure will lead to a pod restart.
+        readinessProbe:
+{{ toYaml . | indent 10 }}
+        {{- end }}
+        {{- with .livenessProbe }}
+        livenessProbe:
+{{ toYaml . | indent 10 }}
+        {{- end }}
+        {{- if .ref }}
+        command:
+          - /opt/stackstorm/st2/bin/st2sensorcontainer
+          - --config-file=/etc/st2/st2.conf
+          - --config-file=/etc/st2/st2.docker.conf
+          - --config-file=/etc/st2/st2.user.conf
+          - --single-sensor-mode
+          - --sensor-ref={{ .ref }}
+        {{- end }}
         envFrom:
         - configMapRef:
-            name: {{ .Release.Name }}-st2-urls
+            name: {{ $.Release.Name }}-st2-urls
         volumeMounts:
         - name: st2-config-vol
           mountPath: /etc/st2/st2.docker.conf
           subPath: st2.docker.conf
         - name: st2-config-vol
           mountPath: /etc/st2/st2.user.conf
           subPath: st2.user.conf
-        {{- if .Values.st2.packs.image.repository }}
+        {{- if $.Values.st2.packs.image.repository }}
         - name: st2-packs-vol
           mountPath: /opt/stackstorm/packs
           readOnly: true
@@ -881,29 +896,30 @@ spec:
           readOnly: true
         {{- end }}
         resources:
-{{ toYaml .Values.st2sensorcontainer.resources | indent 10 }}
+{{ toYaml .resources | indent 10 }}
       volumes:
         - name: st2-config-vol
           configMap:
-            name: {{ .Release.Name }}-st2-config
-        {{- if .Values.st2.packs.image.repository }}
+            name: {{ $.Release.Name }}-st2-config
+        {{- if $.Values.st2.packs.image.repository }}
         - name: st2-packs-vol
           emptyDir: {}
         - name: st2-virtualenvs-vol
           emptyDir: {}
         {{- end }}
-    {{- with .Values.st2sensorcontainer.nodeSelector }}
+    {{- with .nodeSelector }}
       nodeSelector:
 {{ toYaml . | indent 8 }}
     {{- end }}
-    {{- with .Values.st2sensorcontainer.affinity }}
+    {{- with .affinity }}
       affinity:
 {{ toYaml . | indent 8 }}
     {{- end }}
-    {{- with .Values.st2sensorcontainer.tolerations }}
+    {{- with .tolerations }}
       tolerations:
 {{ toYaml . | indent 8 }}
     {{- end }}
+{{- end }}
 
 ---
 apiVersion: apps/v1

values.yaml

@@ -85,6 +85,24 @@ st2:
       name: st2packs
       tag: latest
       pullPolicy: Always
+
+    # https://docs.stackstorm.com/reference/ha.html#st2sensorcontainer
+    # It is possible to run st2sensorcontainer in HA mode by running one process on each compute instance.
+    # Each sensor node needs to be provided with proper partition information to share work with other sensor
+    # nodes so that the same sensor does not run on different nodes.
+    sensors:
+      # Specify default container that executes all sensors.
+      # To partition sensors with one sensor per node, override st2.packs.sensors.
+      # NOTE: Do not modify this file.
+      - name:
+        livenessProbe: {}
+        readinessProbe: {}
+        # TODO: Find out recommended/default resources for this specific service (#5)
+        resources: {}
+        # Additional advanced settings to control pod/deployment placement
+        affinity: {}
+        nodeSelector: {}
+        tolerations: []
   # Import data into StackStorm's Key/Value datastore (https://docs.stackstorm.com/datastore.html)
   keyvalue:
     #- name: st2_version
@@ -314,19 +332,6 @@ st2notifier:
   nodeSelector: {}
   tolerations: []
   affinity: {}
-# https://docs.stackstorm.com/reference/ha.html#st2sensorcontainer
-# It is possible to run st2sensorcontainer in HA mode by running one process on each compute instance. Each sensor node needs to be
-# provided with proper partition information to share work with other sensor nodes so that the same sensor does not run on different nodes.
-st2sensorcontainer:
-  # TODO: Re-work to use single-sensor-per-container mode partitioning instead of running 1 single node of st2sensorcontainer. Proper implementation is now possible with Helm templating (#4)
-  # NB! Number of replicas are hardcoded to 1, see above T0D0 about using single-sensor-per-container mode in future as way of Sensor Partitioning.
-  # replicas: 1
-  # TODO: Find out recommended/default resources for this specific service (#5)
-  resources: {}
-  # Additional advanced settings to control pod/deployment placement
-  nodeSelector: {}
-  tolerations: []
-  affinity: {}
 # https://docs.stackstorm.com/reference/ha.html#st2actionrunner
 # Multiple st2actionrunner processes can run in active-active with only connections to MongoDB and RabbitMQ. Work gets naturally
 # distributed across runners via RabbitMQ. Adding more st2actionrunner processes increases the ability of StackStorm to execute actions.