feat: add livenessProbe and readinessProbe to all services

.changeset/pretty-ants-arrive.md

@@ -0,0 +1,5 @@
+---
+"helm-charts": patch
+---
+
+feat: add livenessProbe and readinessProbe for services

charts/hdx-oss-v2/templates/clickhouse-deployment.yaml

@@ -35,6 +35,26 @@ spec:
           env:
             - name: CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT
               value: "1"
+          {{- if .Values.clickhouse.livenessProbe.enabled }}
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: {{ .Values.clickhouse.port }}
+            initialDelaySeconds: {{ .Values.clickhouse.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.clickhouse.livenessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.clickhouse.livenessProbe.timeoutSeconds }}
+            failureThreshold: {{ .Values.clickhouse.livenessProbe.failureThreshold }}
+          {{- end }}
+          {{- if .Values.clickhouse.readinessProbe.enabled }}
+          readinessProbe:
+            httpGet:
+              path: /ping
+              port: {{ .Values.clickhouse.port }}
+            initialDelaySeconds: {{ .Values.clickhouse.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.clickhouse.readinessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.clickhouse.readinessProbe.timeoutSeconds }}
+            failureThreshold: {{ .Values.clickhouse.readinessProbe.failureThreshold }}
+          {{- end }}
           volumeMounts:
             - name: config
               mountPath: /etc/clickhouse-server/config.xml
@@ -153,4 +173,4 @@ spec:
     requests:
       storage: {{ .Values.clickhouse.persistence.logSize }}
 {{- end }}
-{{- end }}
+{{- end }}

charts/hdx-oss-v2/templates/hyperdx-deployment.yaml

@@ -64,6 +64,26 @@ spec:
           resources:
             {{- toYaml .Values.hyperdx.resources | nindent 12 }}
           {{- end }}
+          {{- if .Values.hyperdx.livenessProbe.enabled }}
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: {{ .Values.hyperdx.apiPort }}
+            initialDelaySeconds: {{ .Values.hyperdx.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.hyperdx.livenessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.hyperdx.livenessProbe.timeoutSeconds }}
+            failureThreshold: {{ .Values.hyperdx.livenessProbe.failureThreshold }}
+          {{- end }}
+          {{- if .Values.hyperdx.readinessProbe.enabled }}
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: {{ .Values.hyperdx.apiPort }}
+            initialDelaySeconds: {{ .Values.hyperdx.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.hyperdx.readinessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.hyperdx.readinessProbe.timeoutSeconds }}
+            failureThreshold: {{ .Values.hyperdx.readinessProbe.failureThreshold }}
+          {{- end }}
           envFrom:
             - configMapRef:
                 name: {{ include "hdx-oss.fullname" . }}-app-config

charts/hdx-oss-v2/templates/mongodb-deployment.yaml

@@ -31,6 +31,24 @@ spec:
           image: "{{ .Values.mongodb.image }}"
           ports:
             - containerPort: {{ .Values.mongodb.port }}
+          {{- if .Values.mongodb.livenessProbe.enabled }}
+          livenessProbe:
+            tcpSocket:
+              port: {{ .Values.mongodb.port }}
+            initialDelaySeconds: {{ .Values.mongodb.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.mongodb.livenessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.mongodb.livenessProbe.timeoutSeconds }}
+            failureThreshold: {{ .Values.mongodb.livenessProbe.failureThreshold }}
+          {{- end }}
+          {{- if .Values.mongodb.readinessProbe.enabled }}
+          readinessProbe:
+            tcpSocket:
+              port: {{ .Values.mongodb.port }}
+            initialDelaySeconds: {{ .Values.mongodb.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.mongodb.readinessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.mongodb.readinessProbe.timeoutSeconds }}
+            failureThreshold: {{ .Values.mongodb.readinessProbe.failureThreshold }}
+          {{- end }}
           volumeMounts:
             - name: mongodb-data
               mountPath: /data/db

charts/hdx-oss-v2/templates/otel-collector-deployment.yaml

@@ -46,6 +46,26 @@ spec:
           resources:
             {{- toYaml .Values.otel.resources | nindent 12 }}
           {{- end }}
+          {{- if .Values.otel.livenessProbe.enabled }}
+          livenessProbe:
+            httpGet:
+              path: /
+              port: {{ .Values.otel.port }}
+            initialDelaySeconds: {{ .Values.otel.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.otel.livenessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.otel.livenessProbe.timeoutSeconds }}
+            failureThreshold: {{ .Values.otel.livenessProbe.failureThreshold }}
+          {{- end }}
+          {{- if .Values.otel.readinessProbe.enabled }}
+          readinessProbe:
+            httpGet:
+              path: /
+              port: {{ .Values.otel.port }}
+            initialDelaySeconds: {{ .Values.otel.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.otel.readinessProbe.periodSeconds}}
+            timeoutSeconds: {{ .Values.otel.readinessProbe.timeoutSeconds}}
+            failureThreshold: {{ .Values.otel.readinessProbe.failureThreshold }}
+          {{- end }}
           env:
             - name: CLICKHOUSE_ENDPOINT
               value: "{{ .Values.otel.clickhouseEndpoint | default (printf "tcp://%s-clickhouse:%v?dial_timeout=10s" (include "hdx-oss.fullname" .) .Values.clickhouse.nativePort) }}"

charts/hdx-oss-v2/tests/clickhouse-deployment_test.yaml

@@ -145,4 +145,125 @@ tests:
           path: spec.template.spec.volumes[3].emptyDir
       - documentIndex: 0
         isNull:
-          path: spec.template.spec.volumes[3].persistentVolumeClaim
+          path: spec.template.spec.volumes[3].persistentVolumeClaim
+
+  - it: should include livenessProbe with default values when enabled
+    set:
+      clickhouse:
+        enabled: true
+        port: 8123
+    asserts:
+      - documentIndex: 0
+        isSubset:
+          path: spec.template.spec.containers[0].livenessProbe
+          content:
+            httpGet:
+              path: /ping
+              port: 8123
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+
+  - it: should include readinessProbe with default values when enabled
+    set:
+      clickhouse:
+        enabled: true
+        port: 8123
+    asserts:
+      - documentIndex: 0
+        isSubset:
+          path: spec.template.spec.containers[0].readinessProbe
+          content:
+            httpGet:
+              path: /ping
+              port: 8123
+            initialDelaySeconds: 1
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+
+  - it: should not include livenessProbe when disabled
+    set:
+      clickhouse:
+        enabled: true
+        livenessProbe:
+          enabled: false
+    asserts:
+      - documentIndex: 0
+        isNull:
+          path: spec.template.spec.containers[0].livenessProbe
+
+  - it: should not include readinessProbe when disabled
+    set:
+      clickhouse:
+        enabled: true
+        readinessProbe:
+          enabled: false
+    asserts:
+      - documentIndex: 0
+        isNull:
+          path: spec.template.spec.containers[0].readinessProbe
+
+  - it: should use custom livenessProbe values when provided
+    set:
+      clickhouse:
+        enabled: true
+        port: 8123
+        livenessProbe:
+          enabled: true
+          initialDelaySeconds: 20
+          periodSeconds: 60
+          timeoutSeconds: 10
+          failureThreshold: 5
+    asserts:
+      - documentIndex: 0
+        isSubset:
+          path: spec.template.spec.containers[0].livenessProbe
+          content:
+            httpGet:
+              path: /ping
+              port: 8123
+            initialDelaySeconds: 20
+            periodSeconds: 60
+            timeoutSeconds: 10
+            failureThreshold: 5
+
+  - it: should use custom readinessProbe values when provided
+    set:
+      clickhouse:
+        enabled: true
+        port: 8123
+        readinessProbe:
+          enabled: true
+          initialDelaySeconds: 5
+          periodSeconds: 20
+          timeoutSeconds: 3
+          failureThreshold: 2
+    asserts:
+      - documentIndex: 0
+        isSubset:
+          path: spec.template.spec.containers[0].readinessProbe
+          content:
+            httpGet:
+              path: /ping
+              port: 8123
+            initialDelaySeconds: 5
+            periodSeconds: 20
+            timeoutSeconds: 3
+            failureThreshold: 2
+
+  - it: should use custom port in probes when provided
+    set:
+      clickhouse:
+        enabled: true
+        port: 8124
+    asserts:
+      - documentIndex: 0
+        equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.port
+          value: 8124
+      - documentIndex: 0
+        equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.port
+          value: 8124

charts/hdx-oss-v2/tests/hyperdx-deployment_test.yaml

@@ -128,4 +128,102 @@ tests:
           content: -c
       - matchRegex:
           path: spec.template.spec.initContainers[0].command[2]
-          pattern: "until nc -z .+-mongodb [0-9]+; do echo waiting for mongodb; sleep 2; done;"
+          pattern: "until nc -z .+-mongodb [0-9]+; do echo waiting for mongodb; sleep 2; done;"
+
+  - it: should include livenessProbe with default values when enabled
+    asserts:
+      - isSubset:
+          path: spec.template.spec.containers[0].livenessProbe
+          content:
+            httpGet:
+              path: /health
+              port: 8000
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+
+  - it: should include readinessProbe with default values when enabled
+    asserts:
+      - isSubset:
+          path: spec.template.spec.containers[0].readinessProbe
+          content:
+            httpGet:
+              path: /health
+              port: 8000
+            initialDelaySeconds: 1
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+
+  - it: should not include livenessProbe when disabled
+    set:
+      hyperdx:
+        livenessProbe:
+          enabled: false
+    asserts:
+      - isNull:
+          path: spec.template.spec.containers[0].livenessProbe
+
+  - it: should not include readinessProbe when disabled
+    set:
+      hyperdx:
+        readinessProbe:
+          enabled: false
+    asserts:
+      - isNull:
+          path: spec.template.spec.containers[0].readinessProbe
+
+  - it: should use custom livenessProbe values when provided
+    set:
+      hyperdx:
+        livenessProbe:
+          enabled: true
+          initialDelaySeconds: 20
+          periodSeconds: 60
+          timeoutSeconds: 10
+          failureThreshold: 5
+    asserts:
+      - isSubset:
+          path: spec.template.spec.containers[0].livenessProbe
+          content:
+            httpGet:
+              path: /health
+              port: 8000
+            initialDelaySeconds: 20
+            periodSeconds: 60
+            timeoutSeconds: 10
+            failureThreshold: 5
+
+  - it: should use custom readinessProbe values when provided
+    set:
+      hyperdx:
+        readinessProbe:
+          enabled: true
+          initialDelaySeconds: 5
+          periodSeconds: 20
+          timeoutSeconds: 3
+          failureThreshold: 2
+    asserts:
+      - isSubset:
+          path: spec.template.spec.containers[0].readinessProbe
+          content:
+            httpGet:
+              path: /health
+              port: 8000
+            initialDelaySeconds: 5
+            periodSeconds: 20
+            timeoutSeconds: 3
+            failureThreshold: 2
+
+  - it: should use custom apiPort in probes when provided
+    set:
+      hyperdx:
+        apiPort: 9000
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.port
+          value: 9000
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.port
+          value: 9000