Skip to content
This repository was archived by the owner on Jun 22, 2024. It is now read-only.

Commit 56f3c49

Browse files
VietND96VietND96
authored
update(chart): Node preStop and startupProbe in autoscaling Deployment (SeleniumHQ#2139)
update(chart): script for Node preStop and startupProbe in autoscaling Deployment Signed-off-by: Viet Nguyen Duc <[email protected]>
1 parent cd10250 commit 56f3c49

File tree

14 files changed

+201
-49
lines changed

14 files changed

+201
-49
lines changed

‎.github/workflows/helm-chart-test.yml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@ on:
1515
description: 'Test parameter for different request timeout'
1616
required: false
1717
default: '370'
18+
cluster:
19+
description: 'Cluster type to test (kind, minikube)'
20+
required: false
1821
schedule:
1922
- cron: '0 0 * * *'
2023

@@ -33,6 +36,8 @@ jobs:
3336
- job_https
3437
- deployment
3538
- deployment_https
39+
env:
40+
CLUSTER: ${{ github.event.inputs.cluster || 'minikube' }}
3641
steps:
3742
- uses: actions/checkout@main
3843
- name: Output Docker info
@@ -63,7 +68,7 @@ jobs:
6368
with:
6469
timeout_minutes: 10
6570
max_attempts: 3
66-
command: make chart_setup_env
71+
command: CLUSTER=${CLUSTER} make chart_setup_env
6772
- name: Build Helm charts
6873
run: |
6974
BUILD_DATE=${BUILD_DATE} make chart_build
@@ -76,7 +81,7 @@ jobs:
7681
with:
7782
timeout_minutes: 10
7883
max_attempts: 3
79-
command: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_cluster_setup
84+
command: CLUSTER=${CLUSTER} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_cluster_setup
8085
- name: Build Docker images
8186
run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build
8287
- name: Test set custom CA certificate
@@ -96,7 +101,7 @@ jobs:
96101
NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_test_autoscaling_${{ matrix.test-strategy }}
97102
- name: Cleanup Kubernetes cluster
98103
if: always()
99-
run: make chart_cluster_cleanup
104+
run: CLUSTER=${CLUSTER} make chart_cluster_cleanup
100105
- name: Upload Helm chart package
101106
if: always()
102107
uses: actions/upload-artifact@main

‎Makefile‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -447,16 +447,22 @@ chart_test_edge:
447447
./tests/charts/make/chart_test.sh NodeEdge
448448

449449
chart_test_autoscaling_deployment_https:
450-
SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test_autoscaling_deployment
450+
SE_FULL_DISTRIBUTED_MODE=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 \
451+
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
452+
./tests/charts/make/chart_test.sh DeploymentAutoscaling
451453

452454
chart_test_autoscaling_deployment:
453-
SE_ENABLE_TRACING=true VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
455+
SE_ENABLE_TRACING=true SELENIUM_GRID_TEST_HEADLESS=true \
456+
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
454457
./tests/charts/make/chart_test.sh DeploymentAutoscaling
455458

456459
chart_test_autoscaling_job_https:
457-
SE_ENABLE_TRACING=true SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test_autoscaling_job
460+
SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 \
461+
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
462+
./tests/charts/make/chart_test.sh JobAutoscaling
458463

459464
chart_test_autoscaling_job:
465+
SE_ENABLE_TRACING=true SE_FULL_DISTRIBUTED_MODE=true \
460466
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
461467
./tests/charts/make/chart_test.sh JobAutoscaling
462468

‎Video/video.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,10 @@ else
159159
echo Checking if node API responds
160160
until curl -sk --request GET ${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status || [[ $attempts = "$max_attempts" ]]
161161
do
162-
echo Waiting before next API check
162+
if [ $(($attempts % 60)) -eq 0 ];
163+
then
164+
echo Waiting before next API check
165+
fi
163166
sleep 0.5
164167
attempts=$((attempts+1))
165168
done

‎charts/selenium-grid/configs/node/nodePreStop.sh‎

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,73 @@
11
#!/bin/bash
2+
3+
function on_exit() {
4+
rm -rf /tmp/preStopOutput
5+
}
6+
trap on_exit EXIT
7+
8+
# Set headers if Node Registration Secret is set
29
if [ ! -z "${SE_REGISTRATION_SECRET}" ];
310
then
411
HEADERS="X-REGISTRATION-SECRET: ${SE_REGISTRATION_SECRET}"
512
else
613
HEADERS="X-REGISTRATION-SECRET;"
714
fi
815

9-
if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status; then
16+
function is_full_distributed_mode() {
17+
if [ -n "${SE_DISTRIBUTOR_HOST}" ] && [ -n "${SE_DISTRIBUTOR_PORT}" ]; then
18+
DISTRIBUTED_MODE=true
19+
echo "Detected full distributed mode: ${DISTRIBUTED_MODE}. Since SE_DISTRIBUTOR_HOST and SE_DISTRIBUTOR_PORT are set in Node ConfigMap"
20+
else
21+
DISTRIBUTED_MODE=false
22+
echo "Detected full distributed mode: ${DISTRIBUTED_MODE}"
23+
fi
24+
}
25+
is_full_distributed_mode
26+
27+
function signal_distributor_to_drain_node() {
28+
if [ "${DISTRIBUTED_MODE}" = true ]; then
29+
echo "Signaling Distributor to drain node"
30+
set -x
31+
curl -k -X POST ${SE_SERVER_PROTOCOL}://${SE_DISTRIBUTOR_HOST}:${SE_DISTRIBUTOR_PORT}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}"
32+
set +x
33+
fi
34+
}
35+
36+
function signal_hub_to_drain_node() {
37+
if [ "${DISTRIBUTED_MODE}" = false ]; then
38+
echo "Signaling Hub to drain node"
39+
curl -k -X POST ${SE_GRID_URL}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}"
40+
fi
41+
}
42+
43+
function signal_node_to_drain() {
44+
echo "Signaling Node to drain itself"
1045
curl -k -X POST ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/se/grid/node/drain --header "${HEADERS}"
46+
}
47+
48+
function replace_localhost_by_service_name() {
49+
internal="${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}"
50+
if [[ "${SE_NODE_GRID_URL}" == *"/localhost"* ]]; then
51+
SE_GRID_URL=${SE_NODE_GRID_URL//localhost/${internal}}
52+
elif [[ "${SE_NODE_GRID_URL}" == *"/127.0.0.1"* ]]; then
53+
SE_GRID_URL=${SE_NODE_GRID_URL//127.0.0.1/${internal}}
54+
elif [[ "${SE_NODE_GRID_URL}" == *"/0.0.0.0"* ]]; then
55+
SE_GRID_URL=${SE_NODE_GRID_URL//0.0.0.0/${internal}}
56+
fi
57+
echo "SE_GRID_URL: ${SE_GRID_URL}"
58+
}
59+
replace_localhost_by_service_name
60+
61+
if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status > /tmp/preStopOutput; then
62+
NODE_ID=$(jq -r '.value.node.nodeId' /tmp/preStopOutput)
63+
if [ -n "${NODE_ID}" ]; then
64+
echo "Current Node ID is: ${NODE_ID}"
65+
signal_hub_to_drain_node
66+
signal_distributor_to_drain_node
67+
echo
68+
fi
69+
signal_node_to_drain
70+
# Wait for the current session to be finished if any
1171
while curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o /tmp/preStopOutput;
1272
do
1373
echo "Node preStop is waiting for current session to be finished if any. Node details: message: $(jq -r '.value.message' /tmp/preStopOutput || "unknown"), availability: $(jq -r '.value.node.availability' /tmp/preStopOutput || "unknown")"

‎charts/selenium-grid/configs/node/nodeProbe.sh‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/bin/bash
2+
3+
function on_exit() {
4+
rm -rf /tmp/nodeProbe${ID}
5+
rm -rf /tmp/gridProbe${ID}
6+
}
7+
trap on_exit EXIT
8+
9+
ID=$(echo $RANDOM)
10+
11+
function replace_localhost_by_service_name() {
12+
internal="${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}"
13+
if [[ "${SE_NODE_GRID_URL}" == *"/localhost"* ]]; then
14+
SE_GRID_URL=${SE_NODE_GRID_URL//localhost/${internal}}
15+
elif [[ "${SE_NODE_GRID_URL}" == *"/127.0.0.1"* ]]; then
16+
SE_GRID_URL=${SE_NODE_GRID_URL//127.0.0.1/${internal}}
17+
elif [[ "${SE_NODE_GRID_URL}" == *"/0.0.0.0"* ]]; then
18+
SE_GRID_URL=${SE_NODE_GRID_URL//0.0.0.0/${internal}}
19+
fi
20+
echo "SE_GRID_URL: ${SE_GRID_URL}"
21+
}
22+
replace_localhost_by_service_name
23+
24+
if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o /tmp/nodeProbe${ID}; then
25+
NODE_ID=$(jq -r '.value.node.nodeId' /tmp/nodeProbe${ID})
26+
NODE_STATUS=$(jq -r '.value.node.availability' /tmp/nodeProbe${ID})
27+
28+
curl -sfk "${SE_GRID_URL}/status" -o /tmp/gridProbe${ID}
29+
GRID_NODE_ID=$(jq -e ".value.nodes[].id|select(. == \"${NODE_ID}\")" /tmp/gridProbe${ID} | tr -d '"' || true)
30+
31+
if [ "${NODE_STATUS}" = "UP" ] && [ -n "${NODE_ID}" ] && [ -n "${GRID_NODE_ID}" ] && [ "${NODE_ID}" = "${GRID_NODE_ID}" ]; then
32+
echo "Node ID: ${NODE_ID} with status: ${NODE_STATUS}"
33+
echo "Found in the Grid a matched Node ID: ${GRID_NODE_ID}"
34+
exit 0
35+
else
36+
echo "Node ID: ${NODE_ID} is not found in the Grid. The registration could be in progress."
37+
exit 1
38+
fi
39+
else
40+
exit 1
41+
fi

‎charts/selenium-grid/templates/_helpers.tpl‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,20 @@ Probe httpGet schema
1717
Check user define custom probe method
1818
*/}}
1919
{{- define "seleniumGrid.probe.fromUserDefine" -}}
20+
{{- $values := index . "values" -}}
21+
{{- $root := index . "root" -}}
2022
{{- $overrideProbe := dict -}}
21-
{{- with .exec -}}
22-
{{- $overrideProbe = dict "exec" . -}}
23+
{{- with $values.exec -}}
24+
{{- $overrideProbe = dict "exec" (tpl (toYaml .) $root | fromYaml) -}}
2325
{{- end }}
24-
{{- with .httpGet -}}
25-
{{- $overrideProbe = dict "httpGet" . -}}
26+
{{- with $values.httpGet -}}
27+
{{- $overrideProbe = dict "httpGet" (tpl (toYaml .) $root | fromYaml) -}}
2628
{{- end }}
27-
{{- with .tcpSocket -}}
28-
{{- $overrideProbe = dict "tcpSocket" . -}}
29+
{{- with $values.tcpSocket -}}
30+
{{- $overrideProbe = dict "tcpSocket" (tpl (toYaml .) $root | fromYaml) -}}
2931
{{- end }}
30-
{{- with .grpc -}}
31-
{{- $overrideProbe = dict "grpc" . -}}
32+
{{- with $values.grpc -}}
33+
{{- $overrideProbe = dict "grpc" (tpl (toYaml .) $root | fromYaml) -}}
3234
{{- end -}}
3335
{{- $overrideProbe | toYaml -}}
3436
{{- end -}}
@@ -258,8 +260,8 @@ template:
258260
{{- if .node.startupProbe.enabled }}
259261
{{- with .node.startupProbe }}
260262
startupProbe:
261-
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
262-
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
263+
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
264+
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
263265
{{- else }}
264266
httpGet:
265267
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
@@ -274,8 +276,8 @@ template:
274276
{{- if .node.readinessProbe.enabled }}
275277
{{- with .node.readinessProbe }}
276278
readinessProbe:
277-
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
278-
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 12 }}
279+
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
280+
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 12 }}
279281
{{- else }}
280282
httpGet:
281283
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
@@ -290,8 +292,8 @@ template:
290292
{{- if .node.livenessProbe.enabled }}
291293
{{- with .node.livenessProbe }}
292294
livenessProbe:
293-
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
294-
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
295+
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
296+
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
295297
{{- else }}
296298
httpGet:
297299
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}

‎charts/selenium-grid/templates/hub-deployment.yaml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ spec:
4545
{{- if .Values.hub.startupProbe.enabled }}
4646
{{- with .Values.hub.startupProbe }}
4747
startupProbe:
48-
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
49-
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
48+
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
49+
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
5050
{{- else }}
5151
httpGet:
5252
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
@@ -61,8 +61,8 @@ spec:
6161
{{- if .Values.hub.readinessProbe.enabled }}
6262
{{- with .Values.hub.readinessProbe }}
6363
readinessProbe:
64-
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
65-
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
64+
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
65+
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
6666
{{- else }}
6767
httpGet:
6868
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
@@ -77,8 +77,8 @@ spec:
7777
{{- if .Values.hub.livenessProbe.enabled }}
7878
{{- with .Values.hub.livenessProbe }}
7979
livenessProbe:
80-
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
81-
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
80+
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
81+
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
8282
{{- else }}
8383
httpGet:
8484
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}

‎charts/selenium-grid/templates/node-configmap.yaml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,18 @@ metadata:
1212
{{- toYaml . | nindent 4 }}
1313
{{- end }}
1414
data:
15+
{{- if .Values.isolateComponents }}
16+
SE_DISTRIBUTOR_HOST: '{{ include "seleniumGrid.distributor.fullname" . }}.{{ .Release.Namespace }}'
17+
SE_DISTRIBUTOR_PORT: '{{ .Values.components.distributor.port }}'
18+
SE_ROUTER_HOST: '{{ include "seleniumGrid.router.fullname" . }}.{{ .Release.Namespace }}'
19+
SE_ROUTER_PORT: '{{ .Values.components.router.port }}'
20+
{{- else }}
21+
SE_HUB_HOST: '{{ include "seleniumGrid.hub.fullname" . }}.{{ .Release.Namespace }}'
22+
SE_HUB_PORT: '{{ .Values.hub.port }}'
23+
{{- end }}
1524
SE_DRAIN_AFTER_SESSION_COUNT: '{{- and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "1" "0" -}}'
1625
SE_NODE_GRID_URL: '{{ include "seleniumGrid.url" .}}'
17-
SE_NODE_GRID_GRAPHQL_URL: '{{ include "seleniumGrid.graphqlURL" .}}'
26+
SE_NODE_GRID_GRAPHQL_URL: '{{ include "seleniumGrid.graphqlURL" . }}'
1827
{{- range $fileName, $value := .Values.nodeConfigMap.extraScripts }}
1928
{{- if not (empty $value) }}
2029
{{ $fileName | indent 2 -}}: |

‎charts/selenium-grid/templates/router-deployment.yaml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,8 @@ spec:
8484
{{- if .Values.components.router.startupProbe.enabled }}
8585
{{- with .Values.components.router.startupProbe }}
8686
startupProbe:
87-
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
88-
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
87+
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
88+
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
8989
{{- else }}
9090
httpGet:
9191
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
@@ -100,8 +100,8 @@ spec:
100100
{{- if .Values.components.router.readinessProbe.enabled }}
101101
{{- with .Values.components.router.readinessProbe }}
102102
readinessProbe:
103-
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
104-
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
103+
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
104+
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
105105
{{- else }}
106106
httpGet:
107107
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
@@ -116,8 +116,8 @@ spec:
116116
{{- if .Values.components.router.livenessProbe.enabled }}
117117
livenessProbe:
118118
{{- with .Values.components.router.livenessProbe }}
119-
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
120-
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
119+
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
120+
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
121121
{{- else }}
122122
httpGet:
123123
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}

0 commit comments

Comments
 (0)