Skip to content

Commit a8d7eb7

Browse files
VietND96VietND96
committed
K8s: Node Relay to extend autoscaling Grid with test cloud resources
Signed-off-by: Viet Nguyen Duc <[email protected]>
1 parent 8da2459 commit a8d7eb7

17 files changed

+171
-17
lines changed

‎.github/workflows/helm-chart-test.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,15 @@ jobs:
102102
service-mesh: true
103103
os: ubuntu-22.04
104104
test-strategy: playwright_connect_grid
105+
- k8s-version: 'v1.32.2'
106+
cluster: 'minikube'
107+
helm-version: 'v3.17.0'
108+
docker-version: '26.1.4'
109+
python-version: '3.10'
110+
test-upgrade: true
111+
service-mesh: true
112+
os: ubuntu-22.04
113+
test-strategy: job_relay
105114
env:
106115
CLUSTER: ${{ matrix.cluster }}
107116
KUBERNETES_VERSION: ${{ matrix.k8s-version }}
@@ -110,6 +119,9 @@ jobs:
110119
DOCKER_VERSION: ${{ matrix.docker-version }}
111120
TEST_UPGRADE_CHART: ${{ matrix.test-upgrade }}
112121
SERVICE_MESH: ${{ matrix.service-mesh }}
122+
SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
123+
SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
124+
SAUCE_REGION: ${{ secrets.SAUCE_REGION }}
113125
steps:
114126
- name: Free Disk Space (Ubuntu)
115127
uses: jlumbroso/free-disk-space@main

‎Makefile‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ KEDA_TAG_PREV_VERSION := $(or $(KEDA_TAG_PREV_VERSION),$(KEDA_TAG_PREV_VERSION),
3434
KEDA_CORE_VERSION := $(or $(KEDA_CORE_VERSION),$(KEDA_CORE_VERSION),2.16.1)
3535
KEDA_TAG_VERSION := $(or $(KEDA_TAG_VERSION),$(KEDA_TAG_VERSION),2.16.1-selenium-grid)
3636
KEDA_BASED_NAME := $(or $(KEDA_BASED_NAME),$(KEDA_BASED_NAME),ndviet)
37-
KEDA_BASED_TAG := $(or $(KEDA_BASED_TAG),$(KEDA_BASED_TAG),2.16.1-selenium-grid-20250225)
37+
KEDA_BASED_TAG := $(or $(KEDA_BASED_TAG),$(KEDA_BASED_TAG),2.16.1-selenium-grid-20250310)
3838
TEST_PATCHED_KEDA := $(or $(TEST_PATCHED_KEDA),$(TEST_PATCHED_KEDA),true)
3939

4040
all: hub \
@@ -990,6 +990,14 @@ chart_test_autoscaling_job_hostname:
990990
TEMPLATE_OUTPUT_FILENAME="k8s_enableTracing_basicAuth_secureIngress_externalCerts_ingressPublicIP_autoScaling_originKEDA_scaledJob_subPath.yaml" \
991991
./tests/charts/make/chart_test.sh JobAutoscaling
992992

993+
chart_test_autoscaling_job_relay:
994+
PLATFORMS=$(PLATFORMS) CHART_ENABLE_TRACING=true CHART_ENABLE_BASIC_AUTH=true SELENIUM_GRID_MONITORING=false TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) \
995+
TEST_MULTIPLE_PLATFORMS=true TEST_MULTIPLE_PLATFORMS_RELAY=true CLEAR_POD_HISTORY=true \
996+
SECURE_INGRESS_ONLY_DEFAULT=true SECURE_USE_EXTERNAL_CERT=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -I | cut -d' ' -f1) SELENIUM_GRID_PORT=443 \
997+
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \
998+
TEMPLATE_OUTPUT_FILENAME="k8s_enableTracing_basicAuth_secureIngress_externalCerts_ingressPublicIP_autoScaling_relay_node_scaledJob_subPath.yaml" \
999+
./tests/charts/make/chart_test.sh JobAutoscaling
1000+
9931001
chart_test_autoscaling_job_multiple_versions_without_explicit:
9941002
TEST_MULTIPLE_VERSIONS=true TEST_MULTIPLE_VERSIONS_EXPLICIT=false make chart_test_autoscaling_job
9951003

‎NodeBase/Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ ENV LANG_WHICH=${LANG_WHICH} \
5050
SE_NODE_REGISTER_CYCLE="10" \
5151
SE_NODE_REGISTER_SHUTDOWN_ON_FAILURE="true" \
5252
SE_OTEL_SERVICE_NAME="selenium-node" \
53+
SE_NODE_RELAY_ONLY="true" \
5354
# Setting Selenium Manager to work offline
5455
SE_OFFLINE="true" \
5556
SE_NODE_BROWSER_VERSION="stable" \

‎NodeBase/generate_config‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,11 @@ fi
6565
SE_NODE_CONTAINER_NAME="${SE_NODE_CONTAINER_NAME:-$(hostname)}"
6666

6767
# 'browserName' is mandatory for default stereotype
68-
if [[ -z "${SE_NODE_STEREOTYPE}" ]] && [[ -n "${SE_NODE_BROWSER_NAME}" ]]; then
69-
SE_NODE_STEREOTYPE="{\"browserName\": \"${SE_NODE_BROWSER_NAME}\", \"browserVersion\": \"${SE_NODE_BROWSER_VERSION}\", \"platformName\": \"${SE_NODE_PLATFORM_NAME}\", ${SE_BROWSER_BINARY_LOCATION}, \"se:containerName\": \"${SE_NODE_CONTAINER_NAME}\", \"container:hostname\": \"$(hostname)\"}"
68+
if [[ -z "${SE_NODE_STEREOTYPE}" ]] && [[ -n "${SE_NODE_BROWSER_NAME}" ]] && ([[ -z "${SE_NODE_RELAY_URL}" ]] || [[ "${SE_NODE_RELAY_ONLY}" = "false" ]]); then
69+
SE_NODE_STEREOTYPE="{\"browserName\": \"${SE_NODE_BROWSER_NAME}\", \"browserVersion\": \"${SE_NODE_BROWSER_VERSION}\", \"platformName\": \"${SE_NODE_PLATFORM_NAME}\", \"se:containerName\": \"${SE_NODE_CONTAINER_NAME}\", \"container:hostname\": \"$(hostname)\"}"
70+
if [[ -n "${SE_BROWSER_BINARY_LOCATION}" ]]; then
71+
SE_NODE_STEREOTYPE="$(python3 /opt/bin/json_merge.py "${SE_NODE_STEREOTYPE}" "{${SE_BROWSER_BINARY_LOCATION}}")"
72+
fi
7073
else
7174
SE_NODE_STEREOTYPE="${SE_NODE_STEREOTYPE}"
7275
fi

‎NodeBase/generate_relay_config‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ fi
88

99
if [[ -n "${SE_NODE_RELAY_URL}" ]]; then
1010
echo "[relay]" >>"$FILENAME"
11-
echo "url = \"${SE_NODE_RELAY_URL}\"" >>"$FILENAME"
11+
echo "url = \"$(envsubst < <(echo ${SE_NODE_RELAY_URL}))\"" >>"$FILENAME"
1212
if [[ -z "${SE_NODE_RELAY_STATUS_ENDPOINT}" ]]; then
1313
echo "status-endpoint = \"/status\"" >>"$FILENAME"
1414
else
@@ -18,7 +18,17 @@ if [[ -n "${SE_NODE_RELAY_URL}" ]]; then
1818
echo "protocol-version = \"${SE_NODE_RELAY_PROTOCOL_VERSION}\"" >>"$FILENAME"
1919
fi
2020
if [[ -z "${SE_NODE_RELAY_STEREOTYPE}" ]]; then
21-
SE_NODE_RELAY_STEREOTYPE="{\"browserName\": \"${SE_NODE_RELAY_BROWSER_NAME}\", \"platformName\": \"${SE_NODE_RELAY_PLATFORM_NAME}\", \"appium:platformVersion\": \"${SE_NODE_RELAY_PLATFORM_VERSION}\"}"
21+
SE_NODE_RELAY_STEREOTYPE="{\"browserName\": \"${SE_NODE_BROWSER_NAME:-${SE_NODE_RELAY_BROWSER_NAME}}\", \"platformName\": \"${SE_NODE_PLATFORM_NAME:-${SE_NODE_RELAY_PLATFORM_NAME}}\"}"
22+
if [[ -n "${SE_NODE_RELAY_PLATFORM_VERSION}" ]]; then
23+
SE_NODE_RELAY_STEREOTYPE="$(python3 /opt/bin/json_merge.py "${SE_NODE_RELAY_STEREOTYPE}" "{\"appium:platformVersion\":\"${SE_NODE_RELAY_PLATFORM_VERSION}\"}")"
24+
fi
25+
BROWSER_VERSION=${SE_NODE_BROWSER_VERSION:-${SE_NODE_RELAY_BROWSER_VERSION}}
26+
if [[ -n "${BROWSER_VERSION}" ]]; then
27+
SE_NODE_RELAY_STEREOTYPE="$(python3 /opt/bin/json_merge.py "${SE_NODE_RELAY_STEREOTYPE}" "{\"browserVersion\":\"${BROWSER_VERSION}\"}")"
28+
fi
29+
if [[ "${SE_NODE_ENABLE_MANAGED_DOWNLOADS}" = "true" ]]; then
30+
SE_NODE_RELAY_STEREOTYPE="$(python3 /opt/bin/json_merge.py "${SE_NODE_RELAY_STEREOTYPE}" "{\"se:downloadsEnabled\": true}")"
31+
fi
2232
else
2333
SE_NODE_RELAY_STEREOTYPE="${SE_NODE_RELAY_STEREOTYPE}"
2434
fi
@@ -31,5 +41,5 @@ if [[ -n "${SE_NODE_RELAY_URL}" ]]; then
3141
echo "Merged relay stereotype: ${SE_NODE_RELAY_STEREOTYPE}"
3242
fi
3343
fi
34-
echo "configs = ['${SE_NODE_RELAY_MAX_SESSIONS}', '${SE_NODE_RELAY_STEREOTYPE}']" >>"$FILENAME"
44+
echo "configs = ['${SE_NODE_MAX_SESSIONS:-${SE_NODE_RELAY_MAX_SESSIONS}}', '${SE_NODE_RELAY_STEREOTYPE}']" >>"$FILENAME"
3545
fi

‎Standalone/generate_config‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,10 @@ fi
4444
SE_NODE_CONTAINER_NAME="${SE_NODE_CONTAINER_NAME:-$(hostname)}"
4545

4646
if [[ -z "$SE_NODE_STEREOTYPE" ]]; then
47-
SE_NODE_STEREOTYPE="{\"browserName\": \"${SE_NODE_BROWSER_NAME}\", \"browserVersion\": \"${SE_NODE_BROWSER_VERSION}\", \"platformName\": \"${SE_NODE_PLATFORM_NAME}\", ${SE_BROWSER_BINARY_LOCATION}, \"se:containerName\": \"${SE_NODE_CONTAINER_NAME}\", \"container:hostname\": \"$(hostname)\"}"
47+
SE_NODE_STEREOTYPE="{\"browserName\": \"${SE_NODE_BROWSER_NAME}\", \"browserVersion\": \"${SE_NODE_BROWSER_VERSION}\", \"platformName\": \"${SE_NODE_PLATFORM_NAME}\", \"se:containerName\": \"${SE_NODE_CONTAINER_NAME}\", \"container:hostname\": \"$(hostname)\"}"
48+
if [[ -n "${SE_BROWSER_BINARY_LOCATION}" ]]; then
49+
SE_NODE_STEREOTYPE="$(python3 /opt/bin/json_merge.py "${SE_NODE_STEREOTYPE}" "{${SE_BROWSER_BINARY_LOCATION}}")"
50+
fi
4851
else
4952
SE_NODE_STEREOTYPE="$SE_NODE_STEREOTYPE"
5053
fi

‎charts/selenium-grid/CONFIGURATION.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -639,10 +639,10 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
639639
| relayNode.scaledOptions | string | `nil` | Override the scaled options for relay nodes |
640640
| relayNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for relay nodes |
641641
| relayNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for relay nodes |
642-
| relayNode.hpa.browserName | string | `"chrome"` | browserName should match with Node stereotype and request capability is scaled by this scaler |
642+
| relayNode.hpa.browserName | string | `""` | browserName should match with Node stereotype and request capability is scaled by this scaler |
643643
| relayNode.hpa.sessionBrowserName | string | `""` | sessionBrowserName if the browserName is different from the sessionBrowserName |
644644
| relayNode.hpa.browserVersion | string | `""` | browserVersion should match with Node stereotype and request capability is scaled by this scaler |
645-
| relayNode.hpa.platformName | string | `"Android"` | platformName should match with Node stereotype and request capability is scaled by this scaler |
645+
| relayNode.hpa.platformName | string | `""` | platformName should match with Node stereotype and request capability is scaled by this scaler |
646646
| relayNode.hpa.unsafeSsl | string | `"{{ template \"seleniumGrid.graphqlURL.unsafeSsl\" . }}"` | Skip check SSL when connecting to the Graphql endpoint |
647647
| relayNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option |
648648
| relayNode.sidecars | list | `[]` | It is used to add sidecars proxy in the same pod of the browser node. It means it will add a new container to the deployment itself. It should be set using the --set-json option |

‎charts/selenium-grid/multiple-nodes-platform-relay.yaml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
# Utilize Relay Node to set up hybrid Autoscaling Grid with using on-premise and test cloud provider (e.g. SauceLabs, BrowserStack, etc.)
2+
#
3+
# For example: below incoming requests will be served by Node container on-premise
4+
# options = ChromeOptions()
5+
# options.set_capability('platformName', 'Linux')
6+
# driver = webdriver.Remote(options=options, command_executor=SELENIUM_GRID_URL)
7+
#
8+
# Below incoming requests will be served by Relay Node where commands are forwarded to test cloud provider
9+
# options = ChromeOptions()
10+
# options.set_capability('platformName', 'macOS 13.0')
11+
# driver = webdriver.Remote(options=options, command_executor=SELENIUM_GRID_URL)
12+
crossBrowsers:
13+
chromeNode:
14+
- nameOverride: '{{ $.Release.Name }}-node-chrome-linux'
15+
hpa:
16+
platformName: 'Linux'
17+
firefoxNode:
18+
- nameOverride: '{{ $.Release.Name }}-node-firefox-linux'
19+
hpa:
20+
platformName: 'Linux'
21+
edgeNode:
22+
- nameOverride: '{{ $.Release.Name }}-node-edge-linux'
23+
hpa:
24+
platformName: 'Linux'
25+
relayNode:
26+
- nameOverride: '{{ $.Release.Name }}-node-relay-chrome-macos'
27+
hpa:
28+
browserName: 'chrome'
29+
platformName: 'macOS'
30+
- nameOverride: '{{ $.Release.Name }}-node-relay-chrome-windows'
31+
hpa:
32+
browserName: 'chrome'
33+
platformName: 'Windows'
34+
- nameOverride: '{{ $.Release.Name }}-node-relay-firefox-macos'
35+
hpa:
36+
browserName: 'firefox'
37+
platformName: 'macOS'
38+
- nameOverride: '{{ $.Release.Name }}-node-relay-firefox-windows'
39+
hpa:
40+
browserName: 'firefox'
41+
platformName: 'Windows'
42+
- nameOverride: '{{ $.Release.Name }}-node-relay-edge-macos'
43+
hpa:
44+
browserName: "MicrosoftEdge"
45+
sessionBrowserName: "msedge"
46+
platformName: 'macOS'
47+
- nameOverride: '{{ $.Release.Name }}-node-relay-edge-windows'
48+
hpa:
49+
browserName: "MicrosoftEdge"
50+
sessionBrowserName: "msedge"
51+
platformName: 'Windows'
52+
- nameOverride: '{{ $.Release.Name }}-node-relay-safari-macos'
53+
hpa:
54+
browserName: 'safari'
55+
platformName: 'macOS'
56+
57+
relayNode:
58+
enabled: true
59+
videoRecorder:
60+
enabled: false
61+
extraEnvironmentVariables:
62+
# - name: SAUCE_USERNAME
63+
# value: "<your_SAUCE_USERNAME>"
64+
# - name: SAUCE_ACCESS_KEY
65+
# value: "<your_SAUCE_ACCESS_KEY>"
66+
# - name: SAUCE_REGION
67+
# value: "<your_SAUCE_REGION>"
68+
# - name: SE_NODE_RELAY_URL
69+
# value: "https://$SAUCE_USERNAME:$SAUCE_ACCESS_KEY@ondemand.$SAUCE_REGION.saucelabs.com:443/wd/hub"
70+
extraEnvFrom:
71+
# - secretRef:
72+
# name: your-secret-with-all-env-vars

‎charts/selenium-grid/multiple-nodes-platform.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ crossBrowsers:
2121
browserVersion: ''
2222
- nameOverride: '{{ $.Release.Name }}-node-chrome-platform-windows'
2323
hpa:
24-
platformName: 'windows'
24+
platformName: 'Windows'
2525
browserVersion: ''
2626
firefoxNode:
2727
- nameOverride: '{{ $.Release.Name }}-node-firefox-platform-any'
@@ -34,7 +34,7 @@ crossBrowsers:
3434
browserVersion: ''
3535
- nameOverride: '{{ $.Release.Name }}-node-firefox-platform-windows'
3636
hpa:
37-
platformName: 'windows'
37+
platformName: 'Windows'
3838
browserVersion: ''
3939
edgeNode:
4040
- nameOverride: '{{ $.Release.Name }}-node-edge-platform-any'
@@ -47,5 +47,5 @@ crossBrowsers:
4747
browserVersion: ''
4848
- nameOverride: '{{ $.Release.Name }}-node-edge-platform-windows'
4949
hpa:
50-
platformName: 'windows'
50+
platformName: 'Windows'
5151
browserVersion: ''

‎charts/selenium-grid/templates/_helpers.tpl‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,10 @@ template:
369369
value: {{ $nodeCustomCapabilities | quote }}
370370
- name: SE_DRAIN_AFTER_SESSION_COUNT
371371
value: {{ and (eq (include "seleniumGrid.useKEDA" $) "true") (eq .Values.autoscaling.scalingType "job") | ternary $nodeMaxSessions 0 | quote }}
372+
{{- if and (eq (include "seleniumGrid.useKEDA" $) "true") }}
373+
- name: SE_NODE_BROWSER_NAME
374+
value: {{ if hasKey .node.hpa "browserName" }}{{ .node.hpa.browserName | quote }}{{ else }}""{{ end }}
375+
{{- end }}
372376
{{- if and (eq (include "seleniumGrid.useKEDA" $) "true") }}
373377
- name: SE_NODE_BROWSER_VERSION
374378
value: {{ if hasKey .node.hpa "browserVersion" }}{{ .node.hpa.browserVersion | quote }}{{ else }}""{{ end }}

0 commit comments

Comments
 (0)