diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml index 18dcba271c..69eaa36fc9 100644 --- a/.github/workflows/helm-chart-test.yml +++ b/.github/workflows/helm-chart-test.yml @@ -47,6 +47,7 @@ jobs: test-upgrade: true service-mesh: false os: ubuntu-22.04 + check-records-output: true test-strategy: disabled - k8s-version: 'v1.27.16' cluster: 'minikube' @@ -56,6 +57,7 @@ jobs: test-upgrade: true service-mesh: true os: ubuntu-22.04 + check-records-output: true test-strategy: job - k8s-version: 'v1.28.15' cluster: 'minikube' @@ -65,8 +67,9 @@ jobs: test-upgrade: true service-mesh: true os: ubuntu-22.04 + check-records-output: true test-strategy: deployment - - k8s-version: 'v1.29.14' + - k8s-version: 'v1.29.15' cluster: 'minikube' helm-version: 'v3.14.3' docker-version: '27.5.1' @@ -74,8 +77,9 @@ jobs: test-upgrade: true service-mesh: false os: ubuntu-22.04 + check-records-output: true test-strategy: job_https - - k8s-version: 'v1.30.10' + - k8s-version: 'v1.30.11' cluster: 'minikube' helm-version: 'v3.15.4' docker-version: '27.5.1' @@ -83,8 +87,9 @@ jobs: test-upgrade: true service-mesh: false os: ubuntu-22.04 + check-records-output: true test-strategy: job_hostname - - k8s-version: 'v1.31.6' + - k8s-version: 'v1.31.7' cluster: 'minikube' helm-version: 'v3.16.4' docker-version: '27.4.1' @@ -92,8 +97,9 @@ jobs: test-upgrade: true service-mesh: false os: ubuntu-22.04 + check-records-output: true test-strategy: deployment_https - - k8s-version: 'v1.32.2' + - k8s-version: 'v1.32.3' cluster: 'minikube' helm-version: 'v3.17.0' docker-version: '26.1.4' @@ -101,7 +107,18 @@ jobs: test-upgrade: true service-mesh: true os: ubuntu-22.04 + check-records-output: true test-strategy: playwright_connect_grid + - k8s-version: 'v1.32.3' + cluster: 'minikube' + helm-version: 'v3.17.0' + docker-version: '26.1.4' + python-version: '3.10' + test-upgrade: true + service-mesh: true + os: ubuntu-22.04 + check-records-output: false + test-strategy: job_relay env: CLUSTER: ${{ matrix.cluster }} KUBERNETES_VERSION: ${{ matrix.k8s-version }} @@ -110,6 +127,10 @@ jobs: DOCKER_VERSION: ${{ matrix.docker-version }} TEST_UPGRADE_CHART: ${{ matrix.test-upgrade }} SERVICE_MESH: ${{ matrix.service-mesh }} + CHECK_RECORD_OUTPUT: ${{ matrix.check-records-output }} + SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} + SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + SAUCE_REGION: ${{ secrets.SAUCE_REGION }} steps: - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main @@ -200,8 +221,13 @@ jobs: timeout_minutes: 30 max_attempts: 3 command: | - NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${{ matrix.test-strategy }} \ - && NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make test_video_integrity + NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${{ matrix.test-strategy }} + exit_code=$? + if [[ "${CHECK_RECORD_OUTPUT}" = "true" ]] && [[ "${exit_code}" -eq 0 ]]; then + NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make test_video_integrity + exit_code=$? + fi + exit ${exit_code} - name: Upload Helm chart package if: always() uses: actions/upload-artifact@main diff --git a/.github/workflows/k8s-scaling-test.yml b/.github/workflows/k8s-scaling-test.yml index 9d6b2683da..768f92e0ff 100644 --- a/.github/workflows/k8s-scaling-test.yml +++ b/.github/workflows/k8s-scaling-test.yml @@ -75,28 +75,28 @@ jobs: python-version: '3.10' os: ubuntu-22.04 test-strategy: test_k8s_autoscaling_job_count_strategy_default_with_node_max_sessions - - k8s-version: 'v1.29.14' + - k8s-version: 'v1.29.15' cluster: 'minikube' helm-version: 'v3.14.3' docker-version: '27.5.1' python-version: '3.11' os: ubuntu-22.04 test-strategy: test_k8s_autoscaling_job_count_strategy_default - - k8s-version: 'v1.30.10' + - k8s-version: 'v1.30.11' cluster: 'minikube' helm-version: 'v3.15.4' docker-version: '27.5.1' python-version: '3.12' os: ubuntu-22.04 test-strategy: test_k8s_autoscaling_deployment_count_in_chaos - - k8s-version: 'v1.31.6' + - k8s-version: 'v1.31.7' cluster: 'minikube' helm-version: 'v3.16.4' docker-version: '27.4.1' python-version: '3.13' os: ubuntu-22.04 test-strategy: test_k8s_autoscaling_deployment_count_with_node_max_sessions - - k8s-version: 'v1.32.2' + - k8s-version: 'v1.32.3' cluster: 'minikube' helm-version: 'v3.17.0' docker-version: '26.1.4' diff --git a/Makefile b/Makefile index e7d715e92b..4a51bf0a64 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ KEDA_TAG_PREV_VERSION := $(or $(KEDA_TAG_PREV_VERSION),$(KEDA_TAG_PREV_VERSION), KEDA_CORE_VERSION := $(or $(KEDA_CORE_VERSION),$(KEDA_CORE_VERSION),2.16.1) KEDA_TAG_VERSION := $(or $(KEDA_TAG_VERSION),$(KEDA_TAG_VERSION),2.16.1-selenium-grid) KEDA_BASED_NAME := $(or $(KEDA_BASED_NAME),$(KEDA_BASED_NAME),ndviet) -KEDA_BASED_TAG := $(or $(KEDA_BASED_TAG),$(KEDA_BASED_TAG),2.16.1-selenium-grid-20250225) +KEDA_BASED_TAG := $(or $(KEDA_BASED_TAG),$(KEDA_BASED_TAG),2.16.1-selenium-grid-20250310) TEST_PATCHED_KEDA := $(or $(TEST_PATCHED_KEDA),$(TEST_PATCHED_KEDA),true) all: hub \ @@ -990,6 +990,14 @@ chart_test_autoscaling_job_hostname: TEMPLATE_OUTPUT_FILENAME="k8s_enableTracing_basicAuth_secureIngress_externalCerts_ingressPublicIP_autoScaling_originKEDA_scaledJob_subPath.yaml" \ ./tests/charts/make/chart_test.sh JobAutoscaling +chart_test_autoscaling_job_relay: + PLATFORMS=$(PLATFORMS) CHART_ENABLE_TRACING=true CHART_ENABLE_BASIC_AUTH=true SELENIUM_GRID_MONITORING=false TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) \ + TEST_MULTIPLE_PLATFORMS=true TEST_MULTIPLE_PLATFORMS_RELAY=true CLEAR_POD_HISTORY=true \ + SECURE_INGRESS_ONLY_DEFAULT=true SECURE_USE_EXTERNAL_CERT=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -I | cut -d' ' -f1) SELENIUM_GRID_PORT=443 \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \ + TEMPLATE_OUTPUT_FILENAME="k8s_enableTracing_basicAuth_secureIngress_externalCerts_ingressPublicIP_autoScaling_relay_node_scaledJob_subPath.yaml" \ + ./tests/charts/make/chart_test.sh JobAutoscaling + chart_test_autoscaling_job_multiple_versions_without_explicit: TEST_MULTIPLE_VERSIONS=true TEST_MULTIPLE_VERSIONS_EXPLICIT=false make chart_test_autoscaling_job diff --git a/NodeBase/Dockerfile b/NodeBase/Dockerfile index 18ad6dccb3..898d78088c 100644 --- a/NodeBase/Dockerfile +++ b/NodeBase/Dockerfile @@ -50,6 +50,7 @@ ENV LANG_WHICH=${LANG_WHICH} \ SE_NODE_REGISTER_CYCLE="10" \ SE_NODE_REGISTER_SHUTDOWN_ON_FAILURE="true" \ SE_OTEL_SERVICE_NAME="selenium-node" \ + SE_NODE_RELAY_ONLY="true" \ # Setting Selenium Manager to work offline SE_OFFLINE="true" \ SE_NODE_BROWSER_VERSION="stable" \ diff --git a/NodeBase/generate_config b/NodeBase/generate_config index aaa5c44997..4f7c38f4c4 100755 --- a/NodeBase/generate_config +++ b/NodeBase/generate_config @@ -65,8 +65,11 @@ fi SE_NODE_CONTAINER_NAME="${SE_NODE_CONTAINER_NAME:-$(hostname)}" # 'browserName' is mandatory for default stereotype -if [[ -z "${SE_NODE_STEREOTYPE}" ]] && [[ -n "${SE_NODE_BROWSER_NAME}" ]]; then - SE_NODE_STEREOTYPE="{\"browserName\": \"${SE_NODE_BROWSER_NAME}\", \"browserVersion\": \"${SE_NODE_BROWSER_VERSION}\", \"platformName\": \"${SE_NODE_PLATFORM_NAME}\", ${SE_BROWSER_BINARY_LOCATION}, \"se:containerName\": \"${SE_NODE_CONTAINER_NAME}\", \"container:hostname\": \"$(hostname)\"}" +if [[ -z "${SE_NODE_STEREOTYPE}" ]] && [[ -n "${SE_NODE_BROWSER_NAME}" ]] && ([[ -z "${SE_NODE_RELAY_URL}" ]] || [[ "${SE_NODE_RELAY_ONLY}" = "false" ]]); then + SE_NODE_STEREOTYPE="{\"browserName\": \"${SE_NODE_BROWSER_NAME}\", \"browserVersion\": \"${SE_NODE_BROWSER_VERSION}\", \"platformName\": \"${SE_NODE_PLATFORM_NAME}\", \"se:containerName\": \"${SE_NODE_CONTAINER_NAME}\", \"container:hostname\": \"$(hostname)\"}" + if [[ -n "${SE_BROWSER_BINARY_LOCATION}" ]]; then + SE_NODE_STEREOTYPE="$(python3 /opt/bin/json_merge.py "${SE_NODE_STEREOTYPE}" "{${SE_BROWSER_BINARY_LOCATION}}")" + fi else SE_NODE_STEREOTYPE="${SE_NODE_STEREOTYPE}" fi diff --git a/NodeBase/generate_relay_config b/NodeBase/generate_relay_config index 6bc8607623..7ae230ddb2 100755 --- a/NodeBase/generate_relay_config +++ b/NodeBase/generate_relay_config @@ -8,7 +8,7 @@ fi if [[ -n "${SE_NODE_RELAY_URL}" ]]; then echo "[relay]" >>"$FILENAME" - echo "url = \"${SE_NODE_RELAY_URL}\"" >>"$FILENAME" + echo "url = \"$(envsubst < <(echo ${SE_NODE_RELAY_URL}))\"" >>"$FILENAME" if [[ -z "${SE_NODE_RELAY_STATUS_ENDPOINT}" ]]; then echo "status-endpoint = \"/status\"" >>"$FILENAME" else @@ -18,7 +18,17 @@ if [[ -n "${SE_NODE_RELAY_URL}" ]]; then echo "protocol-version = \"${SE_NODE_RELAY_PROTOCOL_VERSION}\"" >>"$FILENAME" fi if [[ -z "${SE_NODE_RELAY_STEREOTYPE}" ]]; then - SE_NODE_RELAY_STEREOTYPE="{\"browserName\": \"${SE_NODE_RELAY_BROWSER_NAME}\", \"platformName\": \"${SE_NODE_RELAY_PLATFORM_NAME}\", \"appium:platformVersion\": \"${SE_NODE_RELAY_PLATFORM_VERSION}\"}" + SE_NODE_RELAY_STEREOTYPE="{\"browserName\": \"${SE_NODE_RELAY_BROWSER_NAME:-${SE_NODE_BROWSER_NAME}}\", \"platformName\": \"${SE_NODE_RELAY_PLATFORM_NAME:-${SE_NODE_PLATFORM_NAME}}\"}" + if [[ -n "${SE_NODE_RELAY_PLATFORM_VERSION}" ]]; then + SE_NODE_RELAY_STEREOTYPE="$(python3 /opt/bin/json_merge.py "${SE_NODE_RELAY_STEREOTYPE}" "{\"appium:platformVersion\":\"${SE_NODE_RELAY_PLATFORM_VERSION}\"}")" + fi + BROWSER_VERSION=${SE_NODE_RELAY_BROWSER_VERSION:-${SE_NODE_BROWSER_VERSION}} + if [[ -n "${BROWSER_VERSION}" ]] && [[ "${BROWSER_VERSION}" != "stable" ]]; then + SE_NODE_RELAY_STEREOTYPE="$(python3 /opt/bin/json_merge.py "${SE_NODE_RELAY_STEREOTYPE}" "{\"browserVersion\":\"${BROWSER_VERSION}\"}")" + fi + if [[ "${SE_NODE_ENABLE_MANAGED_DOWNLOADS}" = "true" ]]; then + SE_NODE_RELAY_STEREOTYPE="$(python3 /opt/bin/json_merge.py "${SE_NODE_RELAY_STEREOTYPE}" "{\"se:downloadsEnabled\": true}")" + fi else SE_NODE_RELAY_STEREOTYPE="${SE_NODE_RELAY_STEREOTYPE}" fi @@ -31,5 +41,5 @@ if [[ -n "${SE_NODE_RELAY_URL}" ]]; then echo "Merged relay stereotype: ${SE_NODE_RELAY_STEREOTYPE}" fi fi - echo "configs = ['${SE_NODE_RELAY_MAX_SESSIONS}', '${SE_NODE_RELAY_STEREOTYPE}']" >>"$FILENAME" + echo "configs = ['${SE_NODE_MAX_SESSIONS:-${SE_NODE_RELAY_MAX_SESSIONS}}', '${SE_NODE_RELAY_STEREOTYPE}']" >>"$FILENAME" fi diff --git a/Standalone/generate_config b/Standalone/generate_config index 5ca98d017c..67f98c6061 100755 --- a/Standalone/generate_config +++ b/Standalone/generate_config @@ -44,7 +44,10 @@ fi SE_NODE_CONTAINER_NAME="${SE_NODE_CONTAINER_NAME:-$(hostname)}" if [[ -z "$SE_NODE_STEREOTYPE" ]]; then - SE_NODE_STEREOTYPE="{\"browserName\": \"${SE_NODE_BROWSER_NAME}\", \"browserVersion\": \"${SE_NODE_BROWSER_VERSION}\", \"platformName\": \"${SE_NODE_PLATFORM_NAME}\", ${SE_BROWSER_BINARY_LOCATION}, \"se:containerName\": \"${SE_NODE_CONTAINER_NAME}\", \"container:hostname\": \"$(hostname)\"}" + SE_NODE_STEREOTYPE="{\"browserName\": \"${SE_NODE_BROWSER_NAME}\", \"browserVersion\": \"${SE_NODE_BROWSER_VERSION}\", \"platformName\": \"${SE_NODE_PLATFORM_NAME}\", \"se:containerName\": \"${SE_NODE_CONTAINER_NAME}\", \"container:hostname\": \"$(hostname)\"}" + if [[ -n "${SE_BROWSER_BINARY_LOCATION}" ]]; then + SE_NODE_STEREOTYPE="$(python3 /opt/bin/json_merge.py "${SE_NODE_STEREOTYPE}" "{${SE_BROWSER_BINARY_LOCATION}}")" + fi else SE_NODE_STEREOTYPE="$SE_NODE_STEREOTYPE" fi diff --git a/charts/selenium-grid/CONFIGURATION.md b/charts/selenium-grid/CONFIGURATION.md index 22d66baebd..3b24c595cd 100644 --- a/charts/selenium-grid/CONFIGURATION.md +++ b/charts/selenium-grid/CONFIGURATION.md @@ -593,6 +593,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | edgeNode.sidecars | list | `[]` | It is used to add sidecars proxy in the same pod of the browser node. It means it will add a new container to the deployment itself. It should be set using the --set-json option | | edgeNode.videoRecorder | object | `{}` | Override specific video recording settings for edge node | | relayNode.enabled | bool | `false` | Enable relay nodes | +| relayNode.relayUrl | string | `""` | Specify another Grid, another network, or a cloud vendor that you wish to connect to (e.g. https://ondemand.us-west-1.saucelabs.com/wd/hub) | | relayNode.deploymentEnabled | bool | `true` | NOTE: Only used when autoscaling.enabled is false Enable creation of Deployment true (default) - if you want long-living pods false - for provisioning your own custom type such as Jobs | | relayNode.updateStrategy | object | `{"type":"RollingUpdate"}` | Global update strategy will be overwritten by individual component | | relayNode.replicas | int | `1` | Number of relay nodes | @@ -642,10 +643,10 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | relayNode.scaledOptions | string | `nil` | Override the scaled options for relay nodes | | relayNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for relay nodes | | relayNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for relay nodes | -| relayNode.hpa.browserName | string | `"chrome"` | browserName should match with Node stereotype and request capability is scaled by this scaler | +| relayNode.hpa.browserName | string | `""` | browserName should match with Node stereotype and request capability is scaled by this scaler | | relayNode.hpa.sessionBrowserName | string | `""` | sessionBrowserName if the browserName is different from the sessionBrowserName | | relayNode.hpa.browserVersion | string | `""` | browserVersion should match with Node stereotype and request capability is scaled by this scaler | -| relayNode.hpa.platformName | string | `"Android"` | platformName should match with Node stereotype and request capability is scaled by this scaler | +| relayNode.hpa.platformName | string | `""` | platformName should match with Node stereotype and request capability is scaled by this scaler | | relayNode.hpa.unsafeSsl | string | `"{{ template \"seleniumGrid.graphqlURL.unsafeSsl\" . }}"` | Skip check SSL when connecting to the Graphql endpoint | | relayNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option | | relayNode.sidecars | list | `[]` | It is used to add sidecars proxy in the same pod of the browser node. It means it will add a new container to the deployment itself. It should be set using the --set-json option | diff --git a/charts/selenium-grid/README.md b/charts/selenium-grid/README.md index ea1b84bec9..7946d095b2 100644 --- a/charts/selenium-grid/README.md +++ b/charts/selenium-grid/README.md @@ -360,6 +360,11 @@ For example [multiple-nodes-platform.yaml](./multiple-nodes-platform.yaml) file, For example [multiple-nodes-platform-version.yaml](./multiple-nodes-platform-version.yaml) file, it defines multiple scalers with `platformName: 'Linux'` and last few previous stable versions per browser node to scale against requests with `browserVersion` and `platformName` capabilities. +To extend your Grid, you can use Relay Node (which allows you to route Grid tests to another Grid, another network, or a cloud vendor). +Besides on-prem browser Nodes with Linux-based, you also can serve test requests with other platforms, browsers or even mobile devices which provided by cloud vendors. +Your teams will not worry about the underlying infrastructure, they just request to the single Grid endpoint hosted in your organization. +Check out values file [multiple-nodes-platform-relay.yaml](./multiple-nodes-platform-relay.yaml) for more details. + While deploying the chart, you can quickly use these extra values files by passing the file via `--values` flag to apply. ### Settings fixed-sized thread pool for the Distributor to create new sessions diff --git a/charts/selenium-grid/multiple-nodes-platform-relay.yaml b/charts/selenium-grid/multiple-nodes-platform-relay.yaml new file mode 100644 index 0000000000..3139325f3a --- /dev/null +++ b/charts/selenium-grid/multiple-nodes-platform-relay.yaml @@ -0,0 +1,70 @@ +# Utilize Relay Node to set up hybrid Autoscaling Grid with using on-premise and test cloud provider (e.g. SauceLabs, BrowserStack, etc.) +# +# For example: below incoming requests will be served by Node container on-premise +# options = ChromeOptions() +# options.set_capability('platformName', 'Linux') +# driver = webdriver.Remote(options=options, command_executor=SELENIUM_GRID_URL) +# +# Below incoming requests will be served by Relay Node where commands are forwarded to test cloud provider +# options = ChromeOptions() +# options.set_capability('platformName', 'macOS 13.0') +# driver = webdriver.Remote(options=options, command_executor=SELENIUM_GRID_URL) +crossBrowsers: + chromeNode: + - nameOverride: '{{ $.Release.Name }}-node-chrome-linux' + hpa: + platformName: 'Linux' + firefoxNode: + - nameOverride: '{{ $.Release.Name }}-node-firefox-linux' + hpa: + platformName: 'Linux' + edgeNode: + - nameOverride: '{{ $.Release.Name }}-node-edge-linux' + hpa: + platformName: 'Linux' + relayNode: + - nameOverride: '{{ $.Release.Name }}-node-relay-chrome-macos' + hpa: + browserName: 'chrome' + platformName: 'macOS' + - nameOverride: '{{ $.Release.Name }}-node-relay-chrome-windows' + hpa: + browserName: 'chrome' + platformName: 'Windows 11' + - nameOverride: '{{ $.Release.Name }}-node-relay-firefox-macos' + hpa: + browserName: 'firefox' + platformName: 'macOS' + - nameOverride: '{{ $.Release.Name }}-node-relay-firefox-windows' + hpa: + browserName: 'firefox' + platformName: 'Windows 11' + - nameOverride: '{{ $.Release.Name }}-node-relay-edge-macos' + hpa: + browserName: "MicrosoftEdge" + sessionBrowserName: "msedge" + platformName: 'macOS' + - nameOverride: '{{ $.Release.Name }}-node-relay-edge-windows' + hpa: + browserName: "MicrosoftEdge" + sessionBrowserName: "msedge" + platformName: 'Windows 11' + - nameOverride: '{{ $.Release.Name }}-node-relay-safari-macos' + hpa: + browserName: 'safari' + platformName: 'macOS' + +relayNode: + enabled: true +# relayUrl: "https://ondemand.$SAUCE_REGION.saucelabs.com:443/wd/hub" + videoRecorder: + enabled: false + extraEnvironmentVariables: +# - name: SAUCE_REGION +# value: "" +# Or can give relay url directly to environment variable SE_NODE_RELAY_URL. Value can be referring to value of existing env vars. +# - name: SE_NODE_RELAY_URL +# value: "https://ondemand.$SAUCE_REGION.saucelabs.com:443/wd/hub" + extraEnvFrom: +# - secretRef: +# name: your-secret-with-all-env-vars diff --git a/charts/selenium-grid/multiple-nodes-platform.yaml b/charts/selenium-grid/multiple-nodes-platform.yaml index 1819d8b428..a15e9f112b 100644 --- a/charts/selenium-grid/multiple-nodes-platform.yaml +++ b/charts/selenium-grid/multiple-nodes-platform.yaml @@ -21,7 +21,7 @@ crossBrowsers: browserVersion: '' - nameOverride: '{{ $.Release.Name }}-node-chrome-platform-windows' hpa: - platformName: 'windows' + platformName: 'Windows 11' browserVersion: '' firefoxNode: - nameOverride: '{{ $.Release.Name }}-node-firefox-platform-any' @@ -34,7 +34,7 @@ crossBrowsers: browserVersion: '' - nameOverride: '{{ $.Release.Name }}-node-firefox-platform-windows' hpa: - platformName: 'windows' + platformName: 'Windows 11' browserVersion: '' edgeNode: - nameOverride: '{{ $.Release.Name }}-node-edge-platform-any' @@ -47,5 +47,5 @@ crossBrowsers: browserVersion: '' - nameOverride: '{{ $.Release.Name }}-node-edge-platform-windows' hpa: - platformName: 'windows' + platformName: 'Windows 11' browserVersion: '' diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index fbc286d112..9e609f7a97 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -369,6 +369,14 @@ template: value: {{ $nodeCustomCapabilities | quote }} - name: SE_DRAIN_AFTER_SESSION_COUNT value: {{ and (eq (include "seleniumGrid.useKEDA" $) "true") (eq .Values.autoscaling.scalingType "job") | ternary $nodeMaxSessions 0 | quote }} + {{- with .node.relayUrl }} + - name: SE_NODE_RELAY_URL + value: {{ . | quote }} + {{- end }} + {{- if and (eq (include "seleniumGrid.useKEDA" $) "true") }} + - name: SE_NODE_BROWSER_NAME + value: {{ if hasKey .node.hpa "browserName" }}{{ .node.hpa.browserName | quote }}{{ else }}""{{ end }} + {{- end }} {{- if and (eq (include "seleniumGrid.useKEDA" $) "true") }} - name: SE_NODE_BROWSER_VERSION value: {{ if hasKey .node.hpa "browserVersion" }}{{ .node.hpa.browserVersion | quote }}{{ else }}""{{ end }} diff --git a/charts/selenium-grid/templates/relay-node-scaledjobs.yaml b/charts/selenium-grid/templates/relay-node-scaledjobs.yaml index eec44d8eff..c5096e5fb8 100644 --- a/charts/selenium-grid/templates/relay-node-scaledjobs.yaml +++ b/charts/selenium-grid/templates/relay-node-scaledjobs.yaml @@ -28,7 +28,7 @@ spec: {{- $_ := set $podScope "name" (include "seleniumGrid.relayNode.fullname" (list $nodeConfig $)) -}} {{- $_ = set $podScope "node" $nodeConfig -}} {{- $_ = set $podScope "recorder" (mergeOverwrite $.Values.videoRecorder $nodeConfig.videoRecorder) -}} - {{- $_ = set $podScope "uploader" (get $.Values.videoRecorder (.Values.videoRecorder.uploader.name | toString)) -}} + {{- $_ = set $podScope "uploader" (get $.Values.videoRecorder ($podScope.recorder.uploader.name | toString)) -}} {{- $_ = set $podScope "podTemplate" (include "seleniumGrid.podTemplate" $podScope | fromYaml) }} {{- include "seleniumGrid.autoscalingTemplate" $podScope | nindent 2 }} --- diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 78cb13c196..f752b8d8f1 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -1704,7 +1704,8 @@ edgeNode: relayNode: # -- Enable relay nodes enabled: false - + # -- Specify another Grid, another network, or a cloud vendor that you wish to connect to (e.g. https://ondemand.us-west-1.saucelabs.com/wd/hub) + relayUrl: "" # -- NOTE: Only used when autoscaling.enabled is false # Enable creation of Deployment # true (default) - if you want long-living pods @@ -1875,13 +1876,13 @@ relayNode: scaledObjectOptions: hpa: # -- browserName should match with Node stereotype and request capability is scaled by this scaler - browserName: "chrome" + browserName: "" # -- sessionBrowserName if the browserName is different from the sessionBrowserName sessionBrowserName: "" # -- browserVersion should match with Node stereotype and request capability is scaled by this scaler browserVersion: "" # -- platformName should match with Node stereotype and request capability is scaled by this scaler - platformName: "Android" + platformName: "" # -- Skip check SSL when connecting to the Graphql endpoint unsafeSsl: '{{ template "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional diff --git a/tests/SeleniumTests/__init__.py b/tests/SeleniumTests/__init__.py index b5632ea56b..2802d1fee9 100644 --- a/tests/SeleniumTests/__init__.py +++ b/tests/SeleniumTests/__init__.py @@ -33,15 +33,20 @@ TEST_CUSTOM_SPECIFIC_NAME = os.environ.get('TEST_CUSTOM_SPECIFIC_NAME', 'false').lower() == 'true' TEST_MULTIPLE_VERSIONS = os.environ.get('TEST_MULTIPLE_VERSIONS', 'false').lower() == 'true' TEST_MULTIPLE_PLATFORMS = os.environ.get('TEST_MULTIPLE_PLATFORMS', 'false').lower() == 'true' +TEST_MULTIPLE_PLATFORMS_RELAY = os.environ.get('TEST_MULTIPLE_PLATFORMS_RELAY', 'false').lower() == 'true' TEST_MULTIPLE_VERSIONS_EXPLICIT = os.environ.get('TEST_MULTIPLE_VERSIONS_EXPLICIT', 'true').lower() == 'true' LIST_CHROMIUM_VERSIONS = ['130.0', '129.0', '128.0'] LIST_FIREFOX_VERSIONS = ['132.0', '131.0', '130.0', '129.0', '128.0'] -LIST_PLATFORMS = ['Linux', None, 'Windows'] +LIST_PLATFORMS = ['Linux', None, 'Windows 11'] if not TEST_MULTIPLE_VERSIONS_EXPLICIT: LIST_CHROMIUM_VERSIONS.append(None) LIST_FIREFOX_VERSIONS.append(None) +if TEST_MULTIPLE_PLATFORMS_RELAY: + # Replace index with None to macOS + LIST_PLATFORMS[1] = 'macOS' + SELENIUM_GRID_URL = f"{SELENIUM_GRID_PROTOCOL}://{SELENIUM_GRID_HOST}:{SELENIUM_GRID_PORT}" CLIENT_CONFIG = ClientConfig( remote_server_addr=SELENIUM_GRID_URL, @@ -185,6 +190,13 @@ def setUp(self): platform_name = random.choice(LIST_PLATFORMS) if platform_name: options.set_capability('platformName', platform_name) + if TEST_MULTIPLE_PLATFORMS_RELAY: + options.set_capability('sauce:options', { + 'username': os.environ.get('SAUCE_USERNAME'), + 'accessKey': os.environ.get('SAUCE_ACCESS_KEY'), + 'name': f"{self._testMethodName} ({self.__class__.__name__})", + 'seleniumVersion': '4.29.0', + }) start_time = time.time() self.driver = webdriver.Remote( options=options, @@ -223,6 +235,13 @@ def setUp(self): platform_name = random.choice(LIST_PLATFORMS) if platform_name: options.set_capability('platformName', platform_name) + if TEST_MULTIPLE_PLATFORMS_RELAY: + options.set_capability('sauce:options', { + 'username': os.environ.get('SAUCE_USERNAME'), + 'accessKey': os.environ.get('SAUCE_ACCESS_KEY'), + 'name': f"{self._testMethodName} ({self.__class__.__name__})", + 'seleniumVersion': '4.29.0', + }) start_time = time.time() self.driver = webdriver.Remote( options=options, @@ -266,6 +285,13 @@ def setUp(self): platform_name = random.choice(LIST_PLATFORMS) if platform_name: options.set_capability('platformName', platform_name) + if TEST_MULTIPLE_PLATFORMS_RELAY: + options.set_capability('sauce:options', { + 'username': os.environ.get('SAUCE_USERNAME'), + 'accessKey': os.environ.get('SAUCE_ACCESS_KEY'), + 'name': f"{self._testMethodName} ({self.__class__.__name__})", + 'seleniumVersion': '4.29.0', + }) start_time = time.time() self.driver = webdriver.Remote( options=options, diff --git a/tests/charts/ci/DeploymentAutoscaling-values.yaml b/tests/charts/ci/DeploymentAutoscaling-values.yaml index cb0b103c5f..d9f1949f50 100644 --- a/tests/charts/ci/DeploymentAutoscaling-values.yaml +++ b/tests/charts/ci/DeploymentAutoscaling-values.yaml @@ -100,3 +100,10 @@ firefoxNode: enabled: *livenessProbe extraVolumeMounts: *extraVolumeMounts extraVolumes: *extraVolumes + +# Configuration for relay nodes +relayNode: + relayUrl: "https://ondemand.$SAUCE_REGION.saucelabs.com:443/wd/hub" + extraEnvFrom: + - secretRef: + name: test-cloud-credentials diff --git a/tests/charts/ci/JobAutoscaling-values.yaml b/tests/charts/ci/JobAutoscaling-values.yaml index d19412cc17..6310351970 100644 --- a/tests/charts/ci/JobAutoscaling-values.yaml +++ b/tests/charts/ci/JobAutoscaling-values.yaml @@ -46,3 +46,9 @@ firefoxNode: enabled: *readinessProbe livenessProbe: enabled: *livenessProbe +# Configuration for relay nodes +relayNode: + relayUrl: "https://ondemand.$SAUCE_REGION.saucelabs.com:443/wd/hub" + extraEnvFrom: + - secretRef: + name: test-cloud-credentials diff --git a/tests/charts/ci/base-resources-values.yaml b/tests/charts/ci/base-resources-values.yaml index fbeb383ff8..9269e4329a 100644 --- a/tests/charts/ci/base-resources-values.yaml +++ b/tests/charts/ci/base-resources-values.yaml @@ -76,6 +76,15 @@ edgeNode: cpu: 250m memory: 2500Mi +relayNode: + resources: + requests: + cpu: 50m + memory: 256Mi + limits: + cpu: 250m + memory: 2500Mi + videoRecorder: resources: requests: diff --git a/tests/charts/ci/nameOverride-values.yaml b/tests/charts/ci/nameOverride-values.yaml index 0d156cb080..a9a2e57b52 100644 --- a/tests/charts/ci/nameOverride-values.yaml +++ b/tests/charts/ci/nameOverride-values.yaml @@ -28,4 +28,4 @@ secrets: nameOverride: "selenium-grid-common-secrets" autoscaling: patchObjectFinalizers: - nameOverride: "selenium-grid-job-patch-objects" + nameOverride: "selenium-grid-patch" diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index 7c22d552de..790abdd3bb 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -66,6 +66,7 @@ TEST_EXISTING_PTS=${TEST_EXISTING_PTS:-"false"} TEST_MULTIPLE_VERSIONS=${TEST_MULTIPLE_VERSIONS:-"false"} TEST_MULTIPLE_VERSIONS_EXPLICIT=${TEST_MULTIPLE_VERSIONS_EXPLICIT:-"true"} TEST_MULTIPLE_PLATFORMS=${TEST_MULTIPLE_PLATFORMS:-"false"} +TEST_MULTIPLE_PLATFORMS_RELAY=${TEST_MULTIPLE_PLATFORMS_RELAY:-"false"} TEST_CUSTOM_SPECIFIC_NAME=${TEST_CUSTOM_SPECIFIC_NAME:-"false"} wait_for_terminated() { @@ -91,6 +92,7 @@ cleanup() { kubectl logs -n ${SELENIUM_NAMESPACE} $pod --all-containers --tail=10000 > tests/tests/pod_logs_${pod}.txt done if [ "${SKIP_CLEANUP}" = "false" ] || [ "${CI:-false}" != "false" ]; then + helm ls -A echo "Clean up chart release and namespace" helm delete ${RELEASE_NAME} --namespace ${SELENIUM_NAMESPACE} || true wait_for_terminated @@ -112,12 +114,12 @@ on_failure() { kubectl describe pod -n ${SELENIUM_NAMESPACE} >> tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt echo "There is step failed with exit status $exit_status" sudo chmod -R 777 ${HOST_PATH}/logs + cleanup exit $exit_status } # Trap ERR signal and call on_failure function -trap 'on_failure' ERR EXIT -trap 'cleanup' ERR +trap 'on_failure' ERR if [ "${RENDER_HELM_TEMPLATE_ONLY}" != "true" ]; then rm -rf tests/tests/* @@ -347,11 +349,18 @@ if [ "${SECURE_USE_EXTERNAL_CERT}" = "true" ] && [ "${RENDER_HELM_TEMPLATE_ONLY} --from-file=tls.crt=${cert_dir}/tls.crt \ --from-file=tls.key=${cert_dir}/tls.key \ --from-file=server.jks=${cert_dir}/server.jks \ - --from-file=server.pass=${cert_dir}/server.pass + --from-file=server.pass=${cert_dir}/server.pass \ + --dry-run=client -o yaml | kubectl apply -n ${SELENIUM_NAMESPACE} -f - fi CHART_CERT_PATH="./tests/tests/tls.crt" fi +if [ "${RENDER_HELM_TEMPLATE_ONLY}" != "true" ]; then + kubectl create secret generic -n ${SELENIUM_NAMESPACE} test-cloud-credentials \ + --from-literal=SAUCE_REGION=${SAUCE_REGION} \ + --dry-run=client -o yaml | kubectl apply -n ${SELENIUM_NAMESPACE} -f - +fi + if [ "${SECURE_INGRESS_ONLY_CONFIG_INLINE}" = "true" ]; then if [ "${SECURE_USE_EXTERNAL_CERT}" = "true" ]; then HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ @@ -421,10 +430,14 @@ if [ "${TEST_MULTIPLE_VERSIONS}" = "true" ]; then HELM_COMMAND_SET_BASE_VALUES="${HELM_COMMAND_SET_BASE_VALUES} \ --values ${CHART_PATH}/multiple-nodes-platform-version.yaml \ " -elif [ "${TEST_MULTIPLE_PLATFORMS}" = "true" ]; then +elif [ "${TEST_MULTIPLE_PLATFORMS}" = "true" ] && [ "${TEST_MULTIPLE_PLATFORMS_RELAY}" != "true" ]; then HELM_COMMAND_SET_BASE_VALUES="${HELM_COMMAND_SET_BASE_VALUES} \ --values ${CHART_PATH}/multiple-nodes-platform.yaml \ " +elif [ "${TEST_MULTIPLE_PLATFORMS_RELAY}" = "true" ]; then + HELM_COMMAND_SET_BASE_VALUES="${HELM_COMMAND_SET_BASE_VALUES} \ + --values ${CHART_PATH}/multiple-nodes-platform-relay.yaml \ + " fi HELM_COMMAND_SET_BASE_VALUES="${HELM_COMMAND_SET_BASE_VALUES} \ @@ -504,6 +517,7 @@ export TEST_AUTOSCALING_ITERATIONS=${TEST_AUTOSCALING_ITERATIONS:-"20"} export TEST_MULTIPLE_VERSIONS=${TEST_MULTIPLE_VERSIONS} export TEST_MULTIPLE_VERSIONS_EXPLICIT=${TEST_MULTIPLE_VERSIONS_EXPLICIT} export TEST_MULTIPLE_PLATFORMS=${TEST_MULTIPLE_PLATFORMS} +export TEST_MULTIPLE_PLATFORMS_RELAY=${TEST_MULTIPLE_PLATFORMS_RELAY} export TEST_CUSTOM_SPECIFIC_NAME=${TEST_CUSTOM_SPECIFIC_NAME} if [ "${MATRIX_BROWSER}" = "NoAutoscaling" ]; then ./tests/bootstrap.sh NodeFirefox @@ -527,5 +541,3 @@ else fi wait_for_terminated - -cleanup