From d4eae82f80da6f86cbf2284a35157d3774bd7740 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Thu, 17 Jul 2025 21:02:29 +0700 Subject: [PATCH] K8s: Resilient Helm config to override `SE_DRAIN_AFTER_SESSION_COUNT` Signed-off-by: Viet Nguyen Duc --- Makefile | 4 ++-- charts/selenium-grid/CONFIGURATION.md | 5 +++++ charts/selenium-grid/templates/_helpers.tpl | 3 ++- charts/selenium-grid/values.yaml | 12 ++++++++++++ tests/charts/make/chart_test.sh | 2 ++ 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 2b5437a153..0440d86e4d 100644 --- a/Makefile +++ b/Makefile @@ -1004,7 +1004,7 @@ chart_test_autoscaling_disabled: chart_test_autoscaling_deployment_https: PLATFORMS=$(PLATFORMS) CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_BASIC_AUTH=true TEST_EXTERNAL_DATASTORE=postgresql TEST_MULTIPLE_VERSIONS=true AUTOSCALING_COOLDOWN_PERIOD=30 SELENIUM_GRID_MONITORING=false TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) \ SECURE_INGRESS_ONLY_DEFAULT=true INGRESS_DISABLE_USE_HTTP2=true SELENIUM_GRID_PROTOCOL=https CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_PORT=443 \ - SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=0 MAX_SESSIONS_FIREFOX=1 MAX_SESSIONS_EDGE=1 MAX_SESSIONS_CHROME=1 TEST_NAME_OVERRIDE=true \ + SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=0 MAX_SESSIONS_FIREFOX=1 MAX_SESSIONS_EDGE=1 MAX_SESSIONS_CHROME=1 TEST_NODE_DRAIN_AFTER_SESSION_COUNT=3 TEST_NAME_OVERRIDE=true \ VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \ TEMPLATE_OUTPUT_FILENAME="k8s_fullDistributed_basicAuth_secureIngress_defaultCerts_ingressHostName_disableHttp2_autoScaling_patchKEDA_scaledObject_subPath.yaml" \ ./tests/charts/make/chart_test.sh DeploymentAutoscaling @@ -1013,7 +1013,7 @@ chart_test_autoscaling_deployment: PLATFORMS=$(PLATFORMS) TEST_EXISTING_KEDA=true RELEASE_NAME=selenium CHART_ENABLE_TRACING=true TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) AUTOSCALING_COOLDOWN_PERIOD=30 \ TRACING_EXPORTER_ENDPOINT="http://\$$KUBERNETES_NODE_HOST_IP:4317" TEST_CUSTOM_SPECIFIC_NAME=true \ SECURE_CONNECTION_SERVER=true SECURE_USE_EXTERNAL_CERT=true SERVICE_TYPE_NODEPORT=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -I | cut -d' ' -f1) SELENIUM_GRID_PORT=31444 \ - SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 SET_MAX_REPLICAS=3 TEST_DELAY_AFTER_TEST=2 SELENIUM_GRID_MONITORING=false \ + SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 SET_MAX_REPLICAS=3 TEST_DELAY_AFTER_TEST=2 TEST_NODE_DRAIN_AFTER_SESSION_COUNT=3 SELENIUM_GRID_MONITORING=false \ VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \ TEMPLATE_OUTPUT_FILENAME="k8s_prefixSelenium_enableTracing_secureServer_externalCerts_nodePort_autoScaling_scaledObject_existingKEDA_subPath.yaml" \ ./tests/charts/make/chart_test.sh DeploymentAutoscaling diff --git a/charts/selenium-grid/CONFIGURATION.md b/charts/selenium-grid/CONFIGURATION.md index ccabfed3ca..60b2befd9d 100644 --- a/charts/selenium-grid/CONFIGURATION.md +++ b/charts/selenium-grid/CONFIGURATION.md @@ -50,6 +50,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | global.seleniumGrid.affinity | object | `{}` | Specify affinity for all components, can be overridden individually | | global.seleniumGrid.topologySpreadConstraints | list | `[]` | Specify topologySpreadConstraints for all components, can be overridden individually | | global.seleniumGrid.nodeMaxSessions | int | `1` | Specify number of max sessions per node. Can be overridden by individual component (this is also set to scaler trigger parameter `nodeMaxSessions` if `autoscaling` is enabled) | +| global.seleniumGrid.nodeDrainAfterSessionCount | int | `0` | Set number of sessions will be executed in a Node before detaching it from Hub and shutting it down | | global.seleniumGrid.nodeEnableManagedDownloads | bool | `true` | This causes the Node to auto manage files downloaded for a given session on the Node (https://www.selenium.dev/documentation/webdriver/drivers/remote_webdriver/#enable-downloads-in-the-grid) | | global.seleniumGrid.nodeCustomCapabilities | string | `""` | Setting custom capabilities for matching specific Nodes (https://www.selenium.dev/documentation/grid/configuration/toml_options/#setting-custom-capabilities-for-matching-specific-nodes) | | global.seleniumGrid.nodeRegisterPeriod | int | `120` | How long, in seconds, will the Node try to register to the Distributor for the first time. After this period is completed, the Node will not attempt to register again. | @@ -468,6 +469,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | chromeNode.extraVolumeMounts | list | `[]` | Extra volume mounts for chrome-node container | | chromeNode.extraVolumes | list | `[]` | Extra volumes for chrome-node pod | | chromeNode.nodeMaxSessions | string | `nil` | Override the number of max sessions per node | +| chromeNode.nodeDrainAfterSessionCount | string | `nil` | Override the number of sessions to run before draining the node | | chromeNode.nodeEnableManagedDownloads | string | `nil` | Override the managed downloads in node | | chromeNode.nodeCustomCapabilities | string | `""` | Override the same config at the global level | | chromeNode.nodeRegisterPeriod | string | `nil` | Override the same config at the global level | @@ -526,6 +528,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | firefoxNode.extraVolumeMounts | list | `[]` | Extra volume mounts for firefox-node container | | firefoxNode.extraVolumes | list | `[]` | Extra volumes for firefox-node pod | | firefoxNode.nodeMaxSessions | string | `nil` | Override the number of max sessions per node | +| firefoxNode.nodeDrainAfterSessionCount | string | `nil` | Override the number of sessions to run before draining the node | | firefoxNode.nodeEnableManagedDownloads | string | `nil` | Override the managed downloads in node | | firefoxNode.nodeCustomCapabilities | string | `""` | Override the same config at the global level | | firefoxNode.nodeRegisterPeriod | string | `nil` | Override the same config at the global level | @@ -584,6 +587,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | edgeNode.extraVolumeMounts | list | `[]` | Extra volume mounts for edge-node container | | edgeNode.extraVolumes | list | `[]` | Extra volumes for edge-node pod | | edgeNode.nodeMaxSessions | string | `nil` | Override the number of max sessions per node | +| edgeNode.nodeDrainAfterSessionCount | string | `nil` | Override the number of sessions to run before draining the node | | edgeNode.nodeEnableManagedDownloads | string | `nil` | Override the managed downloads in node | | edgeNode.nodeCustomCapabilities | string | `""` | Override the same config at the global level | | edgeNode.nodeRegisterPeriod | string | `nil` | Override the same config at the global level | @@ -643,6 +647,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | relayNode.extraVolumeMounts | list | `[]` | Extra volume mounts for relay-node container | | relayNode.extraVolumes | list | `[]` | Extra volumes for relay-node pod | | relayNode.nodeMaxSessions | string | `nil` | Override the number of max sessions per node | +| relayNode.nodeDrainAfterSessionCount | string | `nil` | Override the number of sessions to run before draining the node | | relayNode.nodeEnableManagedDownloads | string | `nil` | Override the managed downloads in node | | relayNode.nodeCustomCapabilities | string | `""` | Override the same config at the global level | | relayNode.nodeRegisterPeriod | string | `nil` | Override the same config at the global level | diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index 29c892e534..7ff22228de 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -306,6 +306,7 @@ Common pod template {{- $videoImageRegistry := default $.Values.global.seleniumGrid.imageRegistry .recorder.imageRegistry -}} {{- $videoImageTag := default $.Values.global.seleniumGrid.videoImageTag .recorder.imageTag -}} {{- $nodeMaxSessions := default $.Values.global.seleniumGrid.nodeMaxSessions .node.nodeMaxSessions | int64 -}} +{{- $nodeDrainAfterSessionCount := default $.Values.global.seleniumGrid.nodeDrainAfterSessionCount .node.nodeDrainAfterSessionCount | int64 -}} {{- $nodeEnableManagedDownloads := default $.Values.global.seleniumGrid.nodeEnableManagedDownloads .node.nodeEnableManagedDownloads -}} {{- $nodeCustomCapabilities := default $.Values.global.seleniumGrid.nodeCustomCapabilities .node.nodeCustomCapabilities -}} {{- $nodeRegisterPeriod := default $.Values.global.seleniumGrid.nodeRegisterPeriod .node.nodeRegisterPeriod | int64 -}} @@ -379,7 +380,7 @@ template: - name: SE_NODE_STEREOTYPE_EXTRA value: {{ $nodeCustomCapabilities | quote }} - name: SE_DRAIN_AFTER_SESSION_COUNT - value: {{ and (eq (include "seleniumGrid.useKEDA" $) "true") (eq .Values.autoscaling.scalingType "job") | ternary $nodeMaxSessions 0 | quote }} + value: {{ and (eq (include "seleniumGrid.useKEDA" $) "true") (eq .Values.autoscaling.scalingType "job") | ternary (max $nodeMaxSessions $nodeDrainAfterSessionCount) $nodeDrainAfterSessionCount | quote }} {{- with .node.relayUrl }} - name: SE_NODE_RELAY_URL value: {{ . | quote }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 182a7889b2..79c6aa4475 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -52,6 +52,10 @@ global: # Note: If not define labelSelector, it will be added automatically based on "app" label in each component # -- Specify number of max sessions per node. Can be overridden by individual component (this is also set to scaler trigger parameter `nodeMaxSessions` if `autoscaling` is enabled) nodeMaxSessions: 1 + # Noted:In case of autoscaling enabled, with scaling type `job`, Node will be drained following `nodeMaxSessions` by default + # If changing `nodeDrainAfterSessionCount` > `nodeMaxSessions` it will take precedence over `nodeMaxSessions` in scaling type `job` + # -- Set number of sessions will be executed in a Node before detaching it from Hub and shutting it down + nodeDrainAfterSessionCount: 0 # -- This causes the Node to auto manage files downloaded for a given session on the Node (https://www.selenium.dev/documentation/webdriver/drivers/remote_webdriver/#enable-downloads-in-the-grid) nodeEnableManagedDownloads: true # -- Setting custom capabilities for matching specific Nodes (https://www.selenium.dev/documentation/grid/configuration/toml_options/#setting-custom-capabilities-for-matching-specific-nodes) @@ -1290,6 +1294,8 @@ chromeNode: # -- Override the number of max sessions per node nodeMaxSessions: + # -- Override the number of sessions to run before draining the node + nodeDrainAfterSessionCount: # -- Override the managed downloads in node nodeEnableManagedDownloads: # -- Override the same config at the global level @@ -1486,6 +1492,8 @@ firefoxNode: # -- Override the number of max sessions per node nodeMaxSessions: + # -- Override the number of sessions to run before draining the node + nodeDrainAfterSessionCount: # -- Override the managed downloads in node nodeEnableManagedDownloads: # -- Override the same config at the global level @@ -1682,6 +1690,8 @@ edgeNode: # -- Override the number of max sessions per node nodeMaxSessions: + # -- Override the number of sessions to run before draining the node + nodeDrainAfterSessionCount: # -- Override the managed downloads in node nodeEnableManagedDownloads: # -- Override the same config at the global level @@ -1879,6 +1889,8 @@ relayNode: # -- Override the number of max sessions per node nodeMaxSessions: + # -- Override the number of sessions to run before draining the node + nodeDrainAfterSessionCount: # -- Override the managed downloads in node nodeEnableManagedDownloads: # -- Override the same config at the global level diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index 886a8b937c..c4e7072e51 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -54,6 +54,7 @@ else fi EXTERNAL_TLS_SECRET_NAME=${EXTERNAL_TLS_SECRET_NAME:-"external-tls-secret-${RESOURCE_ID}"} SELENIUM_ENABLE_MANAGED_DOWNLOADS=${SELENIUM_ENABLE_MANAGED_DOWNLOADS:-"true"} +TEST_NODE_DRAIN_AFTER_SESSION_COUNT=${TEST_NODE_DRAIN_AFTER_SESSION_COUNT:-"0"} TEST_NODE_MAX_SESSIONS=${TEST_NODE_MAX_SESSIONS:-"1"} MAX_SESSIONS_CHROME=${MAX_SESSIONS_CHROME:-${TEST_NODE_MAX_SESSIONS}} MAX_SESSIONS_FIREFOX=${MAX_SESSIONS_FIREFOX:-${TEST_NODE_MAX_SESSIONS}} @@ -184,6 +185,7 @@ HELM_COMMAND_SET_IMAGES=" \ --set global.seleniumGrid.httpLogs=${CHART_ENABLE_TRACING} \ --set isolateComponents=${CHART_FULL_DISTRIBUTED_MODE} \ --set global.seleniumGrid.logLevel=${LOG_LEVEL} \ +--set global.seleniumGrid.nodeDrainAfterSessionCount=${TEST_NODE_DRAIN_AFTER_SESSION_COUNT} \ --set chromeNode.nodeMaxSessions=${MAX_SESSIONS_CHROME} \ --set firefoxNode.nodeMaxSessions=${MAX_SESSIONS_FIREFOX} \ --set edgeNode.nodeMaxSessions=${MAX_SESSIONS_EDGE} \