diff --git a/.keda/README.md b/.keda/README.md index 7b8c082812..364e54d37c 100644 --- a/.keda/README.md +++ b/.keda/README.md @@ -20,7 +20,7 @@ docker pull selenium/keda-admission-webhooks:2.15.1-selenium-grid-20240907 Besides that, you also can use image tag `latest` or `nightly`. -If you are deploying KEDA core using their official Helm chart, you can overwrite the image registry and tag by providing the following values in the `values.yaml` file. For example: +If you are deploying KEDA core using their official Helm [chart](https://github.com/kedacore/charts), you can overwrite the image registry and tag by providing the following values in the `values.yaml` file. For example: ```yaml image: diff --git a/Makefile b/Makefile index 811875590a..8742693d06 100644 --- a/Makefile +++ b/Makefile @@ -913,7 +913,7 @@ chart_test_autoscaling_disabled: chart_test_autoscaling_deployment_https: PLATFORMS=$(PLATFORMS) CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_BASIC_AUTH=true \ SECURE_INGRESS_ONLY_DEFAULT=true INGRESS_DISABLE_USE_HTTP2=true SELENIUM_GRID_PROTOCOL=https CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_PORT=443 \ - SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 \ + SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 MAX_SESSIONS_FIREFOX=3 MAX_SESSIONS_EDGE=2 MAX_SESSIONS_CHROME=1 \ VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \ TEMPLATE_OUTPUT_FILENAME="k8s_fullDistributed_basicAuth_secureIngress_defaultCerts_ingressHostName_disableHttp2_autoScaling_scaledObject_subPath.yaml" \ ./tests/charts/make/chart_test.sh DeploymentAutoscaling @@ -929,6 +929,7 @@ chart_test_autoscaling_deployment: chart_test_autoscaling_job_https: PLATFORMS=$(PLATFORMS) TEST_EXISTING_KEDA=true RELEASE_NAME=selenium CHART_ENABLE_BASIC_AUTH=true \ SECURE_CONNECTION_SERVER=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 SUB_PATH=/ \ + MAX_SESSIONS_FIREFOX=1 MAX_SESSIONS_EDGE=2 MAX_SESSIONS_CHROME=3 \ VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) EXTERNAL_UPLOADER_CONFIG=true \ TEMPLATE_OUTPUT_FILENAME="k8s_prefixSelenium_basicAuth_secureServer_autoScaling_scaledJob_existingKEDA.yaml" \ ./tests/charts/make/chart_test.sh JobAutoscaling diff --git a/Video/video.sh b/Video/video.sh index 865a9dce89..97d98b497d 100755 --- a/Video/video.sh +++ b/Video/video.sh @@ -210,7 +210,7 @@ else if [[ "${endpoint_status}" = "401" ]]; then echo "$(date +%FT%T%Z) [${process_name}] - GraphQL endpoint requires authentication, please set env variables SE_ROUTER_USERNAME and SE_ROUTER_PASSWORD" elif [[ "${endpoint_status}" = "404" ]]; then - echo "$(date +%FT%T%Z) [${process_name}] -GraphQL endpoint could not be found, please check the endpoint ${endpoint_url}" + echo "$(date +%FT%T%Z) [${process_name}] - GraphQL endpoint could not be found, please check the endpoint ${endpoint_url}" fi echo "$(date +%FT%T%Z) [${process_name}] - Start recording: $caps_se_video_record, video file name: $video_file_name" log_node_response diff --git a/charts/selenium-grid/CONFIGURATION.md b/charts/selenium-grid/CONFIGURATION.md index e9315167fb..6359c1d871 100644 --- a/charts/selenium-grid/CONFIGURATION.md +++ b/charts/selenium-grid/CONFIGURATION.md @@ -32,7 +32,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | global.seleniumGrid.imageTag | string | `"4.24.0-20240907"` | Image tag for all selenium components | | global.seleniumGrid.nodesImageTag | string | `"4.24.0-20240907"` | Image tag for browser's nodes | | global.seleniumGrid.videoImageTag | string | `"ffmpeg-7.0.2-20240907"` | Image tag for browser's video recorder | -| global.seleniumGrid.kubectlImage | string | `"jitesoft/kubectl:latest"` | kubectl image is used to execute kubectl commands in utility jobs | +| global.seleniumGrid.kubectlImage | string | `"bitnami/kubectl:latest"` | kubectl image is used to execute kubectl commands in utility jobs | | global.seleniumGrid.imagePullSecret | string | `""` | Pull secret for all components, can be overridden individually | | global.seleniumGrid.logLevel | string | `"INFO"` | Log level for all components. Possible values describe here: https://www.selenium.dev/documentation/grid/configuration/cli_options/#logging | | global.seleniumGrid.defaultNodeStartupProbe | string | `"exec"` | Set default startup probe method for all nodes (supplied values: httpGet, exec). If not set, the default is httpGet | @@ -46,6 +46,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | global.seleniumGrid.updateStrategy.rollingUpdate | object | `{"maxSurge":1,"maxUnavailable":0}` | Specify for strategy RollingUpdate | | global.seleniumGrid.affinity | object | `{}` | Specify affinity for all components, can be overridden individually | | global.seleniumGrid.topologySpreadConstraints | list | `[]` | Specify topologySpreadConstraints for all components, can be overridden individually | +| global.seleniumGrid.nodeMaxSessions | int | `1` | Specify number of max sessions per node. Can be overridden by individual component (this is also set to scaler trigger parameter `nodeMaxSessions` if `autoscaling` is enabled) | | tls.nameOverride | string | `nil` | Name of external secret containing the TLS certificate and key | | tls.enabled | bool | `false` | Enable or disable TLS for the server components (and ingress proxy) | | tls.ingress.enabled | bool | `false` | Enable or disable TLS for the ingress proxy only | @@ -362,6 +363,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | chromeNode.lifecycle | object | `{}` | Define postStart and preStop events. This overwrites the defined preStop in deregisterLifecycle if any | | chromeNode.extraVolumeMounts | list | `[]` | Extra volume mounts for chrome-node container | | chromeNode.extraVolumes | list | `[]` | Extra volumes for chrome-node pod | +| chromeNode.nodeMaxSessions | string | `nil` | Override the number of max sessions per node | | chromeNode.scaledOptions | string | `nil` | Override the scaled options for chrome nodes | | chromeNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for chrome nodes | | chromeNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for chrome nodes | @@ -411,6 +413,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | firefoxNode.lifecycle | object | `{}` | Define postStart and preStop events. This overwrites the defined preStop in deregisterLifecycle if any | | firefoxNode.extraVolumeMounts | list | `[]` | Extra volume mounts for firefox-node container | | firefoxNode.extraVolumes | list | `[]` | Extra volumes for firefox-node pod | +| firefoxNode.nodeMaxSessions | string | `nil` | Override the number of max sessions per node | | firefoxNode.scaledOptions | string | `nil` | Override the scaled options for firefox nodes | | firefoxNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for firefox nodes | | firefoxNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for firefox nodes | @@ -460,6 +463,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | edgeNode.lifecycle | object | `{}` | Define postStart and preStop events. This overwrites the defined preStop in deregisterLifecycle if any | | edgeNode.extraVolumeMounts | list | `[]` | Extra volume mounts for edge-node container | | edgeNode.extraVolumes | list | `[]` | Extra volumes for edge-node pod | +| edgeNode.nodeMaxSessions | string | `nil` | Override the number of max sessions per node | | edgeNode.scaledOptions | string | `nil` | Override the scaled options for edge nodes | | edgeNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for edge nodes | | edgeNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for edge nodes | diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index ce330ec76d..866851c886 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -207,6 +207,7 @@ Common autoscaling spec template */}} {{- define "seleniumGrid.autoscalingTemplate" -}} {{- $spec := toYaml (dict) -}} +{{- $nodeMaxSessions := default $.Values.global.seleniumGrid.nodeMaxSessions .node.nodeMaxSessions | int64 -}} {{/* Merge with precedence from right to left */}} {{- with $.Values.autoscaling.scaledOptions -}} {{- $spec = mergeOverwrite ($spec | fromYaml) . | toYaml -}} @@ -239,7 +240,7 @@ Common autoscaling spec template triggers: - type: selenium-grid metadata: - triggerIndex: '{{ default $.Values.autoscaling.scaledOptions.minReplicaCount (.node.scaledOptions).minReplicaCount }}' + nodeMaxSessions: {{ $nodeMaxSessions | quote }} {{- with .node.hpa }} {{- tpl (toYaml .) $ | nindent 6 }} {{- end }} @@ -278,6 +279,7 @@ Common pod template {{- $nodeImageTag := default $.Values.global.seleniumGrid.nodesImageTag .node.imageTag -}} {{- $videoImageRegistry := default $.Values.global.seleniumGrid.imageRegistry $.Values.videoRecorder.imageRegistry -}} {{- $videoImageTag := default $.Values.global.seleniumGrid.videoImageTag $.Values.videoRecorder.imageTag -}} +{{- $nodeMaxSessions := default $.Values.global.seleniumGrid.nodeMaxSessions .node.nodeMaxSessions | int64 -}} template: metadata: labels: @@ -329,6 +331,14 @@ template: image: {{ printf "%s/%s:%s" $nodeImageRegistry .node.imageName $nodeImageTag }} imagePullPolicy: {{ .node.imagePullPolicy }} env: + - name: SE_NODE_MAX_SESSIONS + value: {{ $nodeMaxSessions | quote }} + {{- if gt $nodeMaxSessions 1 }} + - name: SE_NODE_OVERRIDE_MAX_SESSIONS + value: "true" + {{- end }} + - name: SE_DRAIN_AFTER_SESSION_COUNT + value: {{ and (eq (include "seleniumGrid.useKEDA" $) "true") (eq .Values.autoscaling.scalingType "job") | ternary $nodeMaxSessions 0 | quote }} - name: SE_NODE_CONTAINER_NAME valueFrom: fieldRef: @@ -469,6 +479,10 @@ template: image: {{ printf "%s/%s:%s" $videoImageRegistry $.Values.videoRecorder.imageName $videoImageTag }} imagePullPolicy: {{ $.Values.videoRecorder.imagePullPolicy }} env: + - name: SE_NODE_MAX_SESSIONS + value: {{ $nodeMaxSessions | quote }} + - name: SE_DRAIN_AFTER_SESSION_COUNT + value: {{ and (eq (include "seleniumGrid.useKEDA" $) "true") (eq .Values.autoscaling.scalingType "job") | ternary $nodeMaxSessions 0 | quote }} - name: SE_NODE_PORT value: {{ .node.port | quote }} - name: DISPLAY_CONTAINER_NAME diff --git a/charts/selenium-grid/templates/node-configmap.yaml b/charts/selenium-grid/templates/node-configmap.yaml index d5a1953548..abd813d92b 100644 --- a/charts/selenium-grid/templates/node-configmap.yaml +++ b/charts/selenium-grid/templates/node-configmap.yaml @@ -23,7 +23,6 @@ data: {{- end }} NODE_CONFIG_DIRECTORY: '{{ $.Values.nodeConfigMap.extraScriptsDirectory }}' SE_SUB_PATH: '{{ template "seleniumGrid.url.subPath" $ }}' - SE_DRAIN_AFTER_SESSION_COUNT: '{{- and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "1" "0" -}}' {{- if $.Values.videoRecorder.enabled }} SE_VIDEO_CONTAINER_NAME: {{ $.Values.videoRecorder.name | quote }} {{- end }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 92cb79726c..20088478e1 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -13,7 +13,7 @@ global: # -- Image tag for browser's video recorder videoImageTag: ffmpeg-7.0.2-20240907 # -- kubectl image is used to execute kubectl commands in utility jobs - kubectlImage: jitesoft/kubectl:latest + kubectlImage: bitnami/kubectl:latest # -- Pull secret for all components, can be overridden individually imagePullSecret: "" # -- Log level for all components. Possible values describe here: https://www.selenium.dev/documentation/grid/configuration/cli_options/#logging @@ -48,6 +48,8 @@ global: # topologyKey: kubernetes.io/hostname # whenUnsatisfiable: DoNotSchedule # Note: If not define labelSelector, it will be added automatically based on "app" label in each component + # -- Specify number of max sessions per node. Can be overridden by individual component (this is also set to scaler trigger parameter `nodeMaxSessions` if `autoscaling` is enabled) + nodeMaxSessions: 1 tls: # -- Name of external secret containing the TLS certificate and key @@ -1006,6 +1008,8 @@ chromeNode: # persistentVolumeClaim: # claimName: my-pv-claim + # -- Override the number of max sessions per node + nodeMaxSessions: # -- Override the scaled options for chrome nodes scaledOptions: # -- Override the scaledJobOptions for chrome nodes @@ -1185,6 +1189,8 @@ firefoxNode: # persistentVolumeClaim: # claimName: my-pv-claim + # -- Override the number of max sessions per node + nodeMaxSessions: # -- Override the scaled options for firefox nodes scaledOptions: # -- Override the scaledJobOptions for firefox nodes @@ -1363,6 +1369,8 @@ edgeNode: # persistentVolumeClaim: # claimName: my-pv-claim + # -- Override the number of max sessions per node + nodeMaxSessions: # -- Override the scaled options for edge nodes scaledOptions: # -- Override the scaledJobOptions for edge nodes diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index d33a50db3c..b4aefb5ea0 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -49,6 +49,9 @@ else fi EXTERNAL_TLS_SECRET_NAME=${EXTERNAL_TLS_SECRET_NAME:-"external-tls-secret"} SELENIUM_ENABLE_MANAGED_DOWNLOADS=${SELENIUM_ENABLE_MANAGED_DOWNLOADS:-"true"} +MAX_SESSIONS_CHROME=${MAX_SESSIONS_CHROME:-"1"} +MAX_SESSIONS_FIREFOX=${MAX_SESSIONS_FIREFOX:-"1"} +MAX_SESSIONS_EDGE=${MAX_SESSIONS_EDGE:-"1"} cleanup() { # Get the list of pods @@ -126,6 +129,9 @@ HELM_COMMAND_SET_IMAGES=" \ --set global.seleniumGrid.httpLogs=${CHART_ENABLE_TRACING} \ --set isolateComponents=${CHART_FULL_DISTRIBUTED_MODE} \ --set global.seleniumGrid.logLevel=${LOG_LEVEL} \ +--set chromeNode.nodeMaxSessions=${MAX_SESSIONS_CHROME} \ +--set firefoxNode.nodeMaxSessions=${MAX_SESSIONS_FIREFOX} \ +--set edgeNode.nodeMaxSessions=${MAX_SESSIONS_EDGE} \ " if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ] && [ "${TEST_EXISTING_KEDA}" = "true" ]; then diff --git a/tests/charts/templates/render/dummy.yaml b/tests/charts/templates/render/dummy.yaml index 8aa6a3ee19..c481dc4f05 100644 --- a/tests/charts/templates/render/dummy.yaml +++ b/tests/charts/templates/render/dummy.yaml @@ -17,6 +17,7 @@ global: - maxSkew: 4 topologyKey: kubernetes.io/hostname whenUnsatisfiable: DoNotSchedule + nodeMaxSessions: 3 autoscaling: enableWithExistingKEDA: true @@ -95,11 +96,13 @@ components: serviceType: NodePort chromeNode: + nodeMaxSessions: 2 annotations: "restartOnUpdate": "true" terminationGracePeriodSeconds: 7200 firefoxNode: + nodeMaxSessions: 1 annotations: "restartOnUpdate": "true" terminationGracePeriodSeconds: 720 diff --git a/tests/charts/templates/render/dummy_solution.yaml b/tests/charts/templates/render/dummy_solution.yaml index 44443f8e7c..b42936cd9c 100644 --- a/tests/charts/templates/render/dummy_solution.yaml +++ b/tests/charts/templates/render/dummy_solution.yaml @@ -17,6 +17,7 @@ global: - maxSkew: 4 topologyKey: kubernetes.io/hostname whenUnsatisfiable: DoNotSchedule + nodeMaxSessions: 3 selenium-grid: autoscaling: @@ -86,10 +87,12 @@ selenium-grid: serviceType: NodePort chromeNode: + nodeMaxSessions: 2 affinity: *affinity terminationGracePeriodSeconds: 7200 firefoxNode: + nodeMaxSessions: 1 affinity: *affinity terminationGracePeriodSeconds: 720 diff --git a/tests/charts/templates/test.py b/tests/charts/templates/test.py index cd4d7a312e..b3b2fc82b5 100644 --- a/tests/charts/templates/test.py +++ b/tests/charts/templates/test.py @@ -329,6 +329,34 @@ def test_router_envFrom_secretRef_name_use_external_secret_when_basicAuth_nameOv is_present = True self.assertTrue(is_present, "ENV variable from secretRef name is not set to external secret") + def test_scaler_triggers_authenticationRef_name_is_added(self): + resources_name = ['{0}selenium-chrome-node'.format(RELEASE_NAME), + '{0}selenium-edge-node'.format(RELEASE_NAME), + '{0}selenium-firefox-node'.format(RELEASE_NAME),] + is_present = False + for doc in LIST_OF_DOCUMENTS: + if doc['metadata']['name'] in resources_name and doc['kind'] == 'ScaledObject': + logger.info(f"Assert authenticationRef name is added to scaler triggers") + name = doc['spec']['triggers'][0]['authenticationRef']['name'] + self.assertTrue(name, '{0}selenium-scaler-trigger-auth'.format(RELEASE_NAME)) + + def test_scaler_triggers_parameter_nodeMaxSessions_global_and_individual_value(self): + resources_name = {'{0}selenium-chrome-node'.format(RELEASE_NAME): 2, + '{0}selenium-edge-node'.format(RELEASE_NAME): 3, + '{0}selenium-firefox-node'.format(RELEASE_NAME): 1,} + count = 0 + for resource_name in resources_name.keys(): + for doc in LIST_OF_DOCUMENTS: + if doc['metadata']['name'] == resource_name and doc['kind'] == 'ScaledObject': + logger.info(f"Assert nodeMaxSessions parameter is set in scaler triggers") + self.assertTrue(doc['spec']['triggers'][0]['metadata']['nodeMaxSessions'] == str(resources_name[doc['metadata']['name']])) + if doc['metadata']['name'] == resource_name and doc['kind'] == 'Deployment': + for env in doc['spec']['template']['spec']['containers'][0]['env']: + if env['name'] == 'SE_NODE_MAX_SESSIONS': + self.assertTrue(env['value'] == str(resources_name[doc['metadata']['name']]), "Value is not matched") + count += 1 + self.assertEqual(count, len(resources_name.keys()), "Expected {0} resources but found {1}".format(len(resources_name.keys()), count)) + if __name__ == '__main__': failed = False try: