K8s: Selenium Grid in case multiple scaler triggers are activate (#2515)

VietND96 · web-flow · commit 6f90ecbacc65 · 2024-12-24T13:53:26.000+07:00
Signed-off-by: Viet Nguyen Duc &lt;nguyenducviet4496@gmail.com&gt;
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -25,6 +25,10 @@ on:
     paths-ignore:
       - '**.md'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref == github.ref_protected && github.run_id || github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 permissions: write-all
 
 env:
@@ -44,6 +48,7 @@ jobs:
   helm-chart-test:
     name: Test Selenium Grid on Kubernetes
     uses: ./.github/workflows/helm-chart-test.yml
+    secrets: inherit
     with:
       release: ${{ inputs.release == 'true' }}
 
diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml
@@ -2,6 +2,11 @@ name: Test Helm Charts
 
 on:
   workflow_call:
+    secrets:
+      DOCKER_USERNAME:
+        required: false
+      DOCKER_PASSWORD:
+        required: false
     inputs:
       release:
         description: 'Test a new release process'
@@ -142,7 +147,6 @@ jobs:
       - name: Sets build date
         run: |
           echo "BUILD_DATE=$(date '+%Y%m%d')" >> $GITHUB_ENV
-          echo "IMAGE_REGISTRY=artifactory/selenium" >> $GITHUB_ENV
           echo "AUTHORS=${AUTHORS}" >> $GITHUB_ENV
         env:
           AUTHORS: ${{ vars.AUTHORS || 'SeleniumHQ' }}
@@ -158,6 +162,11 @@ jobs:
           max_attempts: 3
           retry_wait_seconds: 60
           command: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build
+      - name: Login Docker Hub
+        run: docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
+        env:
+          DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}}
+          DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
       - name: Setup Kubernetes cluster
         uses: nick-invision/retry@master
         with:
diff --git a/.github/workflows/k8s-scaling-test.yml b/.github/workflows/k8s-scaling-test.yml
@@ -2,6 +2,11 @@ name: Test Grid Autoscaling in Kubernetes
 
 on:
   workflow_call:
+    secrets:
+      DOCKER_USERNAME:
+        required: false
+      DOCKER_PASSWORD:
+        required: false
     inputs:
       release:
         description: 'Test a new release process'
@@ -36,6 +41,10 @@ on:
         type: string
         default: '20'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref == github.ref_protected && github.run_id || github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 permissions:
   contents: write
   pull-requests: write
@@ -135,10 +144,14 @@ jobs:
       - name: Sets build date
         run: |
           echo "BUILD_DATE=$(date '+%Y%m%d')" >> $GITHUB_ENV
-          echo "IMAGE_REGISTRY=artifactory/selenium" >> $GITHUB_ENV
           echo "AUTHORS=${AUTHORS}" >> $GITHUB_ENV
         env:
           AUTHORS: ${{ vars.AUTHORS || 'SeleniumHQ' }}
+      - name: Login Docker Hub
+        run: docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
+        env:
+          DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}}
+          DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
       - name: Setup Kubernetes cluster
         uses: nick-invision/retry@master
         with:
diff --git a/.keda/README.md b/.keda/README.md
@@ -49,12 +49,16 @@ You can involve to review and discuss the pull requests to help us early detect
 
 [kedacore/keda](https://github.com/kedacore/keda)
 
+- https://github.com/kedacore/keda/pull/6437
+
 - https://github.com/kedacore/keda/pull/6368
 
 - ~~https://github.com/kedacore/keda/pull/6169 (merged, v2.16.0)~~
 
 [kedacore/keda-docs](https://github.com/kedacore/keda-docs)
 
+- https://github.com/kedacore/keda-docs/pull/1515
+
 - ~~https://github.com/kedacore/keda-docs/pull/1468 (merged, v2.16.0)~~
 
 # Test results of the patch scaler
diff --git a/Makefile b/Makefile
@@ -31,7 +31,7 @@ SBOM_OUTPUT := $(or $(SBOM_OUTPUT),$(SBOM_OUTPUT),package_versions.txt)
 KEDA_TAG_PREV_VERSION := $(or $(KEDA_TAG_PREV_VERSION),$(KEDA_TAG_PREV_VERSION),2.16.0-selenium-grid)
 KEDA_TAG_VERSION := $(or $(KEDA_TAG_VERSION),$(KEDA_TAG_VERSION),2.16.0-selenium-grid)
 KEDA_BASED_NAME := $(or $(KEDA_BASED_NAME),$(KEDA_BASED_NAME),ndviet)
-KEDA_BASED_TAG := $(or $(KEDA_BASED_TAG),$(KEDA_BASED_TAG),2.16.0-selenium-grid-20241201)
+KEDA_BASED_TAG := $(or $(KEDA_BASED_TAG),$(KEDA_BASED_TAG),2.16.0-selenium-grid-20241224)
 
 all: hub \
 	distributor \
@@ -930,15 +930,15 @@ chart_test_autoscaling_disabled:
 	./tests/charts/make/chart_test.sh NoAutoscaling
 
 chart_test_autoscaling_deployment_https:
-	PLATFORMS=$(PLATFORMS) CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_BASIC_AUTH=true TEST_EXTERNAL_DATASTORE=postgresql \
+	PLATFORMS=$(PLATFORMS) CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_BASIC_AUTH=true TEST_EXTERNAL_DATASTORE=postgresql TEST_MULTIPLE_VERSIONS=true AUTOSCALING_COOLDOWN_PERIOD=60 SELENIUM_GRID_MONITORING=false \
 	SECURE_INGRESS_ONLY_DEFAULT=true INGRESS_DISABLE_USE_HTTP2=true SELENIUM_GRID_PROTOCOL=https CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_PORT=443 \
 	SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=0 MAX_SESSIONS_FIREFOX=1 MAX_SESSIONS_EDGE=1 MAX_SESSIONS_CHROME=1 TEST_NAME_OVERRIDE=true \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \
 	TEMPLATE_OUTPUT_FILENAME="k8s_fullDistributed_basicAuth_secureIngress_defaultCerts_ingressHostName_disableHttp2_autoScaling_patchKEDA_scaledObject_subPath.yaml" \
 	./tests/charts/make/chart_test.sh DeploymentAutoscaling
 
 chart_test_autoscaling_deployment:
-	PLATFORMS=$(PLATFORMS) TEST_EXISTING_KEDA=true RELEASE_NAME=selenium CHART_ENABLE_TRACING=true TEST_PATCHED_KEDA=false \
+	PLATFORMS=$(PLATFORMS) TEST_EXISTING_KEDA=true RELEASE_NAME=selenium CHART_ENABLE_TRACING=true TEST_PATCHED_KEDA=true \
 	SECURE_CONNECTION_SERVER=true SECURE_USE_EXTERNAL_CERT=true SERVICE_TYPE_NODEPORT=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -I | cut -d' ' -f1) SELENIUM_GRID_PORT=31444 \
 	SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 SET_MAX_REPLICAS=3 TEST_DELAY_AFTER_TEST=2 SELENIUM_GRID_MONITORING=false  \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \
@@ -954,14 +954,21 @@ chart_test_autoscaling_job_https:
 	./tests/charts/make/chart_test.sh JobAutoscaling
 
 chart_test_autoscaling_job_hostname:
-	PLATFORMS=$(PLATFORMS) CHART_ENABLE_TRACING=true CHART_ENABLE_BASIC_AUTH=true BASIC_AUTH_EMBEDDED_URL=true TEST_PATCHED_KEDA=false \
+	PLATFORMS=$(PLATFORMS) CHART_ENABLE_TRACING=true CHART_ENABLE_BASIC_AUTH=true BASIC_AUTH_EMBEDDED_URL=true TEST_PATCHED_KEDA=true \
 	SECURE_INGRESS_ONLY_DEFAULT=true SECURE_USE_EXTERNAL_CERT=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -I | cut -d' ' -f1) SELENIUM_GRID_PORT=443 \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \
 	TEMPLATE_OUTPUT_FILENAME="k8s_enableTracing_basicAuth_secureIngress_externalCerts_ingressPublicIP_autoScaling_originKEDA_scaledJob_subPath.yaml" \
 	./tests/charts/make/chart_test.sh JobAutoscaling
 
+chart_test_autoscaling_job_multiple_versions_without_explicit:
+	TEST_MULTIPLE_VERSIONS=true TEST_MULTIPLE_VERSIONS_EXPLICIT=false make chart_test_autoscaling_job
+
+chart_test_autoscaling_job_without_multiple_versions:
+	TEST_MULTIPLE_VERSIONS=false make chart_test_autoscaling_job
+
 chart_test_autoscaling_job:
 	PLATFORMS=$(PLATFORMS) TEST_EXISTING_KEDA=true TEST_CHROMIUM=true RELEASE_NAME=selenium CHART_ENABLE_TRACING=true CHART_FULL_DISTRIBUTED_MODE=true SELENIUM_GRID_MONITORING=false \
+	CLEAR_POD_HISTORY=true TEST_MULTIPLE_VERSIONS=$(or $(TEST_MULTIPLE_VERSIONS), "true") TEST_MULTIPLE_VERSIONS_EXPLICIT=$(or $(TEST_MULTIPLE_VERSIONS_EXPLICIT), "true") \
 	SECURE_INGRESS_ONLY_CONFIG_INLINE=true SECURE_USE_EXTERNAL_CERT=true CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=selenium-grid.prod SUB_PATH=/ SELENIUM_GRID_PORT=443 \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \
 	TEMPLATE_OUTPUT_FILENAME="k8s_fullDistributed_secureIngress_externalCerts_ingressHostName_ingressTLSInline_autoScaling_scaledJob_existingKEDA_prefixSelenium_nodeChromium_enableTracing.yaml" \
diff --git a/charts/selenium-grid/CONFIGURATION.md b/charts/selenium-grid/CONFIGURATION.md
@@ -406,6 +406,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | chromeNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for chrome nodes |
 | chromeNode.hpa.browserName | string | `"chrome"` | browserName from the capability |
 | chromeNode.hpa.sessionBrowserName | string | `"chrome"` | sessionBrowserName if the browserName is different from the sessionBrowserName |
+| chromeNode.hpa.browserVersion | string | `""` | browserVersion from the capability |
 | chromeNode.hpa.platformName | string | `"linux"` | platformName from the capability |
 | chromeNode.hpa.unsafeSsl | string | `"{{ template \"seleniumGrid.graphqlURL.unsafeSsl\" . }}"` | Skip check SSL when connecting to the Graphql endpoint |
 | chromeNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option |
@@ -457,6 +458,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | firefoxNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for firefox nodes |
 | firefoxNode.hpa.browserName | string | `"firefox"` | browserName from the capability |
 | firefoxNode.hpa.sessionBrowserName | string | `"firefox"` | sessionBrowserName if the browserName is different from the sessionBrowserName |
+| firefoxNode.hpa.browserVersion | string | `""` | browserVersion from the capability |
 | firefoxNode.hpa.platformName | string | `"linux"` | platformName from the capability |
 | firefoxNode.hpa.unsafeSsl | string | `"{{ template \"seleniumGrid.graphqlURL.unsafeSsl\" . }}"` | Skip check SSL when connecting to the Graphql endpoint |
 | firefoxNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option |
@@ -508,6 +510,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | edgeNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for edge nodes |
 | edgeNode.hpa.browserName | string | `"MicrosoftEdge"` | browserName from the capability |
 | edgeNode.hpa.sessionBrowserName | string | `"msedge"` | sessionBrowserName if the browserName is different from the sessionBrowserName |
+| edgeNode.hpa.browserVersion | string | `""` | browserVersion from the capability |
 | edgeNode.hpa.platformName | string | `"linux"` | platformName from the capability |
 | edgeNode.hpa.unsafeSsl | string | `"{{ template \"seleniumGrid.graphqlURL.unsafeSsl\" . }}"` | Skip check SSL when connecting to the Graphql endpoint |
 | edgeNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option |
@@ -560,6 +563,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | relayNode.hpa.browserName | string | `"chrome"` | browserName from the capability |
 | relayNode.hpa.sessionBrowserName | string | `""` | sessionBrowserName if the browserName is different from the sessionBrowserName |
 | relayNode.hpa.platformName | string | `"Android"` | platformName from the capability |
+| relayNode.hpa.browserVersion | string | `""` | browserVersion from the capability |
 | relayNode.hpa.unsafeSsl | string | `"{{ template \"seleniumGrid.graphqlURL.unsafeSsl\" . }}"` | Skip check SSL when connecting to the Graphql endpoint |
 | relayNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option |
 | relayNode.sidecars | list | `[]` | It is used to add sidecars proxy in the same pod of the browser node. It means it will add a new container to the deployment itself. It should be set using the --set-json option |
diff --git a/charts/selenium-grid/cross-browsers-values.yaml b/charts/selenium-grid/cross-browsers-values.yaml
@@ -2,6 +2,12 @@ crossBrowsers:
   chromeNode:
     # Keep the first iteration with latest version of Chrome
     - nameOverride:
+      hpa:
+        browserVersion: ''
+    - nameOverride: '{{ $.Release.Name }}-node-chrome-131'
+      imageTag: '131.0'
+      hpa:
+        browserVersion: '131.0'
     - nameOverride: '{{ $.Release.Name }}-node-chrome-130'
       imageTag: '130.0'
       hpa:
@@ -17,6 +23,20 @@ crossBrowsers:
   firefoxNode:
     # Keep the first iteration with latest version of Firefox
     - nameOverride:
+      hpa:
+        browserVersion: ''
+    - nameOverride: '{{ $.Release.Name }}-node-firefox-133'
+      imageTag: '133.0'
+      hpa:
+        browserVersion: '133.0'
+    - nameOverride: '{{ $.Release.Name }}-node-firefox-132'
+      imageTag: '132.0'
+      hpa:
+        browserVersion: '132.0'
+    - nameOverride: '{{ $.Release.Name }}-node-firefox-131'
+      imageTag: '131.0'
+      hpa:
+        browserVersion: '131.0'
     - nameOverride: '{{ $.Release.Name }}-node-firefox-130'
       imageTag: '130.0'
       hpa:
@@ -32,6 +52,12 @@ crossBrowsers:
   edgeNode:
     # Keep the first iteration with latest version of Edge
     - nameOverride:
+      hpa:
+        browserVersion: ''
+    - nameOverride: '{{ $.Release.Name }}-node-edge-131'
+      imageTag: '131.0'
+      hpa:
+        browserVersion: '131.0'
     - nameOverride: '{{ $.Release.Name }}-node-edge-130'
       imageTag: '130.0'
       hpa:
diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl
@@ -343,6 +343,14 @@ template:
         {{- end }}
           - name: SE_DRAIN_AFTER_SESSION_COUNT
             value: {{ and (eq (include "seleniumGrid.useKEDA" $) "true") (eq .Values.autoscaling.scalingType "job") | ternary $nodeMaxSessions 0 | quote }}
+        {{- if and (eq (include "seleniumGrid.useKEDA" $) "true") }}
+          - name: SE_NODE_BROWSER_VERSION
+            value: {{ default "" .node.hpa.browserVersion | quote }}
+        {{- end }}
+        {{- if and (eq (include "seleniumGrid.useKEDA" $) "true") }}
+          - name: SE_NODE_PLATFORM_NAME
+            value: {{ default "Linux" .node.hpa.platformName | quote }}
+        {{- end }}
           - name: SE_NODE_CONTAINER_NAME
             valueFrom:
               fieldRef:
diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml
@@ -1117,6 +1117,8 @@ chromeNode:
     browserName: "chrome"
     # -- sessionBrowserName if the browserName is different from the sessionBrowserName
     sessionBrowserName: "chrome"
+    # -- browserVersion from the capability
+    browserVersion: ""
     # -- platformName from the capability
     platformName: "linux"
     # browserVersion: '91.0' # Optional. Only required when supporting multiple versions of browser in your Selenium Grid.
@@ -1300,6 +1302,8 @@ firefoxNode:
     browserName: "firefox"
     # -- sessionBrowserName if the browserName is different from the sessionBrowserName
     sessionBrowserName: "firefox"
+    # -- browserVersion from the capability
+    browserVersion: ""
     # -- platformName from the capability
     platformName: "linux"
     # -- Skip check SSL when connecting to the Graphql endpoint
@@ -1482,6 +1486,8 @@ edgeNode:
     browserName: "MicrosoftEdge"
     # -- sessionBrowserName if the browserName is different from the sessionBrowserName
     sessionBrowserName: "msedge"
+    # -- browserVersion from the capability
+    browserVersion: ""
     # -- platformName from the capability
     platformName: "linux"
     # -- Skip check SSL when connecting to the Graphql endpoint
@@ -1666,6 +1672,8 @@ relayNode:
     sessionBrowserName: ""
     # -- platformName from the capability
     platformName: "Android"
+    # -- browserVersion from the capability
+    browserVersion: ""
     # browserVersion: '91.0' # Optional. Only required when supporting multiple versions of browser in your Selenium Grid.
     # -- Skip check SSL when connecting to the Graphql endpoint
     unsafeSsl: '{{ template "seleniumGrid.graphqlURL.unsafeSsl" . }}'  # Optional
diff --git a/generate_chart_changelog.sh b/generate_chart_changelog.sh
@@ -7,6 +7,11 @@ TAG_PATTERN="selenium-grid"
 DEFAULT_TAG="trunk"
 SET_TAG=${1:-$(git rev-parse --abbrev-ref HEAD)}
 
+function get_keda_version() {
+  KEDA_TAG_VERSION=$(grep KEDA_TAG_VERSION Makefile | sed 's/.*,\([^)]*\))/\1/p' | head -n 1)
+  KEDA_IMAGE_TAG=$(find . \( -type d -name .git -prune \) -o -type f -wholename '*/selenium-grid/values.yaml' -print0 | xargs -0 cat | grep $KEDA_TAG_VERSION | head -n 1 | cut -d ':' -f 2 | tr -d '[:space:]' | tr -d '"')
+}
+
 # Get current chart app version
 CHART_APP_VERSION=$(find . \( -type d -name .git -prune \) -o -type f -wholename '*/selenium-grid/Chart.yaml' -print0 | xargs -0 cat | grep ^appVersion | cut -d ':' -f 2 | tr -d '[:space:]')
 
@@ -82,8 +87,12 @@ generate_changelog() {
     echo "" >>"$temp_file"
   fi
 
+  get_keda_version
   echo "### Experimental" >>"$temp_file"
   echo "- Selenium Grid Scaler implementation preview. [README](https://github.com/seleniumhq/docker-selenium/tree/trunk/.keda/README.md)" >>"$temp_file"
+  if [ -n "$KEDA_IMAGE_TAG" ]; then
+    echo "- Chart is tested autoscaling capabilities with KEDA image tag: $KEDA_IMAGE_TAG" >>"$temp_file"
+  fi
   echo "" >>"$temp_file"
 
   # Create chart_release_notes.md
diff --git a/tests/AutoscalingTests/common.py b/tests/AutoscalingTests/common.py
@@ -34,7 +34,9 @@ def get_pod_count():
     return len([line for line in result.stdout.splitlines() if "selenium-node-" in line and "Running" in line])
 
 def create_session(browser_name):
-    return webdriver.Remote(command_executor=CLIENT_CONFIG.remote_server_addr, options=BROWSER[browser_name], client_config=CLIENT_CONFIG)
+    options = BROWSER[browser_name]
+    options.set_capability("platformName", "Linux")
+    return webdriver.Remote(command_executor=CLIENT_CONFIG.remote_server_addr, options=options, client_config=CLIENT_CONFIG)
 
 def wait_for_count_matches(sessions, timeout=10, interval=5):
     elapsed = 0
diff --git a/tests/CDPTests/bootstrap.sh b/tests/CDPTests/bootstrap.sh
@@ -15,7 +15,7 @@ if [ -n ${SELENIUM_GRID_USERNAME} ] && [ -n ${SELENIUM_GRID_PASSWORD} ]; then
   echo "SELENIUM_REMOTE_HEADERS={\"Authorization\": \"Basic ${BASIC_AUTH}\"}" >> .env
 fi
 
-echo "SELENIUM_REMOTE_CAPABILITIES={\"browserName\": \"${BROWSER}\"}" >> .env
+echo "SELENIUM_REMOTE_CAPABILITIES={\"browserName\": \"${BROWSER}\", \"platformName\": \"Linux\"}" >> .env
 echo "NODE_EXTRA_CA_CERTS=${CHART_CERT_PATH}" >> .env
 
 cat .env
diff --git a/tests/SeleniumTests/__init__.py b/tests/SeleniumTests/__init__.py
diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh
