K8s: Fix deployment config error in video-manager (#2831)

VietND96 · web-flow · commit 74149de8ec09 · 2025-05-10T02:45:58.000+07:00
Signed-off-by: Viet Nguyen Duc &lt;nguyenducviet4496@gmail.com&gt;
diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml
@@ -65,7 +65,7 @@ jobs:
             docker-version: '26.1.4'
             python-version: '3.9'
             test-upgrade: true
-            service-mesh: true
+            service-mesh: false
             os: ubuntu-22.04
             check-records-output: true
             test-strategy: job
@@ -115,7 +115,7 @@ jobs:
             docker-version: '26.1.4'
             python-version: '3.10'
             test-upgrade: true
-            service-mesh: true
+            service-mesh: false
             os: ubuntu-22.04
             check-records-output: true
             test-strategy: playwright_connect_grid
@@ -126,7 +126,7 @@ jobs:
             python-version: '3.10'
             test-upgrade: true
             service-mesh: true
-            os: blacksmith-8vcpu-ubuntu-2204
+            os: ubuntu-22.04
             check-records-output: false
             test-strategy: job_relay
     env:
diff --git a/Makefile b/Makefile
@@ -5,7 +5,7 @@ BASE_RELEASE := $(or $(BASE_RELEASE),$(BASE_RELEASE),selenium-4.32.0)
 BASE_VERSION := $(or $(BASE_VERSION),$(BASE_VERSION),4.32.0)
 BINDING_VERSION := $(or $(BINDING_VERSION),$(BINDING_VERSION),4.32.0)
 BASE_RELEASE_NIGHTLY := $(or $(BASE_RELEASE_NIGHTLY),$(BASE_RELEASE_NIGHTLY),nightly)
-BASE_VERSION_NIGHTLY := $(or $(BASE_VERSION_NIGHTLY),$(BASE_VERSIO3_NIGHTLY),4.33.0-SNAPSHOT)
+BASE_VERSION_NIGHTLY := $(or $(BASE_VERSION_NIGHTLY),$(BASE_VERSION_NIGHTLY),4.33.0-SNAPSHOT)
 VERSION := $(or $(VERSION),$(VERSION),4.32.0)
 MVN_SELENIUM_VERSION := $(or $(MVN_SELENIUM_VERSION),$(MVN_SELENIUM_VERSION),4.32.0)
 TAG_VERSION := $(VERSION)-$(BUILD_DATE)
diff --git a/README.md b/README.md
@@ -707,6 +707,43 @@ When using in Dynamic Grid, those variables should be combined with the prefix `
 | `SE_UPLOAD_CONFIG_FILE_NAME`  | `upload.conf`                                | Config file for remote host instead of set via env variable prefix SE_RCLONE_*            |
 | `SE_UPLOAD_CONFIG_DIRECTORY`  | `/opt/bin`                                   | Directory of config file (change it when conf file in another directory is mounted)       |
 
+## Video recordings manager
+
+We utilize [File Browser](https://filebrowser.org/) as a video manager. It is a web-based file manager that allows you to manage files and folders in the storage.
+
+The File Browser container dir `/srv` should be mounted to the same storage as video recordings stored. For example a compose file:
+
+```yaml
+services:
+  chrome:
+    deploy:
+      mode: replicated
+      replicas: 3
+    image: selenium/node-chrome:4.32.0-20250505
+    platform: linux/amd64
+    shm_size: 2gb
+    depends_on:
+      - selenium-hub
+    volumes:
+      - /tmp/videos:/videos
+    environment:
+      - SE_EVENT_BUS_HOST=selenium-hub
+      - SE_RECORD_VIDEO=true
+      - SE_VIDEO_FILE_NAME=auto
+      - SE_NODE_GRID_URL=http://selenium-hub:4444
+
+  file_browser:
+      image: filebrowser/filebrowser:latest
+      container_name: file_browser
+      restart: always
+      ports:
+          - "8081:80"
+      volumes:
+          - /tmp/videos:/srv
+      environment:
+          - FB_NOAUTH=true
+```
+
 ___
 
 ## Dynamic Grid
@@ -1010,6 +1047,8 @@ Get started to deploy Selenium Grid on Kubernetes, you can refer to YAML files i
 To simplify the deployment process, hide the complexity of Kubernetes objects, and provide a more straightforward way to deploy Selenium Grid on Kubernetes, we offer a Helm chart to deploy Selenium Grid to Kubernetes.
 Read more details at the Helm [chart README](./charts/selenium-grid/README.md) and [chart CONFIGURATION](./charts/selenium-grid/CONFIGURATION.md).
 
+- Get started to hands-on with Selenium Grid on Kubernetes. See local env setup with [Docker Desktop](./tests/charts/refValues/README.md).
+
 ___
 
 ## Configuring the containers
diff --git a/charts/selenium-grid/.helmignore b/charts/selenium-grid/.helmignore
@@ -23,3 +23,5 @@
 .vscode/
 ci/
 certs/add-*-helper.sh
+CHANGELOG.md
+TESTING.md
diff --git a/charts/selenium-grid/CONFIGURATION.md b/charts/selenium-grid/CONFIGURATION.md
@@ -689,7 +689,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | videoManager.imageTag | string | `"latest"` | File browser image tag (this overwrites global.seleniumGrid.imageTag parameter) |
 | videoManager.imagePullPolicy | string | `"IfNotPresent"` | Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) |
 | videoManager.imagePullSecret | string | `""` | Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) |
-| videoManager.config.baseurl | string | `"recordings"` | Base URL use to access the file browser (e.g. http://public.ip/recordings) |
+| videoManager.config.baseurl | string | `"/recordings"` | Base URL use to access the file browser (in case expose both Grid and file browser via ingress, e.g. Grid at http://public.ip/selenium and FB at http://public.ip/recordings) |
 | videoManager.config.username | string | `""` | Username for the first user when using quick config (default "admin") |
 | videoManager.config.password | string | `""` | Hashed password (bcrypt) for the first user when using quick config (default "admin") |
 | videoManager.config.noauth | bool | `true` | Use the noauth auther when using quick setup |
diff --git a/charts/selenium-grid/README.md b/charts/selenium-grid/README.md
@@ -36,6 +36,7 @@ This chart enables the creation of a Selenium Grid Server in Kubernetes.
     * [Configuration of video recorder and video uploader](#configuration-of-video-recorder-and-video-uploader)
       * [Video recorder](#video-recorder)
       * [Video uploader](#video-uploader)
+      * [Video manager](#video-manager)
     * [Configuration of Secure Communication](#configuration-of-secure-communication)
       * [Create TLS Secret](#create-tls-secret)
       * [Secure Connection to Selenium Grid components](#secure-connection-to-selenium-grid-components)
@@ -793,6 +794,45 @@ videoRecorder:
         imageTag: latest
 ```
 
+#### Video manager
+
+We utilize [File Browser](https://filebrowser.org/) as a video manager. It is a web-based file manager that allows you to manage files and folders in the storage. The video manager is disabled by default. To enable it, you need to set config key `videoManager.enabled` to `true`.
+
+The service can be exposed via NodePort or Ingress (if enabled global ingress). By default, there is a baseurl is `/recordings`, if enabled svc type NodePort: `http://<node-ip>:30080/recordings`, or ingress `http://<ingress-hostname>/recordings`. You also can change the baseurl to another value via config key `videoManager.config.baseurl`
+
+
+The File Browser container dir `/srv` should be mounted to the same storage as video recordings stored. The storage here is persistent volume claim (PVC) that is created by you or dynamically provisioned by the storage class. Configure recorder and manager to use the same PVC. For example
+
+```yaml
+videoRecorder:
+  enabled: true
+  extraVolumeMounts:
+    - name: videos
+      mountPath: /videos
+      subPath: videos
+  extraVolumes:
+    - name: videos
+      persistentVolumeClaim:
+        claimName: local-pv-storage
+
+videoManager:
+  enabled: true
+  extraVolumeMounts:
+    - name: videos
+      mountPath: /srv
+      subPath: videos
+  extraVolumes:
+    - name: videos
+      persistentVolumeClaim:
+        claimName: local-pv-storage
+```
+
+When configuration is done, via File Browser you can centralize all the recordings in one place. You can manage the recordings, delete them, or download them.
+
+![img_1.png](./images/video-manager_1.png)
+
+![img_2.png](./images/video-manager_2.png)
+
 ### Configuration of Secure Communication
 
 Selenium Grid supports secure communication between components. Refer to the [instructions](https://github.com/SeleniumHQ/selenium/blob/trunk/java/src/org/openqa/selenium/grid/commands/security.txt) and [options](https://www.selenium.dev/documentation/grid/configuration/cli_options/#server) are able to configure the secure communication. Below is the details on how to enable secure communication in Selenium Grid chart.
diff --git a/charts/selenium-grid/images/video-manager_1.png b/charts/selenium-grid/images/video-manager_1.png
diff --git a/charts/selenium-grid/images/video-manager_2.png b/charts/selenium-grid/images/video-manager_2.png
diff --git a/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml b/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml
@@ -1,6 +1,6 @@
 {{- range $i, $newNode := .Values.crossBrowsers.chromeNode }}
 {{- $nodeConfig := merge $newNode $.Values.chromeNode -}}
-{{- if and $nodeConfig.enabled (include "seleniumGrid.useKEDA" $) (eq $.Values.autoscaling.scalingType "job") }}
+{{- if and $nodeConfig.enabled (eq (include "seleniumGrid.useKEDA" $) "true") (eq $.Values.autoscaling.scalingType "job") }}
 apiVersion: keda.sh/v1alpha1
 kind: ScaledJob
 metadata:
diff --git a/charts/selenium-grid/templates/relay-node-scaledjobs.yaml b/charts/selenium-grid/templates/relay-node-scaledjobs.yaml
@@ -1,6 +1,6 @@
 {{- range $i, $newNode := .Values.crossBrowsers.relayNode }}
 {{- $nodeConfig := merge $newNode $.Values.relayNode -}}
-{{- if and $nodeConfig.enabled (include "seleniumGrid.useKEDA" $) (eq $.Values.autoscaling.scalingType "job") }}
+{{- if and $nodeConfig.enabled (eq (include "seleniumGrid.useKEDA" $) "true") (eq $.Values.autoscaling.scalingType "job") }}
 apiVersion: keda.sh/v1alpha1
 kind: ScaledJob
 metadata:
diff --git a/charts/selenium-grid/templates/video-manager/file-browser-ingress.yaml b/charts/selenium-grid/templates/video-manager/file-browser-ingress.yaml
@@ -53,7 +53,7 @@ spec:
                 name: {{ include "seleniumGrid.videoManager.fullname" $ | quote }}
                 port:
                   number: 80
-            path: {{ printf "/%s" $.Values.videoManager.config.baseurl | quote }}
+            path: {{ $.Values.videoManager.config.baseurl | quote }}
             pathType: Prefix
         {{- end }}
 {{- end }}
diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml
@@ -2056,8 +2056,8 @@ videoManager:
   # -- Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/)
   imagePullSecret: ""
   config:
-    # -- Base URL use to access the file browser (e.g. http://public.ip/recordings)
-    baseurl: "recordings"
+    # -- Base URL use to access the file browser (in case expose both Grid and file browser via ingress, e.g. Grid at http://public.ip/selenium and FB at http://public.ip/recordings)
+    baseurl: "/recordings"
     # -- Username for the first user when using quick config (default "admin")
     username: ""
     # -- Hashed password (bcrypt) for the first user when using quick config (default "admin")
diff --git a/tests/CDPTests/bootstrap.sh b/tests/CDPTests/bootstrap.sh
@@ -10,7 +10,7 @@ BROWSER=${1:-"chrome"}
 SELENIUM_REMOTE_URL="${SELENIUM_GRID_PROTOCOL}://${SELENIUM_GRID_HOST}:${SELENIUM_GRID_PORT}"
 echo "SELENIUM_REMOTE_URL=${SELENIUM_REMOTE_URL}" > .env
 
-if [ -n ${SELENIUM_GRID_USERNAME} ] && [ -n ${SELENIUM_GRID_PASSWORD} ]; then
+if [ -n "${SELENIUM_GRID_USERNAME}" ] && [ -n "${SELENIUM_GRID_PASSWORD}" ]; then
   BASIC_AUTH="$(echo -en "${SELENIUM_GRID_USERNAME}:${SELENIUM_GRID_PASSWORD}" | base64 -w0)"
   echo "SELENIUM_REMOTE_HEADERS={\"Authorization\": \"Basic ${BASIC_AUTH}\"}" >> .env
   BASIC_AUTH="Authorization: Basic ${BASIC_AUTH}"
@@ -21,7 +21,14 @@ echo "NODE_EXTRA_CA_CERTS=${CHART_CERT_PATH}" >> .env
 
 cat .env
 
+start_time=$(date +%s)
 until [ "$(curl --noproxy "*" -sk -H "${BASIC_AUTH}" -o /dev/null -w "%{http_code}" "${SELENIUM_REMOTE_URL}/status")" = "200" ]; do
+  current_time=$(date +%s)
+  elapsed_time=$((current_time - start_time))
+  if [ $elapsed_time -ge 400 ]; then
+    echo "Timeout reached: Grid is not ready after 5 minutes."
+    exit 1
+  fi
   echo "Waiting for Grid to be ready..."
   sleep 1
 done
diff --git a/tests/charts/make/chart_cluster_setup.sh b/tests/charts/make/chart_cluster_setup.sh
@@ -42,16 +42,18 @@ if [ "${CLUSTER}" = "kind" ]; then
   echo "Start Kind cluster"
   kind create cluster --image kindest/node:${KUBERNETES_VERSION} --wait ${WAIT_TIMEOUT} --name ${CLUSTER_NAME} --config tests/charts/config/kind-cluster.yaml
 elif [ "${CLUSTER}" = "minikube" ]; then
+  HOST_IP="$(hostname -I | cut -d' ' -f1)"
   echo "Start Minikube cluster"
   sudo chmod 777 /tmp
   export CHANGE_MINIKUBE_NONE_USER=true
-  sudo -SE minikube start --vm-driver=none \
+  NO_PROXY="$NO_PROXY,$HOST_IP" sudo -SE minikube start --vm-driver=none \
   --kubernetes-version=${KUBERNETES_VERSION} --network-plugin=cni --cni=${CNI} --container-runtime=${CONTAINER_RUNTIME} --wait=all
   sudo chown -R $USER $HOME/.kube $HOME/.minikube
   if [ "${SERVICE_MESH}" = "true" ]; then
     minikube addons enable istio-provisioner
     minikube addons enable istio
   fi
+  kubectl set env daemonset/calico-node -n kube-system IP_AUTODETECTION_METHOD="can-reach=${HOST_IP}"
 fi
 
 if [ "${CLUSTER}" = "kind" ]; then
diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh
@@ -106,7 +106,10 @@ on_failure() {
     if [ ${RENDER_HELM_TEMPLATE_ONLY} = "true" ]; then
       exit $exit_status
     fi
-    kubectl get pods -A
+    kubectl get pods -n "${SELENIUM_NAMESPACE}" -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | while read pod; do
+      echo "Logs for pod $pod"
+      kubectl logs -n "${SELENIUM_NAMESPACE}" "$pod" --all-containers --tail=10000
+    done
     echo "Get all resources in all namespaces"
     kubectl get all -A >> tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt
     echo "Describe all resources in the ${SELENIUM_NAMESPACE} namespace for debugging purposes"
diff --git a/tests/charts/refValues/README.md b/tests/charts/refValues/README.md
@@ -0,0 +1,95 @@
+<!-- TOC -->
+* [Introduction](#introduction)
+* [Docker Desktop](#docker-desktop)
+  * [Install Required Software](#install-required-software)
+  * [Configure Kubernetes cluster in Docker Desktop](#configure-kubernetes-cluster-in-docker-desktop)
+  * [Deploy Selenium Grid solution using Helm chart](#deploy-selenium-grid-solution-using-helm-chart)
+      * [Deploy PVC for video recording and video manager storage.](#deploy-pvc-for-video-recording-and-video-manager-storage)
+      * [Add Docker Selenium Helm chart repository](#add-docker-selenium-helm-chart-repository)
+      * [Install latest chart with reference values.](#install-latest-chart-with-reference-values)
+      * [Verify Grid installation](#verify-grid-installation)
+      * [Browser Nodes in autoscaling from zero mode.](#browser-nodes-in-autoscaling-from-zero-mode)
+      * [Run a test in Grid](#run-a-test-in-grid)
+<!-- TOC -->
+
+# Introduction
+
+This directory contains the reference values for the Helm charts in the `charts` directory. The reference values are used to test the Helm charts and ensure that they are working correctly. The reference values are stored in YAML files, which can be used to generate the expected output of the Helm charts.
+
+# Docker Desktop
+
+## Install Required Software
+
+_Note: This guide is for macOS. For other operating systems, please refer to particular tool documentation._
+```sh
+brew install docker kubectl helm
+```
+
+Verify the installation
+```sh
+java -version
+docker --version
+kubectl version --client
+helm version
+```
+
+## Configure Kubernetes cluster in Docker Desktop
+
+Most users are able to installed Docker Desktop and start Kubernetes cluster in settings as below.
+
+![img.png](images/docker-desktop-1.png)
+
+If you are having a local machine with good resources, configure Docker Desktop to use more resources. Then you can install orchestration containers with autoscaling capability.
+
+![img_1.png](images/docker-desktop-2.png)
+
+Open a terminal, ensure `kubectl` is able to connect to the cluster
+
+```sh
+kubectl cluster-info
+```
+
+## Deploy Selenium Grid solution using Helm chart
+
+#### Deploy PVC for video recording and video manager storage.
+
+Checkout file [local-pvc-docker-desktop.yaml](local-pvc-docker-desktop.yaml)
+
+```sh
+kubectl apply -f local-pvc-docker-desktop.yaml
+```
+
+#### Add Docker Selenium Helm chart repository
+
+```sh
+helm repo add docker-selenium https://www.selenium.dev/docker-selenium
+helm repo update
+```
+
+#### Install latest chart with reference values.
+
+Checkout file [simplex-docker-desktop.yaml](simplex-docker-desktop.yaml)
+
+```sh
+helm upgrade -i selenium docker-selenium/selenium-grid \
+  --namespace default \
+  -f simplex-docker-desktop.yaml
+```
+
+#### Verify Grid installation
+
+- Grid components in cluster: `kubectl get pod -n default`
+- Grid UI: [http://localhost/selenium](http://localhost/selenium)
+- Recordings manager: [http://localhost/recordings](http://localhost/recordings)
+
+#### Browser Nodes in autoscaling from zero mode.
+
+There is no Node visible on Grid UI. The browser nodes are created on demand when a test is started. The browser nodes are automatically removed when the test is finished.
+
+#### Run a test in Grid
+
+Refer to [get_started.py](../../get_started.py) for a simple test using Python and Selenium. The test will run remotely on the Grid.
+
+- Check the Grid UI to see the test running.
+- Check the Recordings manager to see the video recording of the test.
+- Check if the test passed or failed.
diff --git a/tests/charts/refValues/images/docker-desktop-1.png b/tests/charts/refValues/images/docker-desktop-1.png
diff --git a/tests/charts/refValues/images/docker-desktop-2.png b/tests/charts/refValues/images/docker-desktop-2.png
diff --git a/tests/charts/refValues/local-pvc-docker-desktop.yaml b/tests/charts/refValues/local-pvc-docker-desktop.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: local-pv-storage
+  namespace: default
+spec:
+  accessModes:
+    - "ReadWriteOnce"
+  resources:
+    requests:
+      storage: "2Gi"
+  storageClassName: hostpath
+---
diff --git a/tests/charts/refValues/simplex-docker-desktop.yaml b/tests/charts/refValues/simplex-docker-desktop.yaml
