Skip to content

Commit d6b3ac4

Browse files
committed
Add test for autoscaling count
[skip ci] Signed-off-by: Viet Nguyen Duc <[email protected]>
1 parent 31fba3c commit d6b3ac4

File tree

11 files changed

+284
-6
lines changed

11 files changed

+284
-6
lines changed

Makefile

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ SBOM_OUTPUT := $(or $(SBOM_OUTPUT),$(SBOM_OUTPUT),package_versions.txt)
2929
KEDA_TAG_PREV_VERSION := $(or $(KEDA_TAG_PREV_VERSION),$(KEDA_TAG_PREV_VERSION),2.16.0-selenium-grid)
3030
KEDA_TAG_VERSION := $(or $(KEDA_TAG_VERSION),$(KEDA_TAG_VERSION),2.16.0-selenium-grid)
3131
KEDA_BASED_NAME := $(or $(KEDA_BASED_NAME),$(KEDA_BASED_NAME),ndviet)
32-
KEDA_BASED_TAG := $(or $(KEDA_BASED_TAG),$(KEDA_BASED_TAG),2.16.0-selenium-grid-20241127)
32+
KEDA_BASED_TAG := $(or $(KEDA_BASED_TAG),$(KEDA_BASED_TAG),2.16.0-selenium-grid-20241128)
3333

3434
all: hub \
3535
distributor \
@@ -961,6 +961,40 @@ chart_test_autoscaling_playwright_connect_grid:
961961
TEMPLATE_OUTPUT_FILENAME="k8s_playwright_connect_grid_basicAuth_secureIngress_ingressPublicIP_autoScaling_patchKEDA.yaml" \
962962
./tests/charts/make/chart_test.sh JobAutoscaling
963963

964+
chart_test_autoscaling_job_count_chaos:
965+
MATRIX_TESTS=AutoScalingTestsScaleChaos \
966+
make chart_test_autoscaling_job_count
967+
968+
chart_test_autoscaling_job_count_max_sessions:
969+
MAX_SESSIONS_FIREFOX=3 MAX_SESSIONS_EDGE=2 MAX_SESSIONS_CHROME=2 \
970+
make chart_test_autoscaling_job_count
971+
972+
chart_test_autoscaling_job_count_strategy_accurate:
973+
MATRIX_TESTS=AutoScalingTestsScaleChaos SCALING_STRATEGY=accurate \
974+
make chart_test_autoscaling_job_count
975+
976+
chart_test_autoscaling_job_count:
977+
MATRIX_TESTS=$(or $(MATRIX_TESTS), "AutoscalingTestsScaleUp") SCALING_STRATEGY=$(or $(SCALING_STRATEGY), "default") \
978+
PLATFORMS=$(PLATFORMS) RELEASE_NAME=selenium TEST_PATCHED_KEDA=true SELENIUM_GRID_PROTOCOL=http SELENIUM_GRID_HOST=$$(hostname -I | cut -d' ' -f1) \
979+
SELENIUM_GRID_PORT=80 SELENIUM_GRID_MONITORING=false CLEAR_POD_HISTORY=true SET_MAX_REPLICAS=100 ENABLE_VIDEO_RECORDER=false \
980+
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \
981+
./tests/charts/make/chart_test.sh JobAutoscaling
982+
983+
chart_test_autoscaling_deployment_count_chaos:
984+
MATRIX_TESTS=AutoScalingTestsScaleChaos \
985+
make chart_test_autoscaling_deployment_count
986+
987+
chart_test_autoscaling_deployment_count_max_sessions:
988+
MAX_SESSIONS_FIREFOX=3 MAX_SESSIONS_EDGE=2 MAX_SESSIONS_CHROME=2 \
989+
make chart_test_autoscaling_deployment_count
990+
991+
chart_test_autoscaling_deployment_count:
992+
MATRIX_TESTS=$(or $(MATRIX_TESTS), "AutoscalingTestsScaleUp") \
993+
PLATFORMS=$(PLATFORMS) RELEASE_NAME=selenium TEST_PATCHED_KEDA=true SELENIUM_GRID_PROTOCOL=http SELENIUM_GRID_HOST=$$(hostname -I | cut -d' ' -f1) \
994+
SELENIUM_GRID_PORT=80 SELENIUM_GRID_MONITORING=false CLEAR_POD_HISTORY=true SET_MAX_REPLICAS=100 ENABLE_VIDEO_RECORDER=false \
995+
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \
996+
./tests/charts/make/chart_test.sh DeploymentAutoscaling
997+
964998
chart_test_delete:
965999
helm del test -n selenium || true
9661000
helm del selenium -n selenium || true

tests/AutoscalingTests/__init__.py

Whitespace-only changes.

tests/AutoscalingTests/common.py

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
import unittest
2+
import random
3+
import time
4+
import subprocess
5+
import signal
6+
import concurrent.futures
7+
import csv
8+
import os
9+
from selenium import webdriver
10+
from selenium.webdriver.firefox.options import Options as FirefoxOptions
11+
from selenium.webdriver.edge.options import Options as EdgeOptions
12+
from selenium.webdriver.chrome.options import Options as ChromeOptions
13+
from selenium.webdriver.remote.client_config import ClientConfig
14+
from csv2md.table import Table
15+
16+
BROWSER = {
17+
"chrome": ChromeOptions(),
18+
"firefox": FirefoxOptions(),
19+
"edge": EdgeOptions(),
20+
}
21+
22+
SELENIUM_GRID_HOST = os.getenv("SELENIUM_GRID_HOST", "localhost")
23+
24+
CLIENT_CONFIG = ClientConfig(
25+
remote_server_addr=f"http://{SELENIUM_GRID_HOST}/selenium/wd/hub",
26+
keep_alive=True,
27+
timeout=3600,
28+
)
29+
30+
FIELD_NAMES = ["Iteration", "New request sessions", "Requests accepted time", "Sessions failed", "New scaled pods", "Total sessions", "Total pods", "Gaps"]
31+
32+
def get_pod_count():
33+
result = subprocess.run(["kubectl", "get", "pods", "-A", "--no-headers"], capture_output=True, text=True)
34+
return len([line for line in result.stdout.splitlines() if "selenium-node-" in line and "Running" in line])
35+
36+
def create_session(browser_name):
37+
return webdriver.Remote(command_executor=CLIENT_CONFIG.remote_server_addr, options=BROWSER[browser_name], client_config=CLIENT_CONFIG)
38+
39+
def wait_for_count_matches(sessions, timeout=10, interval=5):
40+
elapsed = 0
41+
while elapsed < timeout:
42+
pod_count = get_pod_count()
43+
if pod_count == len(sessions):
44+
break
45+
print(f"VALIDATING: Waiting for pods to match sessions... ({elapsed}/{timeout} seconds elapsed)")
46+
time.sleep(interval)
47+
elapsed += interval
48+
if pod_count != len(sessions):
49+
print(f"WARN: Mismatch between pod count and session count after {timeout} seconds. Gaps: {pod_count - len(sessions)}")
50+
else:
51+
print(f"PASS: Pod count matches session count after {elapsed} seconds.")
52+
53+
def close_all_sessions(sessions):
54+
for session in sessions:
55+
session.quit()
56+
sessions.clear()
57+
return sessions
58+
59+
def create_sessions_in_parallel(new_request_sessions):
60+
failed_jobs = 0
61+
with concurrent.futures.ThreadPoolExecutor() as executor:
62+
futures = [executor.submit(create_session, random.choice(list(BROWSER.keys()))) for _ in range(new_request_sessions)]
63+
sessions = []
64+
for future in concurrent.futures.as_completed(futures):
65+
try:
66+
sessions.append(future.result())
67+
except Exception as e:
68+
print(f"ERROR: Failed to create session: {e}")
69+
failed_jobs += 1
70+
print(f"Total failed jobs: {failed_jobs}")
71+
return sessions
72+
73+
def randomly_quit_sessions(sessions, sublist_size):
74+
if sessions:
75+
sessions_to_quit = random.sample(sessions, min(sublist_size, len(sessions)))
76+
for session in sessions_to_quit:
77+
session.quit()
78+
sessions.remove(session)
79+
print(f"QUIT: {len(sessions_to_quit)} sessions have been randomly quit.")
80+
return sessions
81+
82+
def export_results_to_csv(output_file, field_names, results):
83+
with open(output_file, mode="w") as csvfile:
84+
writer = csv.DictWriter(csvfile, fieldnames=field_names)
85+
writer.writeheader()
86+
writer.writerows(results)
87+
88+
def export_results_csv_to_md(csv_file, md_file):
89+
with open(csv_file) as f:
90+
table = Table.parse_csv(f)
91+
with open(md_file, mode="w") as f:
92+
f.write(table.markdown())
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import unittest
2+
import random
3+
import time
4+
import signal
5+
import csv
6+
from csv2md.table import Table
7+
from .common import *
8+
9+
SESSIONS = []
10+
RESULTS = []
11+
12+
def signal_handler(signum, frame):
13+
print("Signal received, quitting all sessions...")
14+
close_all_sessions(SESSIONS)
15+
16+
signal.signal(signal.SIGTERM, signal_handler)
17+
signal.signal(signal.SIGINT, signal_handler)
18+
19+
class SeleniumAutoscalingTests(unittest.TestCase):
20+
def test_run_tests(self):
21+
try:
22+
for iteration in range(10):
23+
new_request_sessions = random.randint(2, 20)
24+
start_time = time.time()
25+
start_pods = get_pod_count()
26+
new_sessions = create_sessions_in_parallel(new_request_sessions)
27+
failed_sessions = new_request_sessions - len(new_sessions)
28+
end_time = time.time()
29+
stop_pods = get_pod_count()
30+
SESSIONS.extend(new_sessions)
31+
elapsed_time = end_time - start_time
32+
new_scaled_pods = stop_pods - start_pods
33+
total_sessions = len(SESSIONS)
34+
total_pods = get_pod_count()
35+
RESULTS.append({
36+
FIELD_NAMES[0]: iteration + 1,
37+
FIELD_NAMES[1]: new_request_sessions,
38+
FIELD_NAMES[2]: f"{elapsed_time:.2f} s",
39+
FIELD_NAMES[3]: failed_sessions,
40+
FIELD_NAMES[4]: new_scaled_pods,
41+
FIELD_NAMES[5]: total_sessions,
42+
FIELD_NAMES[6]: total_pods,
43+
FIELD_NAMES[7]: total_pods - total_sessions,
44+
})
45+
print(f"ADDING: Created {new_request_sessions} new sessions in {elapsed_time:.2f} seconds.")
46+
print(f"INFO: Total sessions: {total_sessions}")
47+
print(f"INFO: Total pods: {total_pods}")
48+
randomly_quit_sessions(SESSIONS, 10)
49+
time.sleep(5)
50+
finally:
51+
print(f"FINISH: Closing {len(SESSIONS)} sessions.")
52+
close_all_sessions(SESSIONS)
53+
output_file = f"tests/scale_up_results_{random.randint(1, 10000)}"
54+
export_results_to_csv(f"{output_file}.csv", FIELD_NAMES, RESULTS)
55+
export_results_csv_to_md(f"{output_file}.csv", f"{output_file}.md")
56+
57+
if __name__ == "__main__":
58+
unittest.main()
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import unittest
2+
import random
3+
import time
4+
import signal
5+
import csv
6+
from csv2md.table import Table
7+
from .common import *
8+
9+
SESSIONS = []
10+
RESULTS = []
11+
12+
def signal_handler(signum, frame):
13+
print("Signal received, quitting all sessions...")
14+
close_all_sessions(SESSIONS)
15+
16+
signal.signal(signal.SIGTERM, signal_handler)
17+
signal.signal(signal.SIGINT, signal_handler)
18+
19+
class SeleniumAutoscalingTests(unittest.TestCase):
20+
def test_run_tests(self):
21+
try:
22+
for iteration in range(10):
23+
new_request_sessions = random.randint(1, 3)
24+
start_time = time.time()
25+
start_pods = get_pod_count()
26+
new_sessions = create_sessions_in_parallel(new_request_sessions)
27+
failed_sessions = new_request_sessions - len(new_sessions)
28+
end_time = time.time()
29+
stop_pods = get_pod_count()
30+
SESSIONS.extend(new_sessions)
31+
elapsed_time = end_time - start_time
32+
new_scaled_pods = stop_pods - start_pods
33+
total_sessions = len(SESSIONS)
34+
total_pods = get_pod_count()
35+
RESULTS.append({
36+
FIELD_NAMES[0]: iteration + 1,
37+
FIELD_NAMES[1]: new_request_sessions,
38+
FIELD_NAMES[2]: f"{elapsed_time:.2f} s",
39+
FIELD_NAMES[3]: failed_sessions,
40+
FIELD_NAMES[4]: new_scaled_pods,
41+
FIELD_NAMES[5]: total_sessions,
42+
FIELD_NAMES[6]: total_pods,
43+
FIELD_NAMES[7]: total_pods - total_sessions,
44+
})
45+
print(f"ADDING: Created {new_request_sessions} new sessions in {elapsed_time:.2f} seconds.")
46+
print(f"INFO: Total sessions: {total_sessions}")
47+
print(f"INFO: Total pods: {total_pods}")
48+
if iteration % 4 == 0:
49+
randomly_quit_sessions(SESSIONS, 15)
50+
time.sleep(5)
51+
finally:
52+
print(f"FINISH: Closing {len(SESSIONS)} sessions.")
53+
close_all_sessions(SESSIONS)
54+
output_file = f"tests/scale_up_results_{random.randint(1, 10000)}"
55+
export_results_to_csv(f"{output_file}.csv", FIELD_NAMES, RESULTS)
56+
export_results_csv_to_md(f"{output_file}.csv", f"{output_file}.md")
57+
58+
if __name__ == "__main__":
59+
unittest.main()

tests/bootstrap.sh

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
#!/usr/bin/env bash
2+
set -o xtrace
3+
4+
MATRIX_TESTS=${MATRIX_TESTS:-"default"}
5+
26
cd tests || true
37

48
if [ "${CI:-false}" = "false" ]; then
@@ -14,10 +18,18 @@ else
1418
python3 -m pip install selenium==${BINDING_VERSION} | grep -v 'Requirement already satisfied'
1519
fi
1620

17-
python3 -m pip install docker requests chardet | grep -v 'Requirement already satisfied'
21+
python3 -m pip install -r requirements.txt | grep -v 'Requirement already satisfied'
1822

19-
python3 test.py $1
20-
ret_code=$?
23+
if [ "$1" = "AutoscalingTestsScaleUp" ]; then
24+
python3 -m unittest AutoscalingTests.test_scale_up
25+
ret_code=$?
26+
elif [ "$1" = "AutoScalingTestsScaleChaos" ]; then
27+
python3 -m unittest AutoscalingTests.test_scale_chaos
28+
ret_code=$?
29+
else
30+
python3 test.py $1
31+
ret_code=$?
32+
fi
2133

2234
if [ "${CI:-false}" = "false" ]; then
2335
deactivate

tests/charts/ci/DeploymentAutoscaling-values.yaml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ autoscaling:
55
maxReplicaCount: 4
66
pollingInterval: 10
77
scaledObjectOptions:
8-
cooldownPeriod: 30
8+
cooldownPeriod: ${AUTOSCALING_COOLDOWN_PERIOD}
99
terminationGracePeriodSeconds: 360
1010

1111
# Configuration for chrome nodes
@@ -47,6 +47,8 @@ chromeNode:
4747
value: "1080"
4848
- name: TZ
4949
value: "Asia/Saigon"
50+
- name: SE_NODE_SESSION_TIMEOUT
51+
value: "3600"
5052
readinessProbe:
5153
enabled: &readinessProbe true
5254
livenessProbe:

tests/charts/ci/JobAutoscaling-values.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ chromeNode:
2222
value: "1080"
2323
- name: TZ
2424
value: "Asia/Saigon"
25+
- name: SE_NODE_SESSION_TIMEOUT
26+
value: "3600"
2527
readinessProbe:
2628
enabled: &readinessProbe false
2729
livenessProbe:

tests/charts/ci/base-recorder-values.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
# AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}"
1212

1313
videoRecorder:
14-
enabled: true
14+
enabled: ${ENABLE_VIDEO_RECORDER}
1515
extraVolumes:
1616
# - name: videos
1717
# persistentVolumeClaim:

tests/charts/make/chart_test.sh

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ HUB_CHECKS_INTERVAL=${HUB_CHECKS_INTERVAL:-45}
2323
HUB_CHECKS_MAX_ATTEMPTS=${HUB_CHECKS_MAX_ATTEMPTS:-6}
2424
WEB_DRIVER_WAIT_TIMEOUT=${WEB_DRIVER_WAIT_TIMEOUT:-120}
2525
AUTOSCALING_POLL_INTERVAL=${AUTOSCALING_POLL_INTERVAL:-20}
26+
AUTOSCALING_COOLDOWN_PERIOD=${AUTOSCALING_COOLDOWN_PERIOD:-"1800"}
27+
ENABLE_VIDEO_RECORDER=${ENABLE_VIDEO_RECORDER:-"true"}
2628
SCALING_STRATEGY=${SCALING_STRATEGY:-"default"}
2729
SKIP_CLEANUP=${SKIP_CLEANUP:-"true"} # For debugging purposes, retain the cluster after the test run
2830
CHART_CERT_PATH=${CHART_CERT_PATH:-"${CHART_PATH}/certs/tls.crt"}
@@ -116,6 +118,8 @@ export SELENIUM_NAMESPACE=${SELENIUM_NAMESPACE}
116118
export TEST_PV_CLAIM_NAME=${TEST_PV_CLAIM_NAME}
117119
export HOST_PATH=$(realpath ./tests/videos)
118120
export SELENIUM_ENABLE_MANAGED_DOWNLOADS=${SELENIUM_ENABLE_MANAGED_DOWNLOADS}
121+
export AUTOSCALING_COOLDOWN_PERIOD=${AUTOSCALING_COOLDOWN_PERIOD}
122+
export ENABLE_VIDEO_RECORDER=${ENABLE_VIDEO_RECORDER}
119123
RECORDER_VALUES_FILE=${TEST_VALUES_PATH}/base-recorder-values.yaml
120124
MATRIX_BROWSER_VALUES_FILE=${TEST_VALUES_PATH}/${MATRIX_BROWSER}-values.yaml
121125
envsubst < ${RECORDER_VALUES_FILE} > ./tests/tests/base-recorder-values.yaml
@@ -189,6 +193,13 @@ if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ] && [ -n "${SET_MAX_REPLICAS}" ];
189193
"
190194
fi
191195

196+
if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ] && [ "${CLEAR_POD_HISTORY}" = "true" ]; then
197+
HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \
198+
--set autoscaling.scaledJobOptions.successfulJobsHistoryLimit=0 \
199+
--set autoscaling.scaledJobOptions.failedJobsHistoryLimit=0 \
200+
"
201+
fi
202+
192203
if [ "${CHART_ENABLE_INGRESS_HOSTNAME}" = "true" ]; then
193204
if [[ ! $(cat /etc/hosts) == *"${HOSTNAME_ADDRESS}"* ]]; then
194205
sudo -- sh -c -e "echo \"$(hostname -I | cut -d' ' -f1) ${HOSTNAME_ADDRESS}\" >> /etc/hosts"
@@ -440,6 +451,10 @@ elif [ "${MATRIX_TESTS}" = "CDPTests" ]; then
440451
if [ "${TEST_PLATFORMS}" = "linux/amd64" ]; then
441452
./tests/CDPTests/bootstrap.sh "MicrosoftEdge"
442453
fi
454+
elif [ "${MATRIX_TESTS}" = "AutoscalingTestsScaleUp" ]; then
455+
./tests/bootstrap.sh ${MATRIX_TESTS}
456+
elif [ "${MATRIX_TESTS}" = "AutoScalingTestsScaleChaos" ]; then
457+
./tests/bootstrap.sh ${MATRIX_TESTS}
443458
else
444459
./tests/bootstrap.sh ${MATRIX_BROWSER}
445460
fi

0 commit comments

Comments
 (0)