Skip to content

Commit e61f20c

Browse files
VietND96VietND96
authored
[ci] Add simple Playwright parallel tests connect to autoscaling Grid in K8s (#2409)
Signed-off-by: Viet Nguyen Duc <[email protected]>
1 parent dedd69b commit e61f20c

File tree

11 files changed

+205
-30
lines changed

11 files changed

+205
-30
lines changed

‎.circleci/config.yml‎

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,25 @@ version: 2.1
33
workflows:
44
build-and-test-multi-arch:
55
jobs:
6+
- kubernetes-test:
7+
name: "K8s test - Playwright Connect Autoscaling Grid"
8+
platforms: linux/arm64
9+
machine-type: ubuntu2204arm64large
10+
k8s-version: 'v1.25.16'
11+
test-strategy: playwright_connect_grid
12+
cluster: 'minikube'
13+
helm-version: 'v3.10.3'
14+
docker-version: '24.0.9'
15+
test-upgrade: true
616
- kubernetes-test:
717
name: "K8s test - Autoscaling disabled"
818
platforms: linux/arm64
919
machine-type: ubuntu2204arm64large
1020
k8s-version: 'v1.26.15'
1121
test-strategy: disabled
1222
cluster: 'minikube'
13-
helm-version: 'v3.10.3'
23+
helm-version: 'v3.11.3'
24+
docker-version: '24.0.9'
1425
test-upgrade: true
1526
- kubernetes-test:
1627
name: "K8s test - Autoscaling Jobs"
@@ -19,43 +30,48 @@ workflows:
1930
k8s-version: 'v1.27.16'
2031
test-strategy: job
2132
cluster: 'minikube'
22-
helm-version: 'v3.11.3'
33+
helm-version: 'v3.12.3'
34+
docker-version: '24.0.9'
2335
test-upgrade: true
2436
- kubernetes-test:
2537
name: "K8s test - Autoscaling Deployments"
2638
platforms: linux/arm64
2739
machine-type: ubuntu2204arm64large
28-
k8s-version: 'v1.28.13'
40+
k8s-version: 'v1.28.14'
2941
test-strategy: deployment
3042
cluster: 'minikube'
31-
helm-version: 'v3.12.3'
43+
helm-version: 'v3.13.3'
44+
docker-version: '24.0.9'
3245
test-upgrade: true
3346
- kubernetes-test:
3447
name: "K8s test - Autoscaling Jobs - HTTPS"
3548
platforms: linux/arm64
3649
machine-type: ubuntu2204arm64large
37-
k8s-version: 'v1.29.8'
50+
k8s-version: 'v1.29.9'
3851
test-strategy: job_https
3952
cluster: 'minikube'
40-
helm-version: 'v3.13.3'
53+
helm-version: 'v3.14.3'
54+
docker-version: '25.0.5'
4155
test-upgrade: true
4256
- kubernetes-test:
4357
name: "K8s test - Autoscaling Jobs - Ingress hostname"
4458
platforms: linux/arm64
4559
machine-type: ubuntu2204arm64large
46-
k8s-version: 'v1.30.4'
60+
k8s-version: 'v1.30.5'
4761
test-strategy: job_hostname
4862
cluster: 'minikube'
49-
helm-version: 'v3.14.4'
63+
helm-version: 'v3.15.4'
64+
docker-version: '26.1.4'
5065
test-upgrade: true
5166
- kubernetes-test:
5267
name: "K8s test - Autoscaling Deployments - HTTPS"
5368
platforms: linux/arm64
5469
machine-type: ubuntu2204arm64large
55-
k8s-version: 'v1.31.0'
70+
k8s-version: 'v1.31.1'
5671
test-strategy: deployment_https
5772
cluster: 'minikube'
58-
helm-version: 'v3.15.4'
73+
helm-version: 'v3.16.1'
74+
docker-version: '27.2.0'
5975
test-upgrade: true
6076
- docker-test:
6177
name: "Docker test - Use random user (true)"
@@ -206,6 +222,8 @@ jobs:
206222
type: string
207223
helm-version:
208224
type: string
225+
docker-version:
226+
type: string
209227
test-upgrade:
210228
type: boolean
211229
executor: << parameters.machine-type >>
@@ -217,6 +235,7 @@ jobs:
217235
CLUSTER: << parameters.cluster >>
218236
KUBERNETES_VERSION: << parameters.k8s-version >>
219237
HELM_VERSION: << parameters.helm-version >>
238+
DOCKER_VERSION: << parameters.docker-version >>
220239
TEST_UPGRADE_CHART: << parameters.test-upgrade >>
221240
steps:
222241
- run:
@@ -266,14 +285,20 @@ jobs:
266285
name: "Test Selenium Grid on Kubernetes"
267286
no_output_timeout: 30m
268287
command: |
288+
if [ "${TEST_STRATEGY}" == "playwright_connect_grid" ]; then
289+
PLATFORMS=${PLATFORMS} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${TEST_STRATEGY} && make test_video_integrity
290+
exit $?
291+
fi
269292
N=3
270293
while [ $N -gt 0 ]; do
271294
output=$(eval "PLATFORMS=${PLATFORMS} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${TEST_STRATEGY} && make test_video_integrity")
272295
status=$?
273296
if [ $status -eq 0 ]; then
297+
echo "${output}"
274298
echo "Tests passed"
275299
exit 0
276300
else
301+
echo "${output}"
277302
echo "Tests failed. Retrying..."
278303
N=$((N-1))
279304
sleep 10

‎.github/workflows/helm-chart-test.yml‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,40 +34,46 @@ jobs:
3434
fail-fast: false
3535
matrix:
3636
include:
37+
- k8s-version: 'v1.25.16'
38+
test-strategy: playwright_connect_grid
39+
cluster: 'minikube'
40+
helm-version: 'v3.10.3'
41+
docker-version: '24.0.9'
42+
test-upgrade: true
3743
- k8s-version: 'v1.26.15'
3844
test-strategy: disabled
3945
cluster: 'minikube'
40-
helm-version: 'v3.10.3'
46+
helm-version: 'v3.11.3'
4147
docker-version: '24.0.9'
4248
test-upgrade: true
4349
- k8s-version: 'v1.27.16'
4450
test-strategy: job
4551
cluster: 'minikube'
46-
helm-version: 'v3.11.3'
52+
helm-version: 'v3.12.3'
4753
docker-version: '24.0.9'
4854
test-upgrade: true
49-
- k8s-version: 'v1.28.13'
55+
- k8s-version: 'v1.28.14'
5056
test-strategy: deployment
5157
cluster: 'minikube'
52-
helm-version: 'v3.12.3'
58+
helm-version: 'v3.13.3'
5359
docker-version: '24.0.9'
5460
test-upgrade: true
55-
- k8s-version: 'v1.29.8'
61+
- k8s-version: 'v1.29.9'
5662
test-strategy: job_https
5763
cluster: 'minikube'
58-
helm-version: 'v3.13.3'
64+
helm-version: 'v3.14.3'
5965
docker-version: '25.0.5'
6066
test-upgrade: true
61-
- k8s-version: 'v1.30.4'
67+
- k8s-version: 'v1.30.5'
6268
test-strategy: job_hostname
6369
cluster: 'minikube'
64-
helm-version: 'v3.14.4'
70+
helm-version: 'v3.15.4'
6571
docker-version: '26.1.4'
6672
test-upgrade: true
67-
- k8s-version: 'v1.31.0'
73+
- k8s-version: 'v1.31.1'
6874
test-strategy: deployment_https
6975
cluster: 'minikube'
70-
helm-version: 'v3.15.4'
76+
helm-version: 'v3.16.1'
7177
docker-version: '27.2.0'
7278
test-upgrade: true
7379
env:

‎.keda/README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ If you are deploying KEDA core using their official Helm [chart](https://github.
4040
4141
If you are deployment Selenium Grid chart with `autoscaling.enabled` is `true` (implies installing KEDA sub-chart), KEDA images registry and tag already set in the `values.yaml`. Refer to list [configuration](../charts/selenium-grid/CONFIGURATION.md).
4242

43+
If you want to disable default patched KEDA image tags in Selenium Grid chart, you can set via Helm CLI `--set keda.image=null` or the same in values file.
44+
4345
# Pull requests under testing
4446

4547
Here is list of pull requests that are under testing and will be merged to the upstream KEDA repository.

‎Makefile‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -948,12 +948,12 @@ chart_test_autoscaling_job:
948948
TEMPLATE_OUTPUT_FILENAME="k8s_fullDistributed_secureIngress_externalCerts_ingressHostName_ingressTLSInline_autoScaling_scaledJob_existingKEDA_prefixSelenium_nodeChromium_enableTracing.yaml" \
949949
./tests/charts/make/chart_test.sh JobAutoscaling
950950

951-
chart_test_language_bindings:
952-
PLATFORMS=$(PLATFORMS) \
953-
SELENIUM_GRID_HOST=$$(hostname -i) \
954-
SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 \
951+
chart_test_autoscaling_playwright_connect_grid:
952+
PLATFORMS=$(PLATFORMS) CHART_ENABLE_TRACING=true CHART_ENABLE_BASIC_AUTH=true MATRIX_TESTS=CDPTests \
953+
SECURE_INGRESS_ONLY_DEFAULT=true SECURE_USE_EXTERNAL_CERT=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -i) SELENIUM_GRID_PORT=443 \
955954
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \
956-
./tests/charts/make/chart_test.sh DeploymentAutoscaling
955+
TEMPLATE_OUTPUT_FILENAME="k8s_playwright_connect_grid_basicAuth_secureIngress_ingressPublicIP_autoScaling.yaml" \
956+
./tests/charts/make/chart_test.sh JobAutoscaling
957957

958958
chart_test_delete:
959959
helm del test -n selenium || true

‎tests/CDPTests/.gitignore‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
node_modules/
2+
/test-results/
3+
/playwright-report/
4+
/playwright/.cache/
5+
.idea/
6+
.vscode/
7+
jsonReports/
8+
package-lock.json

‎tests/CDPTests/bootstrap.sh‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/usr/bin/env bash
2+
3+
cd tests/CDPTests || true
4+
5+
npm install
6+
npx playwright install --force chromium
7+
8+
BROWSER=${1:-"chrome"}
9+
10+
SELENIUM_REMOTE_URL="${SELENIUM_GRID_PROTOCOL}://${SELENIUM_GRID_HOST}:${SELENIUM_GRID_PORT}"
11+
echo "SELENIUM_REMOTE_URL=${SELENIUM_REMOTE_URL}" > .env
12+
13+
if [ -n ${SELENIUM_GRID_USERNAME} ] && [ -n ${SELENIUM_GRID_PASSWORD} ]; then
14+
BASIC_AUTH="$(echo -n "${SELENIUM_GRID_USERNAME}:${SELENIUM_GRID_PASSWORD}" | base64)"
15+
echo "SELENIUM_REMOTE_HEADERS={\"Authorization\": \"Basic ${BASIC_AUTH}\"}" >> .env
16+
fi
17+
18+
echo "SELENIUM_REMOTE_CAPABILITIES={\"browserName\": \"${BROWSER}\"}" >> .env
19+
echo "NODE_EXTRA_CA_CERTS=${CHART_CERT_PATH}" >> .env
20+
21+
cat .env
22+
23+
until [ "$(curl --noproxy "*" -sk -H "Authorization: Basic ${BASIC_AUTH}" -o /dev/null -w "%{http_code}" "${SELENIUM_REMOTE_URL}/status")" = "200" ]; do
24+
echo "Waiting for Grid to be ready..."
25+
sleep 1
26+
done
27+
28+
npx playwright test

‎tests/CDPTests/package.json‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"name": "CDPTests",
3+
"version": "1.47.2",
4+
"main": "index.js",
5+
"scripts": {},
6+
"keywords": [],
7+
"author": "",
8+
"license": "ISC",
9+
"devDependencies": {
10+
"@playwright/test": "^1.47.2"
11+
},
12+
"directories": {
13+
"test": "tests"
14+
},
15+
"dependencies": {
16+
"dotenv": "^16.3.1",
17+
"express": "^4.18.2",
18+
"playwright-core": "^1.47.2"
19+
},
20+
"description": ""
21+
}

‎tests/CDPTests/playwright.config.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import type {PlaywrightTestConfig} from '@playwright/test';
2+
import * as dotenv from 'dotenv';
3+
4+
dotenv.config();
5+
6+
const config: PlaywrightTestConfig = {
7+
timeout: 1500000,
8+
testMatch: ["tests/*.ts"],
9+
use: {
10+
headless: false,
11+
screenshot: "on",
12+
video: "on"
13+
},
14+
reporter: [["dot"], ["json", {
15+
outputFile: "jsonReports/jsonReport.json"
16+
}], ["html", {
17+
open: "never"
18+
}]],
19+
workers: 5
20+
};
21+
22+
export default config;

‎tests/CDPTests/tests/Tests.ts‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
const {test, expect} = require('@playwright/test');
2+
const path = require('path');
3+
4+
function sleep(ms: number) {
5+
return new Promise(resolve => setTimeout(resolve, ms));
6+
}
7+
8+
test.describe.parallel('Parallel tests connect to autoscaling Grid', () => {
9+
test('test_title', async ({page}) => {
10+
await page.goto('https://the-internet.herokuapp.com');
11+
await expect(page).toHaveTitle('The Internet');
12+
await sleep(2);
13+
});
14+
15+
test('test_with_frames', async ({page}) => {
16+
await page.goto('http://the-internet.herokuapp.com/nested_frames');
17+
const frame = page.frameLocator('frame[name="frame-top"]').frameLocator('frame[name="frame-middle"]');
18+
await expect(frame.locator('#content')).toHaveText('MIDDLE');
19+
await sleep(2);
20+
});
21+
22+
test('test_select_from_a_dropdown', async ({page}) => {
23+
await page.goto('http://the-internet.herokuapp.com/dropdown');
24+
const dropdown = await page.locator('#dropdown');
25+
await dropdown.selectOption({label: 'Option 1'});
26+
const selectedOption = await dropdown.inputValue();
27+
expect(selectedOption).toBe('1');
28+
await sleep(2);
29+
});
30+
31+
test('test_visit_basic_auth_secured_page', async ({page}) => {
32+
await page.goto('http://admin:[email protected]/basic_auth');
33+
const pageMessage = await page.locator('.example p').textContent();
34+
expect(pageMessage.trim()).toBe('Congratulations! You must have the proper credentials.');
35+
await sleep(2);
36+
});
37+
38+
test('test_download_file', async ({page}) => {
39+
await page.goto('https://the-internet.herokuapp.com/download');
40+
const fileLink = page.locator('a', {hasText: 'some-file.txt'});
41+
await fileLink.scrollIntoViewIfNeeded();
42+
const [download] = await Promise.all([
43+
page.waitForEvent('download'),
44+
fileLink.click()
45+
]);
46+
const fileName = download.suggestedFilename();
47+
expect(fileName).toBe('some-file.txt');
48+
await sleep(2);
49+
});
50+
});

‎tests/charts/make/chart_setup_env.sh‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -161,8 +161,9 @@ rm -rf ct.tar.gz
161161
ct version
162162
echo "==============================="
163163
echo "Installing helm-docs for AMD64 / ARM64"
164-
go install github.com/norwoodj/helm-docs/cmd/helm-docs@latest
165-
$HOME/go/bin/helm-docs -h
164+
mkdir -p $HOME/go/bin
165+
GOBIN=$HOME/go/bin go install github.com/norwoodj/helm-docs/cmd/helm-docs@latest
166+
$HOME/go/bin/helm-docs -h || true
166167
echo "==============================="
167168
echo "Installing envsubst for AMD64 / ARM64"
168169
ENVSUBST_VERSION="v1.4.2"
@@ -172,3 +173,12 @@ chmod +x envsubst
172173
sudo mv envsubst /usr/local/bin
173174
sudo ln -sf /usr/local/bin/envsubst /usr/bin/envsubst
174175
echo "==============================="
176+
echo "Installing Node for AMD64 / ARM64"
177+
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.0/install.sh | bash
178+
export NVM_DIR="$HOME/.nvm"
179+
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
180+
[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"
181+
source $HOME/.bashrc
182+
nvm install --lts
183+
node --version
184+
npm --version

0 commit comments

Comments
 (0)