Skip to content

Commit 65d7ccf

Browse files
oseoinoseoin
authored
Test splitting smoke tests by dependency (nginx#5803)
1 parent 04955be commit 65d7ccf

File tree

8 files changed

+493
-331
lines changed

8 files changed

+493
-331
lines changed

‎.github/actions/smoke-tests/action.yaml

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,20 @@ inputs:
2525
marker:
2626
description: Marker to use
2727
required: false
28+
label:
29+
description: Label for test
30+
required: false
2831
azure-ad-secret:
2932
description: Azure Active Directory secret for JWKs
3033
required: false
3134

3235
outputs:
3336
test-results-name:
3437
description: Test results name
35-
value: tests-${{ steps.k8s.outputs.cluster }}
38+
value: ${{ steps.k8s.outputs.test_name }}
39+
test-results-path:
40+
description: Test results full path
41+
value: ${{ steps.k8s.outputs.test_output_path }}
3642

3743
runs:
3844
using: composite
@@ -42,12 +48,20 @@ runs:
4248
run: |
4349
make -f tests/Makefile create-kind-cluster K8S_CLUSTER_NAME=${{ github.run_id }} K8S_CLUSTER_VERSION=${{ inputs.k8s-version }} K8S_TIMEOUT=${{ inputs.k8s-timeout }}
4450
make -f tests/Makefile image-load REGISTRY="" PREFIX=${{ inputs.image-name }} TAG=${{ inputs.tag }} K8S_CLUSTER_NAME=${{ github.run_id }}
45-
marker="${{ inputs.marker }}"
46-
nospaces="${marker// /_}"
47-
sanitized_marker="${nospaces//\'/}"
48-
name="${sanitized_marker:-${{ inputs.k8s-version }}}"
49-
echo "cluster_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${{ github.run_id }}-control-plane)" >> $GITHUB_OUTPUT
50-
echo "cluster=$(echo nginx-${{ inputs.image-type }}-$name)" >> $GITHUB_OUTPUT
51+
label="${{ inputs.label }}"
52+
nospaces="${label// /_}"
53+
noslash="${nospaces//\//_}"
54+
sanitized_marker="${noslash//\'/}"
55+
name="${sanitized_marker}-${{ inputs.k8s-version }}"
56+
cluster_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${{ github.run_id }}-control-plane)
57+
test_name=tests-nginx-${{ inputs.image-type }}-${name}.html
58+
test_output_path=${{ github.workspace }}/tests/${test_name}
59+
echo "cluster_ip=${cluster_ip}" >> $GITHUB_OUTPUT
60+
echo "test_name=${test_name}" >> $GITHUB_OUTPUT
61+
echo "test_output_path=${test_output_path}" >> $GITHUB_OUTPUT
62+
echo "Output:"
63+
echo " cluster_ip=${cluster_ip}"
64+
echo " test_output_path=${test_output_path}"
5165
shell: bash
5266

5367
- name: Setup Kubeconfig
@@ -56,21 +70,23 @@ runs:
5670
shell: bash
5771

5872
- name: Run Smoke Tests
73+
id: smoke-tests
5974
run: |
60-
touch tests-${{ steps.k8s.outputs.cluster }}.html
75+
touch ${{ steps.k8s.outputs.test_output_path }}
6176
docker run --rm \
6277
--name test-runner-${{ github.run_id }} \
6378
--network=kind \
6479
-v ${{ github.workspace }}/tests:/workspace/tests \
6580
-v ${{ github.workspace }}/deployments:/workspace/deployments \
6681
-v ${{ github.workspace }}/config:/workspace/config \
82+
-v ${{ steps.k8s.outputs.test_output_path }}:${{ steps.k8s.outputs.test_output_path }} \
6783
-v ~/.kube/kind/config:/root/.kube/config ${{ inputs.test-image }} \
6884
--context=kind-${{ github.run_id }} \
6985
--image=${{ inputs.image-name }}:${{ inputs.tag }} \
7086
--image-pull-policy=Never \
7187
--ic-type=nginx${{ contains(inputs.image-type, 'plus') && '-plus' || '' }}-ingress \
7288
--service=nodeport --node-ip=${{ steps.k8s.outputs.cluster_ip }} \
73-
--html=tests-${{ steps.k8s.outputs.cluster }}.html \
89+
--html=${{ steps.k8s.outputs.test_output_path }} \
7490
--self-contained-html \
7591
--durations=10 \
7692
--show-ic-logs=yes \

‎.github/data/matrix-smoke-nap.json

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
{
2+
"images": [
3+
{
4+
"label": "AP_WAF 1/4",
5+
"image": "ubi-9-plus-nap",
6+
"type": "plus",
7+
"nap_modules": "waf",
8+
"marker": "appprotect_waf_policies_allow",
9+
"platforms": "linux/amd64"
10+
},
11+
{
12+
"label": "AP_WAF 2/4",
13+
"image": "alpine-plus-nap-fips",
14+
"type": "plus",
15+
"nap_modules": "waf",
16+
"marker": "'appprotect_waf_policies and not appprotect_waf_policies_allow'",
17+
"platforms": "linux/amd64"
18+
},
19+
{
20+
"label": "AP_WAF 3/4",
21+
"image": "debian-plus-nap",
22+
"type": "plus",
23+
"nap_modules": "waf",
24+
"marker": "appprotect_waf_policies_grpc",
25+
"platforms": "linux/amd64"
26+
},
27+
{
28+
"label": "AP_WAF 4/4",
29+
"image": "debian-plus-nap",
30+
"type": "plus",
31+
"nap_modules": "waf",
32+
"marker": "'appprotect_watch or appprotect_batch or appprotect_integration'",
33+
"platforms": "linux/amd64"
34+
},
35+
{
36+
"label": "AP_DOS",
37+
"image": "debian-plus-nap",
38+
"type": "plus",
39+
"nap_modules": "dos",
40+
"marker": "'dos and not dos_learning'",
41+
"platforms": "linux/amd64"
42+
},
43+
{
44+
"label": "AP_DOS_Learning",
45+
"image": "ubi-8-plus-nap",
46+
"type": "plus",
47+
"nap_modules": "dos",
48+
"marker": "dos_learning",
49+
"platforms": "linux/amd64"
50+
}
51+
],
52+
"k8s": []
53+
}

‎.github/data/matrix-smoke-oss.json

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
{
2+
"images": [
3+
{
4+
"label": "ingresses 1/2",
5+
"image": "debian",
6+
"type": "oss",
7+
"marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls'",
8+
"platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
9+
},
10+
{
11+
"label": "ingresses 2/2",
12+
"image": "debian",
13+
"type": "oss",
14+
"marker": "'annotations or basic_auth or hsts or watch_namespace or wildcard_tls'",
15+
"platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
16+
},
17+
{
18+
"label": "VSR",
19+
"image": "alpine",
20+
"type": "oss",
21+
"marker": "vsr",
22+
"platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
23+
},
24+
{
25+
"label": "policies 1/2",
26+
"image": "alpine",
27+
"type": "oss",
28+
"marker": "'policies and not policies_rl and not policies_ac and not policies_jwt and not policies_mtls'",
29+
"platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
30+
},
31+
{
32+
"label": "policies 2/2",
33+
"image": "alpine",
34+
"type": "oss",
35+
"marker": "'policies_rl or policies_ac or policies_jwt or policies_mtls'",
36+
"platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
37+
},
38+
{
39+
"label": "VS 1/3",
40+
"image": "debian",
41+
"type": "oss",
42+
"marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'",
43+
"platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
44+
},
45+
{
46+
"label": "VS 2/3",
47+
"image": "debian",
48+
"type": "oss",
49+
"marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns'",
50+
"platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
51+
},
52+
{
53+
"label": "VS 3/3",
54+
"image": "debian",
55+
"type": "oss",
56+
"marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'",
57+
"platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
58+
},
59+
{
60+
"label": "TS",
61+
"image": "ubi",
62+
"type": "oss",
63+
"marker": "ts",
64+
"platforms": "linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
65+
}
66+
],
67+
"k8s": []
68+
}

‎.github/data/matrix-smoke-plus.json

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
{
2+
"images": [
3+
{
4+
"label": "VS 1/3",
5+
"image": "debian-plus",
6+
"type": "plus",
7+
"marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'",
8+
"platforms": "linux/arm64, linux/amd64"
9+
},
10+
{
11+
"label": "VS 2/3",
12+
"image": "debian-plus",
13+
"type": "plus",
14+
"marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns'",
15+
"platforms": "linux/arm64, linux/amd64"
16+
},
17+
{
18+
"label": "VS 3/3",
19+
"image": "debian-plus",
20+
"type": "plus",
21+
"marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'",
22+
"platforms": "linux/arm64, linux/amd64"
23+
},
24+
{
25+
"label": "TS",
26+
"image": "debian-plus",
27+
"type": "plus",
28+
"marker": "ts",
29+
"platforms": "linux/arm64, linux/amd64"
30+
},
31+
{
32+
"label": "ingresses 1/2",
33+
"image": "alpine-plus",
34+
"type": "plus",
35+
"marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls'",
36+
"platforms": "linux/arm64, linux/amd64"
37+
},
38+
{
39+
"label": "ingresses 2/2",
40+
"image": "alpine-plus",
41+
"type": "plus",
42+
"marker": "'annotations or basic_auth or hsts or watch_namespace or wildcard_tls'",
43+
"platforms": "linux/arm64, linux/amd64"
44+
},
45+
{
46+
"label": "VSR",
47+
"image": "alpine-plus",
48+
"type": "plus",
49+
"marker": "vsr",
50+
"platforms": "linux/arm64, linux/amd64"
51+
},
52+
{
53+
"label": "policies 1/2",
54+
"image": "ubi-plus",
55+
"type": "plus",
56+
"marker": "'policies and not policies_ac and not policies_jwt and not policies_mtls'",
57+
"platforms": "linux/arm64, linux/amd64, linux/s390x"
58+
},
59+
{
60+
"label": "policies 2/2",
61+
"image": "ubi-plus",
62+
"type": "plus",
63+
"marker": "'policies_ac or policies_jwt or policies_mtls'",
64+
"platforms": "linux/arm64, linux/amd64, linux/s390x"
65+
}
66+
],
67+
"k8s": []
68+
}

0 commit comments

Comments
 (0)