diff --git a/.github/data/matrix-smoke-nap.json b/.github/data/matrix-smoke-nap.json index e0c8c1ac3b..f16b4097a6 100644 --- a/.github/data/matrix-smoke-nap.json +++ b/.github/data/matrix-smoke-nap.json @@ -61,7 +61,7 @@ "image": "ubi-9-plus-nap", "type": "plus", "nap_modules": "dos", - "marker": "'dos_learning or otel'", + "marker": "'dos_learning'", "platforms": "linux/amd64" }, { diff --git a/.github/data/matrix-smoke-oss.json b/.github/data/matrix-smoke-oss.json index a9e87bbb1d..0884e140da 100644 --- a/.github/data/matrix-smoke-oss.json +++ b/.github/data/matrix-smoke-oss.json @@ -36,38 +36,59 @@ "platforms": "linux/arm64, linux/amd64" }, { - "label": "policies 1/2", + "label": "policies 1/4", "image": "alpine", "type": "oss", "marker": "'policies and not policies_rl and not policies_ac and not policies_jwt and not policies_mtls and not policies_cache'", "platforms": "linux/arm64, linux/amd64" }, { - "label": "policies 2/2", + "label": "policies 2/4", "image": "alpine", "type": "oss", - "marker": "'policies_rl or policies_ac or policies_jwt or policies_mtls or policies_cache or otel'", + "marker": "'policies_ac or policies_jwt'", "platforms": "linux/arm64, linux/amd64" }, { - "label": "VS 1/3", + "label": "policies 3/4", + "image": "alpine", + "type": "oss", + "marker": "'policies_rl'", + "platforms": "linux/arm64, linux/amd64" + }, + { + "label": "policies 4/4", + "image": "alpine", + "type": "oss", + "marker": "'policies_mtls or policies_cache or otel'", + "platforms": "linux/arm64, linux/amd64" + }, + { + "label": "VS 1/4", "image": "debian", "type": "oss", "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager and not vs_api and not vs_backup and not vs_use_cluster_ip and not vs_canary and not vs_upstream and not vs_config_map'", "platforms": "linux/arm64, linux/amd64" }, { - "label": "VS 2/3", + "label": "VS 2/4", "image": "debian", "type": "oss", "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns or vs_api or vs_backup or vs_use_cluster_ip or vs_canary or vs_upstream or vs_config_map'", "platforms": "linux/arm64, linux/amd64" }, { - "label": "VS 3/3", + "label": "VS 3/4", + "image": "debian", + "type": "oss", + "marker": "'vs_responses or vs_ipv6 or vs_rewrite'", + "platforms": "linux/arm64, linux/amd64" + }, + { + "label": "VS 4/4", "image": "debian", "type": "oss", - "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager or otel'", + "marker": "'vs_certmanager or otel'", "platforms": "linux/arm64, linux/amd64" }, { diff --git a/.github/data/matrix-smoke-plus.json b/.github/data/matrix-smoke-plus.json index 3750017dfd..80c8c3754f 100644 --- a/.github/data/matrix-smoke-plus.json +++ b/.github/data/matrix-smoke-plus.json @@ -1,42 +1,70 @@ { "images": [ { - "label": "VS 1/3", + "label": "VS 1/5", "image": "debian-plus", "type": "plus", - "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager and not vs_api and not vs_backup and not vs_use_cluster_ip and not vs_canary and not vs_upstream and not vs_config_map'", + "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager and not vs_api and not vs_backup and not vs_use_cluster_ip and not vs_canary and not vs_upstream and not vs_config_map and not vs_listeners'", "platforms": "linux/arm64, linux/amd64" }, { - "label": "VS 2/3", + "label": "VS 2/5", "image": "debian-plus", "type": "plus", "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns or vs_api or vs_backup or vs_use_cluster_ip or vs_canary or vs_upstream or vs_config_map'", "platforms": "linux/arm64, linux/amd64" }, { - "label": "VS 3/3", + "label": "VS 3/5", "image": "debian-plus", "type": "plus", - "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'", + "marker": "'vs_responses or vs_ipv6 or vs_rewrite'", "platforms": "linux/arm64, linux/amd64" }, { - "label": "TS", + "label": "VS 4/5", "image": "debian-plus", "type": "plus", - "marker": "'ts or otel'", + "marker": "'vs_certmanager'", "platforms": "linux/arm64, linux/amd64" }, { - "label": "ingresses 1/2", + "label": "VS 5/5", + "image": "debian-plus", + "type": "plus", + "marker": "'vs_listeners'", + "platforms": "linux/arm64, linux/amd64" + }, + { + "label": "TS 1/2", + "image": "debian-plus", + "type": "plus", + "marker": "'ts and not ts_tcp and not ts_udp'", + "platforms": "linux/arm64, linux/amd64" + }, + { + "label": "TS 2/2", + "image": "debian-plus", + "type": "plus", + "marker": "'ts_tcp or ts_udp or otel'", + "platforms": "linux/arm64, linux/amd64" + }, + { + "label": "ingresses 1/3", "image": "alpine-plus", "type": "plus", - "marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls'", + "marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls and not ingresses_smoke and not ingresses_jwt'", "platforms": "linux/arm64, linux/amd64" }, { - "label": "ingresses 2/2", + "label": "ingresses 2/3", + "image": "alpine-plus", + "type": "plus", + "marker": "'ingresses_smoke or ingresses_jwt'", + "platforms": "linux/arm64, linux/amd64" + }, + { + "label": "ingresses 3/3", "image": "alpine-plus-fips", "type": "plus", "marker": "'annotations or basic_auth or hsts or watch_namespace or wildcard_tls'", @@ -64,24 +92,38 @@ "platforms": "linux/arm64, linux/amd64" }, { - "label": "policies 1/3", + "label": "policies 1/5", "image": "ubi-9-plus", "type": "plus", "marker": "'policies and not policies_ac and not policies_jwt and not policies_mtls and not policies_rl and not policies_cache'", "platforms": "linux/arm64, linux/amd64" }, { - "label": "policies 2/3", + "label": "policies 2/5", + "image": "ubi-9-plus", + "type": "plus", + "marker": "'policies_ac or policies_mtls or policies_cache'", + "platforms": "linux/arm64, linux/amd64" + }, + { + "label": "policies 3/5", + "image": "ubi-9-plus", + "type": "plus", + "marker": "'policies_jwt or otel'", + "platforms": "linux/arm64, linux/amd64" + }, + { + "label": "policies 4/5", "image": "ubi-9-plus", "type": "plus", - "marker": "'policies_ac or policies_jwt or policies_mtls or otel'", + "marker": "'policies_rl_vs'", "platforms": "linux/arm64, linux/amd64" }, { - "label": "policies 3/3", + "label": "policies 5/5", "image": "ubi-9-plus", "type": "plus", - "marker": "'policies_rl or policies_cache'", + "marker": "'policies_rl_vsr'", "platforms": "linux/arm64, linux/amd64" }, { diff --git a/pyproject.toml b/pyproject.toml index 0405d7d069..fbded36641 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,11 +45,15 @@ markers = [ "dos_learning", "hsts", "ingresses", + "ingresses_jwt", + "ingresses_smoke", "multi_ns", "oidc", "otel", "policies", "policies_rl", + "policies_rl_vs", + "policies_rl_vsr", "policies_jwt", "policies_ac", "policies_mtls", @@ -60,6 +64,8 @@ markers = [ "smoke", "startup", "ts", + "ts_tcp", + "ts_udp", "upgrade", "vs", "vs_api", @@ -75,6 +81,7 @@ markers = [ "vs_certmanager", "vs_config_map", "vs_grpc", + "vs_listeners", "vs_upstream", "vs_use_cluster_ip", "vsr", diff --git a/tests/scripts/longest_test_job.py b/tests/scripts/longest_test_job.py new file mode 100755 index 0000000000..562b694ad9 --- /dev/null +++ b/tests/scripts/longest_test_job.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 + +import argparse + +from github import Auth, Github + +# parse args +parser = argparse.ArgumentParser() +parser.add_argument("-t", "--token", required=True, help="GitHub access token") +parser.add_argument("-o", "--owner", required=False, default="nginx", help="GitHub repository owner") +parser.add_argument("-r", "--repo", help="GitHub repository name", required=False, default="kubernetes-ingress") +parser.add_argument("-w", "--workflow", help="GitHub Actions workflow name", required=False, default="CI") +parser.add_argument("-b", "--branch", help="GitHub repository branch", required=False, default="main") +parser.add_argument( + "-d", "--duration", help="Minimum duration of jobs in seconds", required=False, default=900, type=int +) +args = parser.parse_args() +TOKEN = args.token +OWNER = args.owner +REPO = args.repo +BRANCH = args.branch +DURATION = args.duration +WORKFLOW = args.workflow + + +def get_github_handle(token): + # Authenticate to GitHub + auth = Auth.Token(token) + g = Github(auth=auth) + if g is None: + return None + return g + + +def get_github_repo(owner, repo, handle): + # Get the repository + repository = handle.get_repo(f"{owner}/{repo}") + return repository + + +def get_workflow_runs(repo, workflow_name, branch=None): + workflows = repo.get_workflows() + for workflow in workflows: + if workflow.name == workflow_name: + return workflow.get_runs(branch=branch, status="completed") + return None + + +def get_run_branch_jobs(runs): + results = {} + for run in runs: + results[run.id] = run.jobs() + return results + + +def get_run_durations(runs): + results = {} + for run in runs: + results[run.id] = run.timing().run_duration_ms / 1000 + return results + + +def convert_seconds(seconds): + minutes, remaining_seconds = divmod(seconds, 60) + hour, minutes = divmod(minutes, 60) + return "%d:%02d:%02d" % (hour, minutes, remaining_seconds) + + +g = get_github_handle(TOKEN) +if g is None: + print("Failed to authenticate to GitHub") + exit(1) +r = get_github_repo(OWNER, REPO, g) + +# Get the latest workflow runs +runs = get_workflow_runs(r, WORKFLOW, branch=BRANCH) +if not runs: + print("No workflow runs found.") + exit(1) +wj = get_run_branch_jobs(runs) +wd = get_run_durations(runs) +for run_id in sorted(wj.keys()): + duration = wd.get(run_id) + print(f"Workflow Run ID: {run_id}, Duration: {convert_seconds(duration)}") + for job in wj[run_id]: + job_duration = (job.completed_at - job.started_at).total_seconds() + if job.status == "completed" and job.conclusion == "success" and job_duration > DURATION: + print(f" Job: {job.name}, Duration: {convert_seconds(job_duration)}, URL: {job.html_url}") + +g.close() diff --git a/tests/suite/test_jwt_auth_mergeable.py b/tests/suite/test_jwt_auth_mergeable.py index b576e0e548..95151b55e2 100644 --- a/tests/suite/test_jwt_auth_mergeable.py +++ b/tests/suite/test_jwt_auth_mergeable.py @@ -195,6 +195,7 @@ def get_token_from_file(token_type) -> str: @pytest.mark.ingresses +@pytest.mark.ingresses_jwt @pytest.mark.skip_for_nginx_oss class TestJWTAuthMergeableMinions: def test_jwt_auth_response_codes_and_location(self, kube_apis, jwt_auth_setup, test_namespace): diff --git a/tests/suite/test_jwt_policies_jwksuri.py b/tests/suite/test_jwt_policies_jwksuri.py index 6f50e32097..735eb79e6d 100644 --- a/tests/suite/test_jwt_policies_jwksuri.py +++ b/tests/suite/test_jwt_policies_jwksuri.py @@ -171,6 +171,7 @@ def fin(): @pytest.mark.skip_for_nginx_oss @pytest.mark.policies +@pytest.mark.policies_jwt @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", [ diff --git a/tests/suite/test_jwt_secrets.py b/tests/suite/test_jwt_secrets.py index a99e687983..ad9d59e527 100644 --- a/tests/suite/test_jwt_secrets.py +++ b/tests/suite/test_jwt_secrets.py @@ -93,6 +93,7 @@ def fin(): @pytest.mark.ingresses +@pytest.mark.ingresses_jwt @pytest.mark.skip_for_nginx_oss class TestJWTSecrets: def test_response_code_200_and_server_name(self, jwt_secrets_setup, jwt_secret): diff --git a/tests/suite/test_policy_ingress_class.py b/tests/suite/test_policy_ingress_class.py index 7089a20f23..ba3782f703 100644 --- a/tests/suite/test_policy_ingress_class.py +++ b/tests/suite/test_policy_ingress_class.py @@ -18,6 +18,7 @@ @pytest.mark.policies +@pytest.mark.policies_rl @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", [ diff --git a/tests/suite/test_rl_policies.py b/tests/suite/test_rl_policies.py index 016dc9fc19..d9fc65ddab 100644 --- a/tests/suite/test_rl_policies.py +++ b/tests/suite/test_rl_policies.py @@ -100,6 +100,7 @@ @pytest.mark.policies @pytest.mark.policies_rl +@pytest.mark.policies_rl_vs @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", [ @@ -636,6 +637,7 @@ def test_rl_policy_jwt_claim_sub( @pytest.mark.policies @pytest.mark.policies_rl +@pytest.mark.policies_rl_vs @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", [ diff --git a/tests/suite/test_rl_policies_vsr.py b/tests/suite/test_rl_policies_vsr.py index ff8dd02b4a..efc4cb2694 100644 --- a/tests/suite/test_rl_policies_vsr.py +++ b/tests/suite/test_rl_policies_vsr.py @@ -119,6 +119,7 @@ @pytest.mark.policies @pytest.mark.policies_rl +@pytest.mark.policies_rl_vsr @pytest.mark.parametrize( "crd_ingress_controller, v_s_route_setup", [ @@ -683,6 +684,7 @@ def test_rl_policy_jwt_claim_sub_vsr( @pytest.mark.policies @pytest.mark.policies_rl +@pytest.mark.policies_rl_vsr @pytest.mark.parametrize( "crd_ingress_controller, v_s_route_setup", [ diff --git a/tests/suite/test_smoke.py b/tests/suite/test_smoke.py index 39013c7b8a..9de00a16ab 100644 --- a/tests/suite/test_smoke.py +++ b/tests/suite/test_smoke.py @@ -73,6 +73,7 @@ def fin(): @pytest.mark.smoke @pytest.mark.ingresses +@pytest.mark.ingresses_smoke class TestSmoke: @pytest.mark.parametrize( "ingress_controller", diff --git a/tests/suite/test_transport_server_tcp_load_balance.py b/tests/suite/test_transport_server_tcp_load_balance.py index 675df61cbc..c094f2c2b3 100644 --- a/tests/suite/test_transport_server_tcp_load_balance.py +++ b/tests/suite/test_transport_server_tcp_load_balance.py @@ -19,6 +19,7 @@ @pytest.mark.ts +@pytest.mark.ts_tcp @pytest.mark.skip_for_loadbalancer @pytest.mark.parametrize( "crd_ingress_controller, transport_server_setup", diff --git a/tests/suite/test_transport_server_udp_load_balance.py b/tests/suite/test_transport_server_udp_load_balance.py index 86fa8596ec..c0115ca418 100644 --- a/tests/suite/test_transport_server_udp_load_balance.py +++ b/tests/suite/test_transport_server_udp_load_balance.py @@ -42,6 +42,7 @@ def ipfamily_from_host(host): @pytest.mark.ts +@pytest.mark.ts_udp @pytest.mark.skip_for_loadbalancer @pytest.mark.parametrize( "crd_ingress_controller, transport_server_setup", diff --git a/tests/suite/test_virtual_server_custom_ip_listeners.py b/tests/suite/test_virtual_server_custom_ip_listeners.py index 93ba1fde29..a1939432d9 100644 --- a/tests/suite/test_virtual_server_custom_ip_listeners.py +++ b/tests/suite/test_virtual_server_custom_ip_listeners.py @@ -38,6 +38,7 @@ def restore_default_vs(kube_apis, virtual_server_setup) -> None: @pytest.mark.vs +@pytest.mark.vs_listeners @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", [ diff --git a/tests/suite/test_virtual_server_custom_listeners.py b/tests/suite/test_virtual_server_custom_listeners.py index 034ed26f5f..431b3b42c1 100644 --- a/tests/suite/test_virtual_server_custom_listeners.py +++ b/tests/suite/test_virtual_server_custom_listeners.py @@ -39,6 +39,7 @@ def restore_default_vs(kube_apis, virtual_server_setup) -> None: @pytest.mark.vs +@pytest.mark.vs_listeners @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", [