Split Python tests to reduce run time (#8419)

Author: pdabelf5

.github/data/matrix-smoke-nap.json

@@ -61,7 +61,7 @@
       "image": "ubi-9-plus-nap",
       "type": "plus",
       "nap_modules": "dos",
-      "marker": "'dos_learning or otel'",
+      "marker": "'dos_learning'",
       "platforms": "linux/amd64"
     },
     {

.github/data/matrix-smoke-oss.json

@@ -36,38 +36,59 @@
       "platforms": "linux/arm64, linux/amd64"
     },
     {
-      "label": "policies 1/2",
+      "label": "policies 1/4",
       "image": "alpine",
       "type": "oss",
       "marker": "'policies and not policies_rl and not policies_ac and not policies_jwt and not policies_mtls and not policies_cache'",
       "platforms": "linux/arm64, linux/amd64"
     },
     {
-      "label": "policies 2/2",
+      "label": "policies 2/4",
       "image": "alpine",
       "type": "oss",
-      "marker": "'policies_rl or policies_ac or policies_jwt or policies_mtls or policies_cache or otel'",
+      "marker": "'policies_ac or policies_jwt'",
       "platforms": "linux/arm64, linux/amd64"
     },
     {
-      "label": "VS 1/3",
+      "label": "policies 3/4",
+      "image": "alpine",
+      "type": "oss",
+      "marker": "'policies_rl'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "policies 4/4",
+      "image": "alpine",
+      "type": "oss",
+      "marker": "'policies_mtls or policies_cache or otel'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "VS 1/4",
       "image": "debian",
       "type": "oss",
       "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager and not vs_api and not vs_backup and not vs_use_cluster_ip and not vs_canary and not vs_upstream and not vs_config_map'",
       "platforms": "linux/arm64, linux/amd64"
     },
     {
-      "label": "VS 2/3",
+      "label": "VS 2/4",
       "image": "debian",
       "type": "oss",
       "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns or vs_api or vs_backup or vs_use_cluster_ip or vs_canary or vs_upstream or vs_config_map'",
       "platforms": "linux/arm64, linux/amd64"
     },
     {
-      "label": "VS 3/3",
+      "label": "VS 3/4",
+      "image": "debian",
+      "type": "oss",
+      "marker": "'vs_responses or vs_ipv6 or vs_rewrite'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "VS 4/4",
       "image": "debian",
       "type": "oss",
-      "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager or otel'",
+      "marker": "'vs_certmanager or otel'",
       "platforms": "linux/arm64, linux/amd64"
     },
     {

.github/data/matrix-smoke-plus.json

@@ -1,42 +1,70 @@
 {
   "images": [
     {
-      "label": "VS 1/3",
+      "label": "VS 1/5",
       "image": "debian-plus",
       "type": "plus",
-      "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager and not vs_api and not vs_backup and not vs_use_cluster_ip and not vs_canary and not vs_upstream and not vs_config_map'",
+      "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager and not vs_api and not vs_backup and not vs_use_cluster_ip and not vs_canary and not vs_upstream and not vs_config_map and not vs_listeners'",
       "platforms": "linux/arm64, linux/amd64"
     },
     {
-      "label": "VS 2/3",
+      "label": "VS 2/5",
       "image": "debian-plus",
       "type": "plus",
       "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns or vs_api or vs_backup or vs_use_cluster_ip or vs_canary or vs_upstream or vs_config_map'",
       "platforms": "linux/arm64, linux/amd64"
     },
     {
-      "label": "VS 3/3",
+      "label": "VS 3/5",
       "image": "debian-plus",
       "type": "plus",
-      "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'",
+      "marker": "'vs_responses or vs_ipv6 or vs_rewrite'",
       "platforms": "linux/arm64, linux/amd64"
     },
     {
-      "label": "TS",
+      "label": "VS 4/5",
       "image": "debian-plus",
       "type": "plus",
-      "marker": "'ts or otel'",
+      "marker": "'vs_certmanager'",
       "platforms": "linux/arm64, linux/amd64"
     },
     {
-      "label": "ingresses 1/2",
+      "label": "VS 5/5",
+      "image": "debian-plus",
+      "type": "plus",
+      "marker": "'vs_listeners'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "TS 1/2",
+      "image": "debian-plus",
+      "type": "plus",
+      "marker": "'ts and not ts_tcp and not ts_udp'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "TS 2/2",
+      "image": "debian-plus",
+      "type": "plus",
+      "marker": "'ts_tcp or ts_udp or otel'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "ingresses 1/3",
       "image": "alpine-plus",
       "type": "plus",
-      "marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls'",
+      "marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls and not ingresses_smoke and not ingresses_jwt'",
       "platforms": "linux/arm64, linux/amd64"
     },
     {
-      "label": "ingresses 2/2",
+      "label": "ingresses 2/3",
+      "image": "alpine-plus",
+      "type": "plus",
+      "marker": "'ingresses_smoke or ingresses_jwt'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "ingresses 3/3",
       "image": "alpine-plus-fips",
       "type": "plus",
       "marker": "'annotations or basic_auth or hsts or watch_namespace or wildcard_tls'",
@@ -64,24 +92,38 @@
       "platforms": "linux/arm64, linux/amd64"
     },
     {
-      "label": "policies 1/3",
+      "label": "policies 1/5",
       "image": "ubi-9-plus",
       "type": "plus",
       "marker": "'policies and not policies_ac and not policies_jwt and not policies_mtls and not policies_rl and not policies_cache'",
       "platforms": "linux/arm64, linux/amd64"
     },
     {
-      "label": "policies 2/3",
+      "label": "policies 2/5",
+      "image": "ubi-9-plus",
+      "type": "plus",
+      "marker": "'policies_ac or policies_mtls or policies_cache'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "policies 3/5",
+      "image": "ubi-9-plus",
+      "type": "plus",
+      "marker": "'policies_jwt or otel'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "policies 4/5",
       "image": "ubi-9-plus",
       "type": "plus",
-      "marker": "'policies_ac or policies_jwt or policies_mtls or otel'",
+      "marker": "'policies_rl_vs'",
       "platforms": "linux/arm64, linux/amd64"
     },
     {
-      "label": "policies 3/3",
+      "label": "policies 5/5",
       "image": "ubi-9-plus",
       "type": "plus",
-      "marker": "'policies_rl or policies_cache'",
+      "marker": "'policies_rl_vsr'",
       "platforms": "linux/arm64, linux/amd64"
     },
     {

pyproject.toml

@@ -45,11 +45,15 @@ markers = [
     "dos_learning",
     "hsts",
     "ingresses",
+    "ingresses_jwt",
+    "ingresses_smoke",
     "multi_ns",
     "oidc",
     "otel",
     "policies",
     "policies_rl",
+    "policies_rl_vs",
+    "policies_rl_vsr",
     "policies_jwt",
     "policies_ac",
     "policies_mtls",
@@ -60,6 +64,8 @@ markers = [
     "smoke",
     "startup",
     "ts",
+    "ts_tcp",
+    "ts_udp",
     "upgrade",
     "vs",
     "vs_api",
@@ -75,6 +81,7 @@ markers = [
     "vs_certmanager",
     "vs_config_map",
     "vs_grpc",
+    "vs_listeners",
     "vs_upstream",
     "vs_use_cluster_ip",
     "vsr",

tests/scripts/longest_test_job.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+import argparse
+
+from github import Auth, Github
+
+# parse args
+parser = argparse.ArgumentParser()
+parser.add_argument("-t", "--token", required=True, help="GitHub access token")
+parser.add_argument("-o", "--owner", required=False, default="nginx", help="GitHub repository owner")
+parser.add_argument("-r", "--repo", help="GitHub repository name", required=False, default="kubernetes-ingress")
+parser.add_argument("-w", "--workflow", help="GitHub Actions workflow name", required=False, default="CI")
+parser.add_argument("-b", "--branch", help="GitHub repository branch", required=False, default="main")
+parser.add_argument(
+    "-d", "--duration", help="Minimum duration of jobs in seconds", required=False, default=900, type=int
+)
+args = parser.parse_args()
+TOKEN = args.token
+OWNER = args.owner
+REPO = args.repo
+BRANCH = args.branch
+DURATION = args.duration
+WORKFLOW = args.workflow
+
+
+def get_github_handle(token):
+    # Authenticate to GitHub
+    auth = Auth.Token(token)
+    g = Github(auth=auth)
+    if g is None:
+        return None
+    return g
+
+
+def get_github_repo(owner, repo, handle):
+    # Get the repository
+    repository = handle.get_repo(f"{owner}/{repo}")
+    return repository
+
+
+def get_workflow_runs(repo, workflow_name, branch=None):
+    workflows = repo.get_workflows()
+    for workflow in workflows:
+        if workflow.name == workflow_name:
+            return workflow.get_runs(branch=branch, status="completed")
+    return None
+
+
+def get_run_branch_jobs(runs):
+    results = {}
+    for run in runs:
+        results[run.id] = run.jobs()
+    return results
+
+
+def get_run_durations(runs):
+    results = {}
+    for run in runs:
+        results[run.id] = run.timing().run_duration_ms / 1000
+    return results
+
+
+def convert_seconds(seconds):
+    minutes, remaining_seconds = divmod(seconds, 60)
+    hour, minutes = divmod(minutes, 60)
+    return "%d:%02d:%02d" % (hour, minutes, remaining_seconds)
+
+
+g = get_github_handle(TOKEN)
+if g is None:
+    print("Failed to authenticate to GitHub")
+    exit(1)
+r = get_github_repo(OWNER, REPO, g)
+
+# Get the latest workflow runs
+runs = get_workflow_runs(r, WORKFLOW, branch=BRANCH)
+if not runs:
+    print("No workflow runs found.")
+    exit(1)
+wj = get_run_branch_jobs(runs)
+wd = get_run_durations(runs)
+for run_id in sorted(wj.keys()):
+    duration = wd.get(run_id)
+    print(f"Workflow Run ID: {run_id}, Duration: {convert_seconds(duration)}")
+    for job in wj[run_id]:
+        job_duration = (job.completed_at - job.started_at).total_seconds()
+        if job.status == "completed" and job.conclusion == "success" and job_duration > DURATION:
+            print(f"  Job: {job.name}, Duration: {convert_seconds(job_duration)}, URL: {job.html_url}")
+
+g.close()

tests/suite/test_jwt_auth_mergeable.py

@@ -195,6 +195,7 @@ def get_token_from_file(token_type) -> str:
 
 
 @pytest.mark.ingresses
+@pytest.mark.ingresses_jwt
 @pytest.mark.skip_for_nginx_oss
 class TestJWTAuthMergeableMinions:
     def test_jwt_auth_response_codes_and_location(self, kube_apis, jwt_auth_setup, test_namespace):

tests/suite/test_jwt_policies_jwksuri.py

@@ -171,6 +171,7 @@ def fin():
 
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.policies
+@pytest.mark.policies_jwt
 @pytest.mark.parametrize(
     "crd_ingress_controller, virtual_server_setup",
     [

tests/suite/test_jwt_secrets.py

@@ -93,6 +93,7 @@ def fin():
 
 
 @pytest.mark.ingresses
+@pytest.mark.ingresses_jwt
 @pytest.mark.skip_for_nginx_oss
 class TestJWTSecrets:
     def test_response_code_200_and_server_name(self, jwt_secrets_setup, jwt_secret):

tests/suite/test_policy_ingress_class.py

@@ -18,6 +18,7 @@
 
 
 @pytest.mark.policies
+@pytest.mark.policies_rl
 @pytest.mark.parametrize(
     "crd_ingress_controller, virtual_server_setup",
     [

tests/suite/test_rl_policies.py

@@ -100,6 +100,7 @@
 
 @pytest.mark.policies
 @pytest.mark.policies_rl
+@pytest.mark.policies_rl_vs
 @pytest.mark.parametrize(
     "crd_ingress_controller, virtual_server_setup",
     [
@@ -636,6 +637,7 @@ def test_rl_policy_jwt_claim_sub(
 
 @pytest.mark.policies
 @pytest.mark.policies_rl
+@pytest.mark.policies_rl_vs
 @pytest.mark.parametrize(
     "crd_ingress_controller, virtual_server_setup",
     [