From bfbca03badba58b43ee9ea04a4760fb9a162d8a0 Mon Sep 17 00:00:00 2001 From: macgyver13 <> Date: Tue, 11 Mar 2025 11:24:28 -0400 Subject: [PATCH 1/5] - Add local_test_runner script - add delay to graph_test expect - increases reliability --- test/graph_test.py | 10 ++++++- test/local_test_runner.py | 56 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 test/local_test_runner.py diff --git a/test/graph_test.py b/test/graph_test.py index 3d0ad5848..9fec27993 100755 --- a/test/graph_test.py +++ b/test/graph_test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import json -import os +import os, time import pexpect from test_base import TestBase @@ -39,21 +39,29 @@ def directory_exists(self): self.sut = pexpect.spawn("warnet create") self.sut.expect("name", timeout=10) self.sut.sendline("ANewNetwork") + time.sleep(.1) self.sut.expect("many", timeout=10) self.sut.sendline("") + time.sleep(.1) self.sut.expect("connections", timeout=10) self.sut.sendline("") + time.sleep(.1) self.sut.expect("version", timeout=10) self.sut.sendline("") + time.sleep(.1) self.sut.expect("enable fork-observer", timeout=10) self.sut.sendline("") + time.sleep(.1) self.sut.expect("seconds", timeout=10) self.sut.sendline("") + time.sleep(.1) self.sut.expect("enable grafana", timeout=10) self.sut.sendline("") + time.sleep(.1) self.sut.expect("successfully", timeout=50) except Exception as e: print(f"\nReceived prompt text:\n {self.sut.before.decode('utf-8')}\n") + print(os.listdir(self.tmpdir)) raise e def run_created_network(self): diff --git a/test/local_test_runner.py b/test/local_test_runner.py new file mode 100644 index 000000000..2055047b8 --- /dev/null +++ b/test/local_test_runner.py @@ -0,0 +1,56 @@ +import yaml +import subprocess, sys + +# developers can use this tool to verify all repo workflow test execute to completion +# +# execute from repo root like this to execute all tests +# python test/local_test_runner.py +# +# add optional argument to just execute any matching tests +# python test/local_test_runner.py [ln_ | graph] + + +def has_key_path(d, key_path, separator="."): + """Check if a nested key path (dotted notation) exists in a dictionary.""" + keys = key_path.split(separator) + for key in keys: + if not isinstance(d, dict) or key not in d: + return False + d = d[key] + return True + +# Load the workflow file +with open('.github/workflows/test.yml', 'r') as file: + workflow = yaml.safe_load(file) + +tests_total=0 +tests_skipped=0 +tests_completed=0 + +for job_name, job_details in workflow.get("jobs", {}).items(): + if has_key_path(job_details, "strategy.matrix.test"): + print("Found test strategy job, starting serial execution of each test") + tests = job_details["strategy"]["matrix"]["test"] + + for test in tests: + tests_total+=1 + if len(sys.argv) > 1 and sys.argv[1] not in test: + print("skipping test as requested:",test) + tests_skipped+=1 + continue + command = f"python test/{test}" + print("###################################################################################################") + print("############## executing:", command) + print("###################################################################################################") + process = subprocess.run(command, shell=True) + if process.returncode != 0: + print("******** testing failed") + if process.stdout: print("stdout:", process.stdout) + if process.stderr: print("stderr:", process.stderr) + sys.exit(1) + tests_completed+=1 + +print("###################################################################################################") +print("testing complete") +print(f"{tests_completed} of {tests_total} complete - skipped: {tests_skipped} tests") +print("###################################################################################################") \ No newline at end of file From 6eab91874c01da62f972012fc43317694138ee72 Mon Sep 17 00:00:00 2001 From: macgyver13 <> Date: Tue, 11 Mar 2025 11:36:57 -0400 Subject: [PATCH 2/5] Resolve ruff formatting issues --- test/graph_test.py | 17 +++++++------- test/local_test_runner.py | 47 +++++++++++++++++++++++++-------------- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/test/graph_test.py b/test/graph_test.py index 9fec27993..22c560c4c 100755 --- a/test/graph_test.py +++ b/test/graph_test.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 import json -import os, time +import os +import time import pexpect from test_base import TestBase @@ -39,25 +40,25 @@ def directory_exists(self): self.sut = pexpect.spawn("warnet create") self.sut.expect("name", timeout=10) self.sut.sendline("ANewNetwork") - time.sleep(.1) + time.sleep(0.1) self.sut.expect("many", timeout=10) self.sut.sendline("") - time.sleep(.1) + time.sleep(0.1) self.sut.expect("connections", timeout=10) self.sut.sendline("") - time.sleep(.1) + time.sleep(0.1) self.sut.expect("version", timeout=10) self.sut.sendline("") - time.sleep(.1) + time.sleep(0.1) self.sut.expect("enable fork-observer", timeout=10) self.sut.sendline("") - time.sleep(.1) + time.sleep(0.1) self.sut.expect("seconds", timeout=10) self.sut.sendline("") - time.sleep(.1) + time.sleep(0.1) self.sut.expect("enable grafana", timeout=10) self.sut.sendline("") - time.sleep(.1) + time.sleep(0.1) self.sut.expect("successfully", timeout=50) except Exception as e: print(f"\nReceived prompt text:\n {self.sut.before.decode('utf-8')}\n") diff --git a/test/local_test_runner.py b/test/local_test_runner.py index 2055047b8..e585fe977 100644 --- a/test/local_test_runner.py +++ b/test/local_test_runner.py @@ -1,5 +1,7 @@ +import subprocess +import sys + import yaml -import subprocess, sys # developers can use this tool to verify all repo workflow test execute to completion # @@ -19,38 +21,49 @@ def has_key_path(d, key_path, separator="."): d = d[key] return True + # Load the workflow file -with open('.github/workflows/test.yml', 'r') as file: +with open(".github/workflows/test.yml") as file: workflow = yaml.safe_load(file) -tests_total=0 -tests_skipped=0 -tests_completed=0 +tests_total = 0 +tests_skipped = 0 +tests_completed = 0 -for job_name, job_details in workflow.get("jobs", {}).items(): +for job_details in workflow.get("jobs", {}).values(): if has_key_path(job_details, "strategy.matrix.test"): print("Found test strategy job, starting serial execution of each test") tests = job_details["strategy"]["matrix"]["test"] - + for test in tests: - tests_total+=1 + tests_total += 1 if len(sys.argv) > 1 and sys.argv[1] not in test: - print("skipping test as requested:",test) - tests_skipped+=1 + print("skipping test as requested:", test) + tests_skipped += 1 continue command = f"python test/{test}" - print("###################################################################################################") + print( + "###################################################################################################" + ) print("############## executing:", command) - print("###################################################################################################") + print( + "###################################################################################################" + ) process = subprocess.run(command, shell=True) if process.returncode != 0: print("******** testing failed") - if process.stdout: print("stdout:", process.stdout) - if process.stderr: print("stderr:", process.stderr) + if process.stdout: + print("stdout:", process.stdout) + if process.stderr: + print("stderr:", process.stderr) sys.exit(1) - tests_completed+=1 + tests_completed += 1 -print("###################################################################################################") +print( + "###################################################################################################" +) print("testing complete") print(f"{tests_completed} of {tests_total} complete - skipped: {tests_skipped} tests") -print("###################################################################################################") \ No newline at end of file +print( + "###################################################################################################" +) From 43e28a81e02a92a1f0a396534815608a756dc4f1 Mon Sep 17 00:00:00 2001 From: macgyver13 <4712150+macgyver13@users.noreply.github.com> Date: Mon, 24 Mar 2025 15:21:00 -0400 Subject: [PATCH 3/5] reverse graph_test change did not improve stability make local_test_runner executable --- test/graph_test.py | 7 ------- test/local_test_runner.py | 1 + 2 files changed, 1 insertion(+), 7 deletions(-) mode change 100644 => 100755 test/local_test_runner.py diff --git a/test/graph_test.py b/test/graph_test.py index 22c560c4c..898076e3f 100755 --- a/test/graph_test.py +++ b/test/graph_test.py @@ -40,25 +40,18 @@ def directory_exists(self): self.sut = pexpect.spawn("warnet create") self.sut.expect("name", timeout=10) self.sut.sendline("ANewNetwork") - time.sleep(0.1) self.sut.expect("many", timeout=10) self.sut.sendline("") - time.sleep(0.1) self.sut.expect("connections", timeout=10) self.sut.sendline("") - time.sleep(0.1) self.sut.expect("version", timeout=10) self.sut.sendline("") - time.sleep(0.1) self.sut.expect("enable fork-observer", timeout=10) self.sut.sendline("") - time.sleep(0.1) self.sut.expect("seconds", timeout=10) self.sut.sendline("") - time.sleep(0.1) self.sut.expect("enable grafana", timeout=10) self.sut.sendline("") - time.sleep(0.1) self.sut.expect("successfully", timeout=50) except Exception as e: print(f"\nReceived prompt text:\n {self.sut.before.decode('utf-8')}\n") diff --git a/test/local_test_runner.py b/test/local_test_runner.py old mode 100644 new mode 100755 index e585fe977..0fe744ed2 --- a/test/local_test_runner.py +++ b/test/local_test_runner.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python import subprocess import sys From 996a5e1907d62eafbbbeeb146b44968e8b323cee Mon Sep 17 00:00:00 2001 From: macgyver13 <4712150+macgyver13@users.noreply.github.com> Date: Mon, 24 Mar 2025 15:22:55 -0400 Subject: [PATCH 4/5] remove unused import --- test/graph_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/graph_test.py b/test/graph_test.py index 898076e3f..9bd8f5310 100755 --- a/test/graph_test.py +++ b/test/graph_test.py @@ -2,7 +2,6 @@ import json import os -import time import pexpect from test_base import TestBase From 06f167a57b3d8e79ad7b3339f6c3a529fa8927f8 Mon Sep 17 00:00:00 2001 From: macgyver13 <4712150+macgyver13@users.noreply.github.com> Date: Mon, 24 Mar 2025 15:24:34 -0400 Subject: [PATCH 5/5] remove added print --- test/graph_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/graph_test.py b/test/graph_test.py index 9bd8f5310..3d0ad5848 100755 --- a/test/graph_test.py +++ b/test/graph_test.py @@ -54,7 +54,6 @@ def directory_exists(self): self.sut.expect("successfully", timeout=50) except Exception as e: print(f"\nReceived prompt text:\n {self.sut.before.decode('utf-8')}\n") - print(os.listdir(self.tmpdir)) raise e def run_created_network(self):