refactor: rename run_loop to run_multiple for clarity and consistency

Author: paulz

docs/local-development.md

@@ -184,7 +184,7 @@ def test_allocations():
         run_allocation_test,
         reporter=test_reporter,
     )
-    results = test_runner.run_loop(tries)
+    results = test_runner.run_multiple(tries)
     assert False not in results
 
 
@@ -337,7 +337,7 @@ def test_allocations():
         ),
         reporter=test_reporter,
     )
-    results = test_runner.run_loop(tries)
+    results = test_runner.run_multiple(tries)
     assert False not in results
 
 

examples/team_recommender/tests/example_2_loop/test_allocations_loop.py

@@ -43,7 +43,7 @@ def test_allocations():
         run_allocation_test,
         reporter=test_reporter,
     )
-    results = test_runner.run_loop(tries)
+    results = test_runner.run_multiple(tries)
     assert False not in results
 
 

examples/team_recommender/tests/example_3_loop_no_hallucinating/test_allocations_hallucinating.py

@@ -51,7 +51,7 @@ def test_allocations():
         lambda reporter: run_allocation_test(reporter=reporter, skills_data=skills_data),
         reporter=test_reporter,
     )
-    results = test_runner.run_loop(tries)
+    results = test_runner.run_multiple(tries)
     assert False not in results
 
 

examples/team_recommender/tests/example_4_gate_on_success_threshold/test_allocations_threshold.py

@@ -1,9 +1,11 @@
 import json
 import os
+
+from openai import OpenAI
+from tests.settings import ROOT_DIR
+
 from cat_ai.reporter import Reporter
 from cat_ai.runner import Runner
-from tests.settings import ROOT_DIR
-from openai import OpenAI
 
 
 def get_all_developer_names(skills_data) -> set[str]:
@@ -26,7 +28,7 @@ def has_expected_success_rate(results: list[bool], expected_success_rate: float)
 
 
 def test_allocations():
-    tries = Runner.sample_size(3)
+    tries = Runner.get_sample_size(3)
     skills_json_path = os.path.join(ROOT_DIR, "fixtures", "skills.json")
     with open(skills_json_path, "r") as file:
         skills_data = json.load(file)
@@ -62,7 +64,7 @@ def test_allocations():
         lambda reporter: run_allocation_test(reporter=reporter, skills_data=skills_data),
         reporter=test_reporter,
     )
-    results = test_runner.run_loop(tries)
+    results = test_runner.run_multiple(tries)
     failure_threshold = 0.8
     assert has_expected_success_rate(results, failure_threshold)
 

src/cat_ai/runner.py

@@ -7,7 +7,7 @@
 class Runner:
     """Executes test functions and collects results using a reporter."""
 
-    def __init__(self, test_function: Callable[..., bool], reporter: Reporter) -> None:
+    def __init__(self, test_function: Callable[[Reporter], bool], reporter: Reporter) -> None:
         """
         Initialize the Runner with a test function and reporter.
 
@@ -42,9 +42,9 @@ def run_once(self, run_number: int = 0) -> bool:
             Result from the test function
         """
         self.reporter.run_number = run_number
-        return self.test_function(reporter=self.reporter)
+        return self.test_function(self.reporter)
 
-    def run(self, sample_size: Optional[int] = None) -> List[bool]:
+    def run_multiple(self, sample_size: Optional[int] = None) -> List[bool]:
         """
         Execute the test function multiple times based on sample size.
 
@@ -55,5 +55,5 @@ def run(self, sample_size: Optional[int] = None) -> List[bool]:
         Returns:
             List of results from all test runs
         """
-        runs = sample_size or self.get_sample_size()
+        runs = sample_size if sample_size is not None else self.get_sample_size()
         return [self.run_once(i) for i in range(runs)]

tests/cat_ai/test_runner.py

@@ -3,9 +3,10 @@
 
 
 # Dummy test function that will be passed to Runner
-def dummy_test_function(reporter: Reporter):
+def dummy_test_function(reporter: Reporter) -> bool:
     # Imagine that this function does something meaningful
-    return f"Running test with run number {reporter.run_number}"
+    # Simply returning True instead of trying to log
+    return True
 
 
 def test_runner_sample_size(monkeypatch):
@@ -27,7 +28,7 @@ def test_run_once():
 
     # Test run_once
     result = runner.run_once(run_number=1)
-    assert result == "Running test with run number 1"
+    assert result is True
     assert reporter.run_number == 1
 
 
@@ -39,14 +40,10 @@ def test_run():
     runner = Runner(test_function=dummy_test_function, reporter=reporter)
 
     # Test with explicit sample size parameter
-    results = runner.run(sample_size=3)
+    results = runner.run_multiple(sample_size=3)
     assert len(results) == 3
-    assert all(res.startswith("Running test with run number ") for res in results)
-    expected_results = [
-        "Running test with run number 0",
-        "Running test with run number 1",
-        "Running test with run number 2",
-    ]
+    assert all(results)
+    expected_results = [True, True, True]
     assert results == expected_results
 
 
@@ -61,11 +58,7 @@ def test_run_with_env_variable(monkeypatch):
     runner = Runner(test_function=dummy_test_function, reporter=reporter)
 
     # Test without explicit sample size (should use environment variable)
-    results = runner.run()
+    results = runner.run_multiple()
     assert len(results) == 3
-    expected_results = [
-        "Running test with run number 0",
-        "Running test with run number 1",
-        "Running test with run number 2",
-    ]
+    expected_results = [True, True, True]
     assert results == expected_results