Pull test runners into classes

Author: JamesVorder

TestRunner/GenericTestRunner.py

@@ -0,0 +1,47 @@
+from abc import ABCMeta, abstractmethod
+
+import pytest
+
+from pytest_plugins import ResultsCollector, SessionStartPlugin
+
+
+class GenericTestRunner(metaclass=ABCMeta):
+    @abstractmethod
+    def run(self, *args, **kwargs) -> (int, str): pass
+
+import subprocess
+
+
+class SubProcessTestRunner(GenericTestRunner):
+    code_dir: str
+    test_dir: str
+
+    def __init__(self, _code, _test) -> None:
+        self.code_dir = _code
+        self.test_dir = _test
+
+    def run(self, *args, **kwargs) -> (int, str):
+        # TODO: check that code_dir and test_dir exist
+        proc = subprocess.run(["pytest", self.test_dir], capture_output=True)
+        return proc.returncode, proc.stdout
+
+
+class InlineTestRunner(GenericTestRunner):
+
+    def __init__(self, _code, _test) -> None:
+        self.code_dir = _code
+        self.test_dir = _test
+
+    def run(self, *args, **kwargs) -> (int, str):
+        collector = ResultsCollector()
+        setup = SessionStartPlugin()
+        pytest.main(args=["-k", "ExampleClass"], plugins=[collector, setup])
+        _out = ""
+
+        if collector.exitcode > 0:
+            for report in collector.reports:
+                _out += f"{report.outcome.upper()} {report.nodeid} ... Outcome: - {report.longrepr.reprcrash.message}"
+                _out += "\n"
+                _out += report.longreprtext
+                _out += "\n"
+        return collector.exitcode, _out

TestRunner/SubProcessTestRunner.py

@@ -0,0 +1 @@
+from . import GenericTestRunner

TestRunner/__init__.py

@@ -1,15 +1,11 @@
 import os
 import typer
 
-# for the test runner
-import pytest
-# ------------------
-
 import langroid as lr
 from langroid.utils.configuration import set_global, Settings
 from langroid.utils.logging import setup_colored_logging
 
-from pytest_plugins import ResultsCollector, SessionStartPlugin
+from TestRunner.GenericTestRunner import InlineTestRunner
 
 app = typer.Typer()
 setup_colored_logging()
@@ -40,21 +36,6 @@ def generate_first_attempt() -> None:
         _out.write(response.content)
 
 
-def get_test_results() -> str:
-    collector = ResultsCollector()
-    setup = SessionStartPlugin()
-    pytest.main(args=["-k", "ExampleClass"], plugins=[collector, setup])
-    _out = ""
-
-    if collector.exitcode > 0:
-        for report in collector.reports:
-            _out += f"{report.outcome.upper()} {report.nodeid} ... Outcome: - {report.longrepr.reprcrash.message}"
-            _out += "\n"
-            _out += report.longreprtext
-            _out += "\n"
-    return collector.exitcode, _out
-
-
 def generate_next_attempt(test_results: str) -> None:
     cfg = lr.ChatAgentConfig(
         llm=lr.language_models.OpenAIGPTConfig(
@@ -100,8 +81,11 @@ def teardown() -> None:
 
 def chat() -> None:
     generate_first_attempt()
+    test_runner = InlineTestRunner("", os.path.join(".", "test"))
     for _ in range(5):
-        test_exit_code, test_results = get_test_results()
+        # test_exit_code, test_results = get_test_results()
+        test_exit_code, test_results = test_runner.run()
+        print(test_results)
         if test_exit_code == 1:
             generate_next_attempt(test_results)
         else: