Merge pull request #45 from codebeaver-ai/rc

Release candidate 0.1.3

Author: erlapso

.coveragerc

@@ -1,3 +1,10 @@
+## [0.1.2] - 2025-03-26
+
+### Added
+
+- Added a new command `codebeaver e2e` to run end-to-end tests defined in configuration
+- Added XML report generation for end-to-end tests
+
 ## [0.1.0] - 2024-03-10
 
 ### Added

CHANGELOG.md

@@ -166,11 +166,11 @@ CodeBeaver currently supports:
 
 - [✅] Unit tests
 - [✅] E2E Tests
+- [✅] Add support for more models (thank you [VinciGit00](https://github.com/VinciGit00)!)
 - [ ] Better reporting
 - [ ] Integration Tests
 - [ ] Unit Tests: Add support for more languages and frameworks
 - [ ] Unit Tests: Add support for more testing frameworks
-- [ ] Add support for more models
 
 ## Let's chat!
 

README.md

@@ -1,7 +1,8 @@
 from codebeaver.types import TestErrorType
-import openai
+import os
 from pathlib import Path
 import logging
+from .models.provider_factory import ProviderFactory, ProviderType
 
 logger = logging.getLogger("codebeaver")
 
@@ -40,6 +41,8 @@ def __init__(
 ---ERROR LOG---
 {error}
 """
+        provider_type = os.getenv("CODEBEAVER_PROVIDER", "openai")
+        self.provider = ProviderFactory.get_provider(ProviderType(provider_type))
 
     def analyze(self) -> tuple[TestErrorType, str | None]:
         """
@@ -51,8 +54,7 @@ def analyze(self) -> tuple[TestErrorType, str | None]:
         if self.error == "exit status 1":
             return TestErrorType.TEST, self.error
 
-        response = openai.chat.completions.create(
-            model="o3-mini",
+        response = self.provider.create_chat_completion(
             messages=[
                 {
                     "role": "user",

coverage.json

@@ -5,34 +5,19 @@
 from browser_use.browser.context import BrowserContext
 from dotenv import load_dotenv
 from langchain_openai import ChatOpenAI
-from pydantic import BaseModel
+
 from .GitUtils import GitUtils
 import logging
 from browser_use.browser.context import BrowserContextConfig
 from pathlib import Path
+from .types import End2endTest, TestCase
+from .Report import Report
 
 load_dotenv()
 
 logger = logging.getLogger("codebeaver")
 
 
-class End2endTest(BaseModel):
-    steps: list[str]
-    url: str
-    passed: bool = False
-    errored: bool = False
-    comment: str = ""
-    name: str
-
-    def __init__(self, name: str, steps: list[str], url: str):
-        super().__init__(name=name, steps=steps, url=url)
-
-
-class TestCase(BaseModel):
-    passed: bool
-    comment: str
-
-
 controller = Controller(output_model=TestCase)
 
 
@@ -61,13 +46,20 @@ async def run(self) -> list[End2endTest]:
                 url=test["url"],
             )
             test_result = await self.run_test(test)
-            all_tests.append(test_result)
+            test.passed = not test_result.failure
+            test.errored = test_result.errored
+            test.comment = test_result.comment
+            all_tests.append(test)
         # write the results to e2e.json. this is temporary, we will eventually use the report class
         with open(Path.cwd() / ".codebeaver/e2e.json", "w") as f:
             json.dump([test.model_dump() for test in all_tests], f)
+        report = Report()
+        report.add_e2e_results(all_tests)
+        with open(Path.cwd() / ".codebeaver/e2e.xml", "w") as f:
+            f.write(report.generate_xml_report())
         return all_tests
 
-    async def run_test(self, test: End2endTest) -> End2endTest:
+    async def run_test(self, test: End2endTest) -> TestCase:
         GitUtils.ensure_codebeaver_folder_exists_and_in_gitignore()  # avoid committing logs, screenshots and so on
         config_context = BrowserContextConfig(
             save_recording_path=Path.cwd() / ".codebeaver/",
@@ -81,24 +73,22 @@ async def run_test(self, test: End2endTest) -> End2endTest:
         )
         context = BrowserContext(browser=browser, config=config_context)
         agent = Agent(
-            task=f"""You are a QA tester. Follow these steps:
+            task=f"""You are a QA tester. Follow these instructions to perform the test called {test.name}:
 * Go to {test.url}
 """
-            + "\n".join(f"* {step}" for step in test.steps),
+            + "\n".join(f"* {step}" for step in test.steps)
+            + "\n\nIf any step that starts with 'Check' fails, the result is a failure",
             llm=ChatOpenAI(model="gpt-4o"),
-            # browser=browser,
             controller=controller,
             browser_context=context,
         )
         history = await agent.run()
         await context.close()
         result = history.final_result()
         if result:
-            parsed: TestCase = TestCase.model_validate_json(result)
-            test.passed = parsed.passed
-            test.comment = parsed.comment
-            return test
+            test_result: TestCase = TestCase.model_validate_json(result)
+            return test_result
         else:
-            test.errored = True
-            test.comment = "No result from the test"
-            return test
+            test_result.errored = True
+            test_result.comment = "No result from the test"
+            return test_result

src/codebeaver/AnalyzeError.py

@@ -0,0 +1,69 @@
+from .types import End2endTest
+
+
+class Report:
+    """
+    A class that generates a report of the test results. For now, only used for E2E test results, but in the future it will produce one unique report if it will be the case.
+    """
+
+    def __init__(self) -> None:
+        self.e2e_results: list[End2endTest] = []
+
+    def add_e2e_results(self, e2e_results: list[End2endTest]) -> None:
+        self.e2e_results.extend(e2e_results)
+
+    def generate_xml_report(self) -> str:
+        """
+        Generate a XML report of the test results, in a format that is compatible with junit.xml
+        """
+        xml_lines = ['<?xml version="1.0" encoding="UTF-8"?>']
+        xml_lines.append('<testsuites name="End2End Tests">')
+
+        # Create a single test suite for all E2E tests
+        xml_lines.append(
+            '  <testsuite name="E2E Test Suite" tests="{}"'.format(
+                len(self.e2e_results)
+            )
+        )
+
+        # Count failures and errors
+        failures = sum(
+            1 for test in self.e2e_results if not test.passed and not test.errored
+        )
+        errors = sum(1 for test in self.e2e_results if test.errored)
+        xml_lines.append(f'    failures="{failures}" errors="{errors}">')
+
+        # Add individual test cases
+        for test in self.e2e_results:
+            xml_lines.append(
+                '    <testcase name="{}" classname="E2ETest">'.format(test.name)
+            )
+
+            # Add steps as system-out
+            steps_text = "\n".join(test.steps)
+            xml_lines.append(f"      <system-out>{steps_text}</system-out>")
+
+            # Add failure or error information if present
+            if test.errored:
+                xml_lines.append(
+                    '      <error message="Test execution error" type="Error">'
+                )
+                xml_lines.append(f"        {test.comment}")
+                xml_lines.append("      </error>")
+            elif not test.passed:
+                xml_lines.append('      <failure message="Test failed" type="Failure">')
+                xml_lines.append(f"        {test.comment}")
+                xml_lines.append("      </failure>")
+
+            xml_lines.append("    </testcase>")
+
+        xml_lines.append("  </testsuite>")
+        xml_lines.append("</testsuites>")
+
+        return "\n".join(xml_lines)
+
+    def generate_html_report(self) -> str:
+        raise NotImplementedError("HTML report generation not implemented")
+
+    def generate_json_report(self) -> str:
+        raise NotImplementedError("JSON report generation not implemented")

src/codebeaver/E2E.py

@@ -3,7 +3,7 @@
 class ResponseParser:
     @staticmethod
     def parse(response: str) -> str:
-        test_regex_match = re.findall(r"```test\n(.*?)```", response, re.DOTALL)
+        test_regex_match = re.findall(r"<test>\s*\[test\](.*?)</test>", response, re.DOTALL)
         test_content = test_regex_match[0] if test_regex_match else ""
-        test_content = test_content.replace("```test", "").replace("``test", "")
+        test_content = test_content.strip()
         return test_content