Merge pull request #23 from anurag03/ansinha/ai_agent_debug

Ansinha/ai agent debug

Author: anurag03

requirements.txt

@@ -1 +1,4 @@
+
 pytest
+requests
+pyyaml

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name="testcato",
-    version="1.2.0",
+    version="1.2.1",
     packages=find_packages(),
     description="A package for categorizing test results.",
     long_description=long_description,

testcato/__init__.py

@@ -1,7 +1,7 @@
 
 # Package metadata for wheel
 __title__ = "testcato"
-__version__ = "1.2.0"
+__version__ = "1.2.1"
 
 # Automatically create config file if not present
 import os

testcato/ai_agent.py

@@ -0,0 +1,85 @@
+import os
+import glob
+import xml.etree.ElementTree as ET
+import datetime
+import requests
+import yaml
+
+def get_latest_xml(result_dir):
+    files = glob.glob(os.path.join(result_dir, 'test_run_*.xml'))
+    if not files:
+        return None
+    return max(files, key=os.path.getctime)
+
+def load_agent_config(config_path):
+    with open(config_path, 'r') as f:
+        config = yaml.safe_load(f)
+    if not isinstance(config, dict):
+        return None
+    default_agent = config.get('default', 'agent1')
+    agent = config.get(default_agent, {})
+    # Check if agent config is empty or missing required fields
+    if not agent or not agent.get('api_key') or not agent.get('api_url'):
+        return None
+    return agent
+
+def send_to_ai_agent(agent, test_name, traceback):
+    api_url = agent.get('api_url')
+    if not api_url:
+        return "No api_url provided in agent config."
+    headers = {
+        'Authorization': f"Bearer {agent.get('api_key', '')}",
+        'Content-Type': 'application/json'
+    }
+    # The payload structure should be defined by the user in their config and usage
+    payload = {
+        "test_name": test_name,
+        "traceback": traceback
+    }
+    response = requests.post(api_url, json=payload, headers=headers)
+    if response.ok:
+        return response.text
+    return "AI agent failed to respond."
+
+def debug_latest_xml():
+    result_dir = os.path.join(os.getcwd(), 'testcato_result')
+    config_path = os.path.join(os.getcwd(), 'testcato_config.yaml')
+    latest_xml = get_latest_xml(result_dir)
+    if not latest_xml:
+        print("No test_run XML file found.")
+        return
+    agent = load_agent_config(config_path)
+    if not agent:
+        # Print warning in yellow in pytest output or console
+        YELLOW = '\033[33m'
+        RESET = '\033[0m'
+        warning_msg = f"{YELLOW}WARNING: TESTCATO: No valid AI agent config found. Debugging is disabled.{RESET}"
+        import sys
+        tr = getattr(sys, '_pytest_terminalreporter', None)
+        if tr:
+            tr.write_line(warning_msg)
+        else:
+            print(warning_msg)
+        return
+    import json
+    tree = ET.parse(latest_xml)
+    root = tree.getroot()
+    lines = []
+    for test_elem in root.findall('Test'):
+        name = test_elem.get('name')
+        tb_elem = test_elem.find('Traceback')
+        debug_result = None
+        if tb_elem is not None:
+            debug_result = send_to_ai_agent(agent, name, tb_elem.text)
+        line = {
+            "name": name,
+            "traceback": tb_elem.text if tb_elem is not None else None,
+            "debug": debug_result
+        }
+        lines.append(line)
+    timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+    debug_jsonl_path = os.path.join(result_dir, f'test_debugg_{timestamp}.jsonl')
+    with open(debug_jsonl_path, 'w', encoding='utf-8') as f:
+        for line in lines:
+            f.write(json.dumps(line, ensure_ascii=False) + '\n')
+    print(f"Debug results saved to {debug_jsonl_path}")

testcato/pytest_testcato.py

@@ -1,3 +1,4 @@
+import re
 import pytest
 import os
 import datetime
@@ -20,6 +21,9 @@ def pytest_configure(config):
             config.option.verbose = 3
 
 def pytest_terminal_summary(terminalreporter, exitstatus, config):
+    def remove_ansi_codes(text):
+        ansi_escape = re.compile(r'\x1B[@-_][0-?]*[ -/]*[@-~]')
+        return ansi_escape.sub('', text)
     if config.getoption("--testcato"):
         results = []
         tracebacks = []
@@ -28,25 +32,22 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config):
         for report in terminalreporter.getreports("failed"):
             results.append({'name': report.nodeid, 'status': 'failed'})
             if hasattr(report, 'longrepr') and report.longrepr:
-                tb = str(report.longrepr)
+                tb = remove_ansi_codes(str(report.longrepr))
                 tracebacks.append({'name': report.nodeid, 'traceback': tb})
         for report in terminalreporter.getreports("skipped"):
             results.append({'name': report.nodeid, 'status': 'skipped'})
 
-        # Save tracebacks to XML
+        # Save tracebacks to JSON Lines
         if tracebacks:
+            import json
             result_dir = os.path.join(os.getcwd(), 'testcato_result')
             os.makedirs(result_dir, exist_ok=True)
-            root = ET.Element('TestTracebacks')
-            for tb in tracebacks:
-                test_elem = ET.SubElement(root, 'Test', name=tb['name'])
-                tb_elem = ET.SubElement(test_elem, 'Traceback')
-                tb_elem.text = tb['traceback']
-            tree = ET.ElementTree(root)
             timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
-            xml_path = os.path.join(result_dir, f'test_run_{timestamp}.xml')
-            tree.write(xml_path, encoding='utf-8', xml_declaration=True)
-            terminalreporter.write_line(f"Tracebacks saved to {xml_path}")
+            jsonl_path = os.path.join(result_dir, f'test_run_{timestamp}.jsonl')
+            with open(jsonl_path, 'w', encoding='utf-8') as f:
+                for tb in tracebacks:
+                    f.write(json.dumps(tb, ensure_ascii=False) + '\n')
+            terminalreporter.write_line(f"Tracebacks saved to {jsonl_path}")
 
         categorizer = TestCategorizer()
         categories = categorizer.categorize(results)
@@ -55,3 +56,10 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config):
             terminalreporter.write_line(f"{category}:")
             for test in tests:
                 terminalreporter.write_line(f"  {test}")
+
+        # Automatically send latest XML to AI agent and save debug file
+        try:
+            from .ai_agent import debug_latest_xml
+            debug_latest_xml()
+        except Exception as e:
+            terminalreporter.write_line(f"Error sending XML to AI agent: {e}")

testcato_config.yaml

@@ -7,13 +7,14 @@
 #
 # agent1:
 #   type: openai
-#   model: gpt-4
-#   api_key: YOUR_OPENAI_API_KEY
+#   model: gpt-4.1-mini
+#   api_key: <YOUR_OPENAI_API_KEY>
+#   api_url: https://api.openai.com/v1/chat/completions
 #
 # Add more agents below as agent2, agent3, etc.
 # agent2:
 #   type: azure
 #   model: gpt-4
 #   api_key: YOUR_AZURE_API_KEY
-
+#   api_url: https://your.azure.endpoint/api