|
1 | 1 | import os |
2 | 2 | import glob |
3 | | -## import xml.etree.ElementTree as ET # No longer needed, switched to JSONL |
| 3 | + |
| 4 | +# import xml.etree.ElementTree as ET # No longer needed, switched to JSONL |
4 | 5 | import datetime |
5 | 6 | import requests |
6 | 7 | import yaml |
7 | 8 |
|
| 9 | + |
8 | 10 | def get_latest_jsonl(result_dir): |
9 | | - files = glob.glob(os.path.join(result_dir, 'test_run_*.jsonl')) |
| 11 | + files = glob.glob(os.path.join(result_dir, "test_run_*.jsonl")) |
10 | 12 | if not files: |
11 | 13 | return None |
12 | 14 | return max(files, key=os.path.getctime) |
13 | 15 |
|
| 16 | + |
14 | 17 | def load_agent_config(config_path): |
15 | | - with open(config_path, 'r') as f: |
| 18 | + with open(config_path, "r") as f: |
16 | 19 | config = yaml.safe_load(f) |
17 | 20 | if not isinstance(config, dict): |
18 | 21 | return None |
19 | | - default_agent = config.get('default', 'agent1') |
| 22 | + default_agent = config.get("default", "agent1") |
20 | 23 | agent = config.get(default_agent, {}) |
21 | 24 | # Check if agent config is empty or missing required fields |
22 | | - if not agent or not agent.get('api_key') or not agent.get('api_url'): |
| 25 | + if not agent or not agent.get("api_key") or not agent.get("api_url"): |
23 | 26 | return None |
24 | 27 | return agent |
25 | 28 |
|
| 29 | + |
26 | 30 | def send_to_ai_agent(agent, test_name, traceback): |
27 | | - api_url = agent.get('api_url') |
| 31 | + api_url = agent.get("api_url") |
28 | 32 | if not api_url: |
29 | 33 | return "No api_url provided in agent config." |
30 | 34 | headers = { |
31 | | - 'Authorization': f"Bearer {agent.get('api_key', '')}", |
32 | | - 'Content-Type': 'application/json' |
| 35 | + "Authorization": f"Bearer {agent.get('api_key', '')}", |
| 36 | + "Content-Type": "application/json", |
33 | 37 | } |
34 | 38 | # Payload for OpenAI chat completions API |
35 | 39 | payload = { |
36 | | - "model": agent.get('model', 'gpt-4'), |
| 40 | + "model": agent.get("model", "gpt-4"), |
37 | 41 | "messages": [ |
38 | | - {"role": "system", "content": "You are a helpful test debugging assistant."}, |
39 | | - {"role": "user", "content": f"Debug this test failure: {test_name}\nTraceback:\n{traceback}"} |
40 | | - ] |
| 42 | + { |
| 43 | + "role": "system", |
| 44 | + "content": "You are a helpful test debugging assistant.", |
| 45 | + }, |
| 46 | + { |
| 47 | + "role": "user", |
| 48 | + "content": f"Debug this test failure: {test_name}\nTraceback:\n{traceback}", |
| 49 | + }, |
| 50 | + ], |
41 | 51 | } |
42 | 52 | response = requests.post(api_url, json=payload, headers=headers) |
43 | 53 | if response.ok: |
44 | 54 | try: |
45 | | - return response.json().get('choices', [{}])[0].get('message', {}).get('content', '') |
| 55 | + return ( |
| 56 | + response.json() |
| 57 | + .get("choices", [{}])[0] |
| 58 | + .get("message", {}) |
| 59 | + .get("content", "") |
| 60 | + ) |
46 | 61 | except Exception: |
47 | 62 | return response.text |
48 | 63 | # Print error details in red to CLI for visibility |
49 | | - RED = '\033[31m' |
50 | | - RESET = '\033[0m' |
51 | | - print(f"{RED}TESTCATO AI agent error for test '{test_name}': {response.status_code} - {response.text}{RESET}") |
| 64 | + RED = "\033[31m" |
| 65 | + RESET = "\033[0m" |
| 66 | + print( |
| 67 | + f"{RED}TESTCATO AI agent error for test '{test_name}': {response.status_code} - {response.text}{RESET}" |
| 68 | + ) |
52 | 69 | return "AI agent failed to respond." |
53 | 70 |
|
| 71 | + |
54 | 72 | def debug_latest_jsonl(): |
55 | 73 | # Generate debug JSONL and HTML report after lines, result_dir, and timestamp are defined |
56 | | - result_dir = os.path.join(os.getcwd(), 'testcato_result') |
57 | | - config_path = os.path.join(os.getcwd(), 'testcato_config.yaml') |
| 74 | + result_dir = os.path.join(os.getcwd(), "testcato_result") |
| 75 | + config_path = os.path.join(os.getcwd(), "testcato_config.yaml") |
58 | 76 | latest_jsonl = get_latest_jsonl(result_dir) |
59 | 77 | if not latest_jsonl: |
60 | 78 | print("No test_run JSONL file found.") |
61 | 79 | return |
62 | 80 | agent = load_agent_config(config_path) |
63 | 81 | if not agent: |
64 | 82 | # Print warning in yellow in pytest output or console |
65 | | - YELLOW = '\033[33m' |
66 | | - RESET = '\033[0m' |
| 83 | + YELLOW = "\033[33m" |
| 84 | + RESET = "\033[0m" |
67 | 85 | warning_msg = f"{YELLOW}WARNING: TESTCATO: No valid AI agent config found. Debugging is disabled.{RESET}" |
68 | 86 | import sys |
69 | | - tr = getattr(sys, '_pytest_terminalreporter', None) |
| 87 | + |
| 88 | + tr = getattr(sys, "_pytest_terminalreporter", None) |
70 | 89 | if tr: |
71 | 90 | tr.write_line(warning_msg) |
72 | 91 | else: |
73 | 92 | print(warning_msg) |
74 | 93 | return |
75 | 94 | import json |
| 95 | + |
76 | 96 | lines = [] |
77 | | - with open(latest_jsonl, 'r', encoding='utf-8') as f: |
| 97 | + with open(latest_jsonl, "r", encoding="utf-8") as f: |
78 | 98 | for raw_line in f: |
79 | 99 | try: |
80 | 100 | test_data = json.loads(raw_line) |
81 | 101 | except Exception: |
82 | 102 | continue |
83 | | - test_name = test_data.get('name') or test_data.get('test_name') |
84 | | - status = test_data.get('status', 'failed') |
85 | | - traceback = test_data.get('traceback') |
| 103 | + test_name = test_data.get("name") or test_data.get("test_name") |
| 104 | + status = test_data.get("status", "failed") |
| 105 | + traceback = test_data.get("traceback") |
86 | 106 | debug_result = None |
87 | 107 | if traceback: |
88 | 108 | debug_result = send_to_ai_agent(agent, test_name, traceback) |
89 | 109 | line = { |
90 | 110 | "test_name": test_name, |
91 | 111 | "status": status, |
92 | 112 | "traceback": traceback, |
93 | | - "debug_result": debug_result |
| 113 | + "debug_result": debug_result, |
94 | 114 | } |
95 | 115 | lines.append(line) |
96 | | - timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') |
97 | | - debug_jsonl_path = os.path.join(result_dir, f'test_debug_{timestamp}.jsonl') |
98 | | - with open(debug_jsonl_path, 'w', encoding='utf-8') as f: |
| 116 | + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") |
| 117 | + debug_jsonl_path = os.path.join(result_dir, f"test_debug_{timestamp}.jsonl") |
| 118 | + with open(debug_jsonl_path, "w", encoding="utf-8") as f: |
99 | 119 | for line in lines: |
100 | | - f.write(json.dumps(line, ensure_ascii=False) + '\n') |
101 | | - GREEN = '\033[32m' |
102 | | - RESET = '\033[0m' |
| 120 | + f.write(json.dumps(line, ensure_ascii=False) + "\n") |
| 121 | + GREEN = "\033[32m" |
| 122 | + RESET = "\033[0m" |
103 | 123 | print(f"{GREEN}Debug results saved to {debug_jsonl_path}{RESET}") |
104 | 124 |
|
105 | 125 | # Also generate a human-readable HTML report |
106 | | - html_path = os.path.join(result_dir, f'test_debug_{timestamp}.html') |
107 | | - with open(html_path, 'w', encoding='utf-8') as html: |
108 | | - html.write('<html><head><title>Test Debug Report</title></head><body>') |
109 | | - html.write('<h1>Test Debug Report</h1>') |
| 126 | + html_path = os.path.join(result_dir, f"test_debug_{timestamp}.html") |
| 127 | + with open(html_path, "w", encoding="utf-8") as html: |
| 128 | + html.write("<html><head><title>Test Debug Report</title></head><body>") |
| 129 | + html.write("<h1>Test Debug Report</h1>") |
110 | 130 | html.write('<table border="1" cellpadding="5" cellspacing="0">') |
111 | | - html.write('<tr><th>Test Name</th><th>Status</th><th>Traceback</th><th>Debug Result</th></tr>') |
| 131 | + html.write( |
| 132 | + "<tr><th>Test Name</th><th>Status</th><th>Traceback</th><th>Debug Result</th></tr>" |
| 133 | + ) |
112 | 134 | for line in lines: |
113 | | - html.write('<tr>') |
| 135 | + html.write("<tr>") |
114 | 136 | html.write(f'<td>{line["test_name"]}</td>') |
115 | 137 | html.write(f'<td>{line["status"]}</td>') |
116 | 138 | html.write(f'<td><pre>{line["traceback"]}</pre></td>') |
117 | 139 | html.write(f'<td><pre>{line["debug_result"]}</pre></td>') |
118 | | - html.write('</tr>') |
119 | | - html.write('</table></body></html>') |
| 140 | + html.write("</tr>") |
| 141 | + html.write("</table></body></html>") |
120 | 142 | print(f"{GREEN}HTML debug report saved to {html_path}{RESET}") |
0 commit comments