step

Author: pillo79

extra/ci_collect_logs.py

@@ -5,28 +5,28 @@
 import json
 import re
 
-EXPECTED_ERROR = -2
 SKIP = -1
 PASS = 0
 WARNING = 1
-ERROR = 2
-FAILURE = 3
+EXPECTED_ERROR = 2
+ERROR = 3
+FAILURE = 4
 
 TEST_STATUS = [
 		":green_circle:",
 		":yellow_circle:",
+		":red_circle:*",
 		":red_circle:",
 		":fire:",
-		"[:red_circle:]", # -2
 		":new_moon:" # -1
 ]
 
 BOARD_STATUS = [
 		":white_check_mark:",
-		":grey_exclamation:",
+		":white_check_mark:*",
+		":heavy_check_mark:*",
 		":x:",
 		":fire:",
-		"[:x:]", # -2
 		":new_moon:" # -1
 ]
 
@@ -53,6 +53,7 @@ def example_name(sketch_name):
 TEST_MATRIX = {}
 BOARD_SUMMARY = {}
 EXPECTED_ERRORS = set()
+EXPECTED_ERRORS.add(('portentac33', 'ArduinoCore-zephyr/libraries/Ethernet/examples/UdpNtpClient'))
 
 def log_test(name, board, status, issues, job_link=None):
 	"""
@@ -62,8 +63,8 @@ def log_test(name, board, status, issues, job_link=None):
 	if not board in BOARD_SUMMARY:
 		BOARD_SUMMARY[board] = {
 			'tests': 0,
+			'expected-errors': 0,
 			'errors': 0,
-			'errors-expected': 0,
 			'warnings': 0,
 			'job_link': job_link,
 			'status': 0
@@ -79,7 +80,7 @@ def log_test(name, board, status, issues, job_link=None):
 		board_info['warnings'] += 1
 	elif status == ERROR:
 		if (board, name) in EXPECTED_ERRORS:
-			board_info['errors-expected'] += 1
+			board_info['expected-errors'] += 1
 			status = EXPECTED_ERROR
 		else:
 			board_info['errors'] += 1
@@ -139,7 +140,7 @@ def process_build_reports():
 
 		reports = report_data.get('boards', [{}])[0].get('sketches', [])
 		if not reports:
-			log_test('CI test', board, FAILURE, "Test report is empty, check full log.", job_link)
+			log_test('CI test', board, FAILURE, "Test report is empty, check CI log.", job_link)
 			continue # Skip to the next board
 
 		# 7. Sketch Loop: Iterate through individual sketch reports
@@ -164,32 +165,41 @@ def process_build_reports():
 	artifacts = set(item['artifact'] for item in ALL_BOARD_DATA)
 
 	# Print the recap table
-	print("<table>\n<tr><th>Artifact</th><th>Board</th><th>Status</th><th>Tests</th><th>Warnings</th><th>Errors</th></tr>")
+	print("<table>\n<tr><th>Artifact</th><th>Board</th><th>Status</th><th>Passed</th><th>Warnings</th><th>Errors</th></tr>")
 
 	for artifact in sorted(list(artifacts), reverse=True):
-		artifact_boards = [item for item in ALL_BOARD_DATA if item['artifact'] == artifact]
+		artifact_boards = [ item['board'] for item in ALL_BOARD_DATA if item['artifact'] == artifact ]
+
+		artifact_status = PASS
+		for board in artifact_boards:
+			artifact_status = max(artifact_status, BOARD_SUMMARY[board]['status'])
 
 		first_line = True
 		for board in artifact_boards:
-			print(f"<tr>", f"<td rowspan='{len(artifact_boards)}'><a href='#{artifact}'><code>{artifact}</code></a></td>" if first_line else "")
+			print(f"<tr>", f"<td rowspan='{len(artifact_boards)}'>{BOARD_STATUS[artifact_status]} <a href='#{artifact}'><code>{artifact}</code></a></td>" if first_line else "")
 			first_line = False
 
-			board_name = board['board']
-			summary = BOARD_SUMMARY[board_name]
+			summary = BOARD_SUMMARY[board]
 			job_link = summary['job_link']
 			status = summary['status']
-			board_link = f"<a href='{job_link}'>{board_name}</a>" if job_link else board_name
+			board_link = f"<a href='{job_link}'>{board}</a>" if job_link else board
 
 			print(f"<td><code>{board_link}</code></td><td align='center'>{BOARD_STATUS[status]}</td>")
 			if status == FAILURE:
-				print(f"<td colspan=3>{TEST_MATRIX['CI test'][board_name]['issues'][0]}</td></tr>")
+				print(f"<td colspan=3>{TEST_MATRIX['CI test'][board]['issues'][0]}</td></tr>")
 			else:
-				print(f"<td align='right'>{(summary['tests'] - summary['errors']) or '-'}</td><td align='right'>{summary['warnings'] or '-'}</td><td align='right'>{summary['errors'] or '-'}</td></tr>")
+				passed = summary['tests'] - summary['errors']
+				passed_str = passed or "-"
+				warnings_str = summary['warnings'] or "-"
+				errors_str = f"<b>{summary['errors']}</b>" if summary['errors'] else "-"
+				if summary['expected-errors']:
+					errors_str += f" (+{summary['expected-errors']}*)"
+				print(f"<td align='right'>{passed_str}</td><td align='right'>{warnings_str}</td><td align='right'>{errors_str}</td></tr>")
 	print("</table>\n")
 
 	for artifact in sorted(list(artifacts), reverse=True):
-		artifact_boards = [ item for item in ALL_BOARD_DATA
-							if item['artifact'] == artifact and BOARD_SUMMARY[item['board']]['status'] != FAILURE ]
+		artifact_boards = [ item['board'] for item in ALL_BOARD_DATA
+										  if item['artifact'] == artifact and BOARD_SUMMARY[item['board']]['status'] != FAILURE ]
 
 		if not artifact_boards:
 			continue
@@ -200,8 +210,7 @@ def process_build_reports():
 		# Print the test matrix
 
 		header_row = "<tr><th colspan=2>Sketch / Board</th>"
-		for board_data in artifact_boards:
-			board = board_data['board']
+		for board in artifact_boards:
 			status = BOARD_SUMMARY[board]['status']
 			header_col = f"<code>{board}</code>"
 			if BOARD_SUMMARY[board]['job_link']:
@@ -214,34 +223,33 @@ def process_build_reports():
 		print("<table id='#{artifact}'>")
 		print(header_row)
 
-		artifact_matrix = { sketch: results for sketch, results in TEST_MATRIX.items() if any(board_data['board'] in results for board_data in artifact_boards) }
+		artifact_matrix = { sketch: results for sketch, results in TEST_MATRIX.items() if any(board in results for board in artifact_boards) }
 
 		for sketch_name, board_results in artifact_matrix.items():
 			sketch_status = SKIP
 			row_data = ""
-			for board_data in artifact_boards:
-				board_name = board_data['board']
-				test_result = board_results.get(board_name)
+			for board in artifact_boards:
+				test_result = board_results.get(board)
 				status = test_result['status'] if test_result else SKIP
-				sketch_status = max(sketch_status, status)
 				issues = test_result['issues'] if test_result else ""
+
+				sketch_status = max(sketch_status, status)
 				sketch_id = sketch_name.replace('/', '_').replace(' ', '_').replace('-', '_')
-				icon_link = f"<a href='#{sketch_id}_{board_name}'>{TEST_STATUS[status]}</a>" if issues else TEST_STATUS[status]
+				icon_link = f"<a href='#{sketch_id}_{board}'>{TEST_STATUS[status]}</a>" if issues else TEST_STATUS[status]
 				row_data += f"<td align='center'>{icon_link}</a></td>"
 			print(f"<tr><td>{TEST_STATUS[sketch_status]}</td><td><code>{example_name(sketch_name)}</code></td>{row_data}</tr>")
 		print("</table>\n")
 
-		print(f"<details><summary><b>Error and warning logs for <code>{artifact}</code></b></summary>\n")
+		print(f"<details><summary>:scroll: <b>Detailed test issues for <code>{artifact}</code></b></summary>\n")
 		print("<table>")
 		print("<tr><th>Sketch</th><th>Build Details</th></tr>")
 		for sketch_name, board_results in artifact_matrix.items():
 			row_open = False
-			for board_data in artifact_boards:
-				board_name = board_data['board']
-				if not board_name in board_results:
+			for board in artifact_boards:
+				if not board in board_results:
 					continue
 
-				test_result = board_results[board_name]
+				test_result = board_results[board]
 				status = test_result['status']
 				issues = test_result['issues']
 				if not issues:
@@ -256,7 +264,7 @@ def process_build_reports():
 				if job_link:
 					job_link = f" <a href='{job_link}'> :scroll:</a>"
 				sketch_id = sketch_name.replace('/', '_').replace(' ', '_').replace('-', '_')
-				print(f"<details id='{sketch_id}_{board_name}'><summary>{artifact_count(len(issues), status, TEST_STATUS)} <code>{board_name}</code>{job_link}</summary>")
+				print(f"<details id='{sketch_id}_{board}'><summary>{artifact_count(len(issues), status, TEST_STATUS)} <code>{board}</code>{job_link}</summary>")
 				print("\n```\n" + "\n".join(issues) + "\n```")
 				print("</details>")