Skip to content

Commit 81dfccd

Browse files
author
Razvan Lupusoru
committed
Merge remote-tracking branch 'upstream/main' into firaccimplicitdata
2 parents 2f15a1a + bda7289 commit 81dfccd

File tree

53 files changed

+3441
-2124
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+3441
-2124
lines changed

‎.ci/generate_test_report_github.py‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,10 @@
44
"""Script to generate a build report for Github."""
55

66
import argparse
7-
import platform
87

98
import generate_test_report_lib
109

1110

12-
def compute_platform_title() -> str:
13-
logo = ":window:" if platform.system() == "Windows" else ":penguin:"
14-
# On Linux the machine value is x86_64 on Windows it is AMD64.
15-
if platform.machine() == "x86_64" or platform.machine() == "AMD64":
16-
arch = "x64"
17-
else:
18-
arch = platform.machine()
19-
return f"{logo} {platform.system()} {arch} Test Results"
20-
21-
2211
if __name__ == "__main__":
2312
parser = argparse.ArgumentParser()
2413
parser.add_argument("return_code", help="The build's return code.", type=int)
@@ -28,7 +17,9 @@ def compute_platform_title() -> str:
2817
args = parser.parse_args()
2918

3019
report = generate_test_report_lib.generate_report_from_files(
31-
compute_platform_title(), args.return_code, args.build_test_logs
20+
generate_test_report_lib.compute_platform_title(),
21+
args.return_code,
22+
args.build_test_logs,
3223
)
3324

3425
print(report)

‎.ci/generate_test_report_lib.py‎

Lines changed: 51 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,20 @@
33
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
44
"""Library to parse JUnit XML files and return a markdown report."""
55

6+
from typing import TypedDict
7+
import platform
8+
69
from junitparser import JUnitXml, Failure
710

11+
12+
# This data structure should match the definition in llvm-zorg in
13+
# premerge/advisor/advisor_lib.py
14+
class FailureExplanation(TypedDict):
15+
name: str
16+
explained: bool
17+
reason: str | None
18+
19+
820
SEE_BUILD_FILE_STR = "Download the build's log file to see the details."
921
UNRELATED_FAILURES_STR = (
1022
"If these failures are unrelated to your changes (for example "
@@ -82,16 +94,29 @@ def find_failure_in_ninja_logs(ninja_logs: list[list[str]]) -> list[tuple[str, s
8294
return failures
8395

8496

85-
def _format_ninja_failures(ninja_failures: list[tuple[str, str]]) -> list[str]:
86-
"""Formats ninja failures into summary views for the report."""
97+
def _format_failures(
98+
failures: list[tuple[str, str]], failure_explanations: dict[str, FailureExplanation]
99+
) -> list[str]:
100+
"""Formats failures into summary views for the report."""
87101
output = []
88-
for build_failure in ninja_failures:
102+
for build_failure in failures:
89103
failed_action, failure_message = build_failure
104+
failure_explanation = None
105+
if failed_action in failure_explanations:
106+
failure_explanation = failure_explanations[failed_action]
107+
output.append("<details>")
108+
if failure_explanation:
109+
output.extend(
110+
[
111+
f"<summary>{failed_action} (Likely Already Failing)</summary>" "",
112+
failure_explanation["reason"],
113+
"",
114+
]
115+
)
116+
else:
117+
output.extend([f"<summary>{failed_action}</summary>", ""])
90118
output.extend(
91119
[
92-
"<details>",
93-
f"<summary>{failed_action}</summary>",
94-
"",
95120
"```",
96121
failure_message,
97122
"```",
@@ -132,12 +157,19 @@ def generate_report(
132157
ninja_logs: list[list[str]],
133158
size_limit=1024 * 1024,
134159
list_failures=True,
160+
failure_explanations_list: list[FailureExplanation] = [],
135161
):
136162
failures = get_failures(junit_objects)
137163
tests_run = 0
138164
tests_skipped = 0
139165
tests_failed = 0
140166

167+
failure_explanations: dict[str, FailureExplanation] = {}
168+
for failure_explanation in failure_explanations_list:
169+
if not failure_explanation["explained"]:
170+
continue
171+
failure_explanations[failure_explanation["name"]] = failure_explanation
172+
141173
for results in junit_objects:
142174
for testsuite in results:
143175
tests_run += testsuite.tests
@@ -176,7 +208,7 @@ def generate_report(
176208
"",
177209
]
178210
)
179-
report.extend(_format_ninja_failures(ninja_failures))
211+
report.extend(_format_failures(ninja_failures, failure_explanations))
180212
report.extend(
181213
[
182214
"",
@@ -212,18 +244,7 @@ def plural(num_tests):
212244

213245
for testsuite_name, failures in failures.items():
214246
report.extend(["", f"### {testsuite_name}"])
215-
for name, output in failures:
216-
report.extend(
217-
[
218-
"<details>",
219-
f"<summary>{name}</summary>",
220-
"",
221-
"```",
222-
output,
223-
"```",
224-
"</details>",
225-
]
226-
)
247+
report.extend(_format_failures(failures, failure_explanations))
227248
elif return_code != 0:
228249
# No tests failed but the build was in a failed state. Bring this to the user's
229250
# attention.
@@ -248,7 +269,7 @@ def plural(num_tests):
248269
"",
249270
]
250271
)
251-
report.extend(_format_ninja_failures(ninja_failures))
272+
report.extend(_format_failures(ninja_failures, failure_explanations))
252273

253274
if failures or return_code != 0:
254275
report.extend(["", UNRELATED_FAILURES_STR])
@@ -285,3 +306,13 @@ def load_info_from_files(build_log_files):
285306
def generate_report_from_files(title, return_code, build_log_files):
286307
junit_objects, ninja_logs = load_info_from_files(build_log_files)
287308
return generate_report(title, return_code, junit_objects, ninja_logs)
309+
310+
311+
def compute_platform_title() -> str:
312+
logo = ":window:" if platform.system() == "Windows" else ":penguin:"
313+
# On Linux the machine value is x86_64 on Windows it is AMD64.
314+
if platform.machine() == "x86_64" or platform.machine() == "AMD64":
315+
arch = "x64"
316+
else:
317+
arch = platform.machine()
318+
return f"{logo} {platform.system()} {arch} Test Results"

‎.ci/generate_test_report_lib_test.py‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -781,6 +781,160 @@ def test_report_size_limit(self):
781781
),
782782
)
783783

784+
def test_report_ninja_explanation(self):
785+
self.assertEqual(
786+
generate_test_report_lib.generate_report(
787+
"Foo",
788+
1,
789+
[],
790+
[
791+
[
792+
"[1/5] test/1.stamp",
793+
"[2/5] test/2.stamp",
794+
"[3/5] test/3.stamp",
795+
"[4/5] test/4.stamp",
796+
"FAILED: test/4.stamp",
797+
"touch test/4.stamp",
798+
"Half Moon Bay.",
799+
"[5/5] test/5.stamp",
800+
]
801+
],
802+
failure_explanations_list=[
803+
{
804+
"name": "test/4.stamp",
805+
"explained": True,
806+
"reason": "Failing at head",
807+
}
808+
],
809+
),
810+
dedent(
811+
"""\
812+
# Foo
813+
814+
The build failed before running any tests. Click on a failure below to see the details.
815+
816+
<details>
817+
<summary>test/4.stamp (Likely Already Failing)</summary>
818+
Failing at head
819+
820+
```
821+
FAILED: test/4.stamp
822+
touch test/4.stamp
823+
Half Moon Bay.
824+
```
825+
</details>
826+
827+
If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
828+
),
829+
)
830+
831+
def test_report_test_failure_explanation(self):
832+
self.assertEqual(
833+
generate_test_report_lib.generate_report(
834+
"Foo",
835+
1,
836+
[
837+
junit_from_xml(
838+
dedent(
839+
"""\
840+
<?xml version="1.0" encoding="UTF-8"?>
841+
<testsuites time="8.89">
842+
<testsuite name="Bar" tests="1" failures="1" skipped="0" time="410.63">
843+
<testcase classname="Bar/test_3" name="test_3" time="0.02">
844+
<failure><![CDATA[Error! Expected Big Sur to be next to the ocean.]]></failure>
845+
</testcase>
846+
</testsuite>
847+
</testsuites>"""
848+
)
849+
)
850+
],
851+
[],
852+
failure_explanations_list=[
853+
{
854+
"name": "Bar/test_3/test_3",
855+
"explained": True,
856+
"reason": "Big Sur is next to the Pacific.",
857+
}
858+
],
859+
),
860+
(
861+
dedent(
862+
"""\
863+
# Foo
864+
865+
* 1 test failed
866+
867+
## Failed Tests
868+
(click on a test name to see its output)
869+
870+
### Bar
871+
<details>
872+
<summary>Bar/test_3/test_3 (Likely Already Failing)</summary>
873+
Big Sur is next to the Pacific.
874+
875+
```
876+
Error! Expected Big Sur to be next to the ocean.
877+
```
878+
</details>
879+
880+
If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
881+
)
882+
),
883+
)
884+
885+
def test_report_test_failure_have_explanation_explained_false(self):
886+
self.assertEqual(
887+
generate_test_report_lib.generate_report(
888+
"Foo",
889+
1,
890+
[
891+
junit_from_xml(
892+
dedent(
893+
"""\
894+
<?xml version="1.0" encoding="UTF-8"?>
895+
<testsuites time="8.89">
896+
<testsuite name="Bar" tests="1" failures="1" skipped="0" time="410.63">
897+
<testcase classname="Bar/test_3" name="test_3" time="0.02">
898+
<failure><![CDATA[Error! Expected Mt. Shasta to be next in the Eastern Sierras.]]></failure>
899+
</testcase>
900+
</testsuite>
901+
</testsuites>"""
902+
)
903+
)
904+
],
905+
[],
906+
failure_explanations_list=[
907+
{
908+
"name": "Bar/test_3/test_3",
909+
"explained": False,
910+
"reason": "Mt. Shasta is in the Cascades",
911+
}
912+
],
913+
),
914+
(
915+
dedent(
916+
"""\
917+
# Foo
918+
919+
* 1 test failed
920+
921+
## Failed Tests
922+
(click on a test name to see its output)
923+
924+
### Bar
925+
<details>
926+
<summary>Bar/test_3/test_3</summary>
927+
928+
```
929+
Error! Expected Mt. Shasta to be next in the Eastern Sierras.
930+
```
931+
</details>
932+
933+
If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
934+
)
935+
),
936+
)
937+
784938
def test_generate_report_end_to_end(self):
785939
with tempfile.TemporaryDirectory() as temp_dir:
786940
junit_xml_file = os.path.join(temp_dir, "junit.xml")

0 commit comments

Comments
 (0)