Skip to content

Commit 8d27883

Browse files
0xOmarAgithub-actions[bot]
authored and
pgherveou
committed
Adding Retester to CI (#10071)
This PR adds differential tests as part of the CI of the polkadot SDK. Currently, a job will be started when pushing to master or when a PR is opened that runs the [differential testing framework](https://github.com/paritytech/revive-differential-tests) with the `revive-dev-node-revm-resolc` target. --------- Co-authored-by: cmd[bot] <41898282+github-actions[bot]@users.noreply.github.com>
1 parent ebe2cca commit 8d27883

File tree

3 files changed

+250
-0
lines changed

3 files changed

+250
-0
lines changed

‎.github/scripts/process-differential-tests-report.py‎

Lines changed: 241 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,241 @@
1+
"""
2+
This script is used to turn the JSON report produced by the revive differential tests tool into an
3+
easy to consume markdown document for the purpose of reporting this information in the Polkadot SDK
4+
CI. The full models used in the JSON report can be found in the revive differential tests repo and
5+
the models used in this script are just a partial reproduction of the full report models.
6+
"""
7+
8+
import json, typing, io, sys
9+
10+
11+
class Report(typing.TypedDict):
12+
context: "Context"
13+
execution_information: dict["MetadataFilePathString", "MetadataFileReport"]
14+
15+
16+
class MetadataFileReport(typing.TypedDict):
17+
case_reports: dict["CaseIdxString", "CaseReport"]
18+
19+
20+
class CaseReport(typing.TypedDict):
21+
mode_execution_reports: dict["ModeString", "ExecutionReport"]
22+
23+
24+
class ExecutionReport(typing.TypedDict):
25+
status: "TestCaseStatus"
26+
27+
28+
class Context(typing.TypedDict):
29+
Test: "TestContext"
30+
31+
32+
class TestContext(typing.TypedDict):
33+
corpus_configuration: "CorpusConfiguration"
34+
35+
36+
class CorpusConfiguration(typing.TypedDict):
37+
test_specifiers: list["TestSpecifier"]
38+
39+
40+
class CaseStatusSuccess(typing.TypedDict):
41+
status: typing.Literal["Succeeded"]
42+
steps_executed: int
43+
44+
45+
class CaseStatusFailure(typing.TypedDict):
46+
status: typing.Literal["Failed"]
47+
reason: str
48+
49+
50+
class CaseStatusIgnored(typing.TypedDict):
51+
status: typing.Literal["Ignored"]
52+
reason: str
53+
54+
55+
TestCaseStatus = typing.Union[CaseStatusSuccess, CaseStatusFailure, CaseStatusIgnored]
56+
"""A union type of all of the possible statuses that could be reported for a case."""
57+
58+
TestSpecifier = str
59+
"""A test specifier string. For example resolc-compiler-tests/fixtures/solidity/test.json::0::Y+"""
60+
61+
ModeString = str
62+
"""The mode string. For example Y+ >=0.8.13"""
63+
64+
MetadataFilePathString = str
65+
"""The path to a metadata file. For example resolc-compiler-tests/fixtures/solidity/test.json"""
66+
67+
CaseIdxString = str
68+
"""The index of a case as a string. For example '0'"""
69+
70+
71+
def path_relative_to_resolc_compiler_test_directory(path: str) -> str:
72+
"""
73+
Given a path, this function returns the path relative to the resolc-compiler-test directory. The
74+
following is an example of an input and an output:
75+
76+
Input: ~/polkadot-sdk/revive-differential-tests/resolc-compiler-tests/fixtures/solidity/test.json
77+
Output: test.json
78+
"""
79+
80+
return f"{path.split('resolc-compiler-tests/fixtures/solidity')[-1].strip('/')}"
81+
82+
83+
def main() -> None:
84+
with open(sys.argv[1], "r") as file:
85+
report: Report = json.load(file)
86+
87+
# Starting the markdown document and adding information to it as we go.
88+
markdown_document: io.TextIOWrapper = open("report.md", "w")
89+
print("# Differential Tests Results", file=markdown_document)
90+
91+
# Getting all of the test specifiers from the report and making them relative to the tests dir.
92+
test_specifiers: list[str] = list(
93+
map(
94+
path_relative_to_resolc_compiler_test_directory,
95+
report["context"]["Test"]["corpus_configuration"]["test_specifiers"],
96+
)
97+
)
98+
print("## Specified Tests", file=markdown_document)
99+
for test_specifier in test_specifiers:
100+
print(f"* `{test_specifier}`", file=markdown_document)
101+
102+
# Counting the total number of test cases, successes, failures, and ignored tests
103+
total_number_of_cases: int = 0
104+
total_number_of_successes: int = 0
105+
total_number_of_failures: int = 0
106+
total_number_of_ignores: int = 0
107+
for _, mode_to_case_mapping in report["execution_information"].items():
108+
for _, case_idx_to_report_mapping in mode_to_case_mapping[
109+
"case_reports"
110+
].items():
111+
for _, execution_report in case_idx_to_report_mapping[
112+
"mode_execution_reports"
113+
].items():
114+
status: TestCaseStatus = execution_report["status"]
115+
116+
total_number_of_cases += 1
117+
if status["status"] == "Succeeded":
118+
total_number_of_successes += 1
119+
elif status["status"] == "Failed":
120+
total_number_of_failures += 1
121+
elif status["status"] == "Ignored":
122+
total_number_of_ignores += 1
123+
else:
124+
raise Exception(
125+
f"Encountered a status that's unknown to the script: {status}"
126+
)
127+
128+
print("## Counts", file=markdown_document)
129+
print(
130+
f"* **Total Number of Test Cases:** {total_number_of_cases}",
131+
file=markdown_document,
132+
)
133+
print(
134+
f"* **Total Number of Successes:** {total_number_of_successes}",
135+
file=markdown_document,
136+
)
137+
print(
138+
f"* **Total Number of Failures:** {total_number_of_failures}",
139+
file=markdown_document,
140+
)
141+
print(
142+
f"* **Total Number of Ignores:** {total_number_of_ignores}",
143+
file=markdown_document,
144+
)
145+
146+
# Grouping the various test cases into dictionaries and groups depending on their status to make
147+
# them easier to include in the markdown document later on.
148+
successful_cases: dict[
149+
MetadataFilePathString, dict[CaseIdxString, set[ModeString]]
150+
] = {}
151+
for metadata_file_path, mode_to_case_mapping in report[
152+
"execution_information"
153+
].items():
154+
for case_idx_string, case_idx_to_report_mapping in mode_to_case_mapping[
155+
"case_reports"
156+
].items():
157+
for mode_string, execution_report in case_idx_to_report_mapping[
158+
"mode_execution_reports"
159+
].items():
160+
status: TestCaseStatus = execution_report["status"]
161+
metadata_file_path: str = (
162+
path_relative_to_resolc_compiler_test_directory(metadata_file_path)
163+
)
164+
mode_string: str = mode_string.replace(" M3", "+").replace(" M0", "-")
165+
166+
if status["status"] == "Succeeded":
167+
successful_cases.setdefault(
168+
metadata_file_path,
169+
{},
170+
).setdefault(
171+
case_idx_string, set()
172+
).add(mode_string)
173+
174+
print("## Failures", file=markdown_document)
175+
print(
176+
"The test specifiers seen in this section have the format 'path::case_idx::compilation_mode'\
177+
and they're compatible with the revive differential tests framework and can be specified\
178+
to it directly in the same way that they're provided through the `--test` argument of the\
179+
framework.\n",
180+
file=markdown_document,
181+
)
182+
print(
183+
"The failures are provided in an expandable section to ensure that the PR does not get \
184+
polluted with information. Please click on the section below for more information",
185+
file=markdown_document,
186+
)
187+
print(
188+
"<details><summary>Detailed Differential Tests Failure Information</summary>\n\n",
189+
file=markdown_document,
190+
)
191+
print("| Test Specifier | Failure Reason | Note |", file=markdown_document)
192+
print("| -- | -- | -- |", file=markdown_document)
193+
194+
for metadata_file_path, mode_to_case_mapping in report[
195+
"execution_information"
196+
].items():
197+
for case_idx_string, case_idx_to_report_mapping in mode_to_case_mapping[
198+
"case_reports"
199+
].items():
200+
for mode_string, execution_report in case_idx_to_report_mapping[
201+
"mode_execution_reports"
202+
].items():
203+
status: TestCaseStatus = execution_report["status"]
204+
metadata_file_path: str = (
205+
path_relative_to_resolc_compiler_test_directory(metadata_file_path)
206+
)
207+
mode_string: str = mode_string.replace(" M3", "+").replace(" M0", "-")
208+
209+
if status["status"] != "Failed":
210+
continue
211+
212+
failure_reason: str = status["reason"].replace("\n", " ")
213+
214+
note: str = ""
215+
modes_where_this_case_succeeded: set[ModeString] = (
216+
successful_cases.setdefault(
217+
metadata_file_path,
218+
{},
219+
).setdefault(case_idx_string, set())
220+
)
221+
if len(modes_where_this_case_succeeded) != 0:
222+
note: str = (
223+
f"This test case succeeded with other compilation modes: {modes_where_this_case_succeeded}"
224+
)
225+
226+
test_specifier: str = (
227+
f"{metadata_file_path}::{case_idx_string}::{mode_string}"
228+
)
229+
print(
230+
f"| `{test_specifier}` | `{failure_reason}` | {note} |",
231+
file=markdown_document,
232+
)
233+
print("\n\n</details>", file=markdown_document)
234+
235+
# The primary downside of not using `with`, but I guess it's better since I don't want to over
236+
# indent the code.
237+
markdown_document.close()
238+
239+
240+
if __name__ == "__main__":
241+
main()

‎.github/workflows/tests-evm.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,3 +164,4 @@ jobs:
164164
else
165165
echo '### Good job! All the required jobs passed 🚀' >> $GITHUB_STEP_SUMMARY
166166
fi
167+

‎prdoc/pr_10071.prdoc‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
title: Adding Retester to CI
2+
doc:
3+
- audience: Runtime Dev
4+
description: |-
5+
# Description
6+
7+
This PR adds differential tests as part of the CI of the polkadot SDK. Currently, a job will be started when pushing to master or when a PR is opened that runs the [differential testing framework](https://github.com/paritytech/revive-differential-tests) with the `revive-dev-node-revm-resolc` target.
8+
crates: []

0 commit comments

Comments
 (0)