diff --git a/.github/scripts/differential-tests-error-out-on-failures.py b/.github/scripts/differential-tests-error-out-on-failures.py new file mode 100644 index 0000000000000..85b0c8d187e3a --- /dev/null +++ b/.github/scripts/differential-tests-error-out-on-failures.py @@ -0,0 +1,87 @@ +""" +This script is used to turn the JSON report produced by the revive differential tests, detecting +failures and exiting with the appropriate code if a failure is encountered. +""" + +import json, typing, sys + + +class Report(typing.TypedDict): + context: "Context" + execution_information: dict["MetadataFilePathString", "MetadataFileReport"] + + +class MetadataFileReport(typing.TypedDict): + case_reports: dict["CaseIdxString", "CaseReport"] + + +class CaseReport(typing.TypedDict): + mode_execution_reports: dict["ModeString", "ExecutionReport"] + + +class ExecutionReport(typing.TypedDict): + status: "TestCaseStatus" + + +class Context(typing.TypedDict): + Test: "TestContext" + + +class TestContext(typing.TypedDict): + corpus_configuration: "CorpusConfiguration" + + +class CorpusConfiguration(typing.TypedDict): + test_specifiers: list["TestSpecifier"] + + +class CaseStatusSuccess(typing.TypedDict): + status: typing.Literal["Succeeded"] + steps_executed: int + + +class CaseStatusFailure(typing.TypedDict): + status: typing.Literal["Failed"] + reason: str + + +class CaseStatusIgnored(typing.TypedDict): + status: typing.Literal["Ignored"] + reason: str + + +TestCaseStatus = typing.Union[CaseStatusSuccess, CaseStatusFailure, CaseStatusIgnored] +"""A union type of all of the possible statuses that could be reported for a case.""" + +TestSpecifier = str +"""A test specifier string. For example resolc-compiler-tests/fixtures/solidity/test.json::0::Y+""" + +ModeString = str +"""The mode string. For example Y+ >=0.8.13""" + +MetadataFilePathString = str +"""The path to a metadata file. For example resolc-compiler-tests/fixtures/solidity/test.json""" + +CaseIdxString = str +"""The index of a case as a string. For example '0'""" + + +def main() -> None: + with open(sys.argv[1], "r") as file: + report: Report = json.load(file) + + for _, mode_to_case_mapping in report["execution_information"].items(): + for _, case_idx_to_report_mapping in mode_to_case_mapping[ + "case_reports" + ].items(): + for _, execution_report in case_idx_to_report_mapping[ + "mode_execution_reports" + ].items(): + if execution_report["status"]["status"] == "Failed": + exit(1) + + exit(0) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 9fb417542999b..4db614a2a0179 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -51,7 +51,7 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: repository: paritytech/revive-differential-tests - ref: 66feb36b4ef2c79415ca8ea765d8235d48dfa8f8 + ref: cd6b7969acafbd3c29907e543a9751d886df6d00 path: revive-differential-tests submodules: recursive - name: Installing Retester @@ -96,6 +96,9 @@ jobs: with: header: diff-tests-report-${{ matrix.platform }} path: report.md + - name: Check for Failures + if: ${{ matrix.platform == 'revive-dev-node-revm-solc' }} + run: ./.github/scripts/differential-tests-error-out-on-failures.py report.json evm-test-suite: needs: [preflight] diff --git a/prdoc/pr_10544.prdoc b/prdoc/pr_10544.prdoc new file mode 100644 index 0000000000000..cc1d708abc527 --- /dev/null +++ b/prdoc/pr_10544.prdoc @@ -0,0 +1,8 @@ +title: Check for failures in DT CI +doc: +- audience: Runtime Dev + description: |- + # Description + + This is a small PR that makes the CI of the differential testing framework fail if any failure is encountered. We only do this with the `revive-dev-node-revm-solc` target since this is the only target at the moment that has no failures. +crates: []