From 9cfebb3a01e0ae707e0e970888c137a276e11334 Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Thu, 4 Dec 2025 17:57:43 +0300 Subject: [PATCH 1/5] Check for failures in DT Ci --- ...ifferential-tests-error-out-on-failures.py | 90 +++++++++++++++++++ .github/workflows/tests-evm.yml | 3 + 2 files changed, 93 insertions(+) create mode 100644 .github/scripts/differential-tests-error-out-on-failures.py diff --git a/.github/scripts/differential-tests-error-out-on-failures.py b/.github/scripts/differential-tests-error-out-on-failures.py new file mode 100644 index 0000000000000..c825b9232cf22 --- /dev/null +++ b/.github/scripts/differential-tests-error-out-on-failures.py @@ -0,0 +1,90 @@ +""" +This script is used to turn the JSON report produced by the revive differential tests tool into an +easy to consume markdown document for the purpose of reporting this information in the Polkadot SDK +CI. The full models used in the JSON report can be found in the revive differential tests repo and +the models used in this script are just a partial reproduction of the full report models. +""" + +import json, typing, sys + + +class Report(typing.TypedDict): + context: "Context" + execution_information: dict["MetadataFilePathString", "MetadataFileReport"] + + +class MetadataFileReport(typing.TypedDict): + case_reports: dict["CaseIdxString", "CaseReport"] + + +class CaseReport(typing.TypedDict): + mode_execution_reports: dict["ModeString", "ExecutionReport"] + + +class ExecutionReport(typing.TypedDict): + status: "TestCaseStatus" + + +class Context(typing.TypedDict): + Test: "TestContext" + + +class TestContext(typing.TypedDict): + corpus_configuration: "CorpusConfiguration" + + +class CorpusConfiguration(typing.TypedDict): + test_specifiers: list["TestSpecifier"] + + +class CaseStatusSuccess(typing.TypedDict): + status: typing.Literal["Succeeded"] + steps_executed: int + + +class CaseStatusFailure(typing.TypedDict): + status: typing.Literal["Failed"] + reason: str + + +class CaseStatusIgnored(typing.TypedDict): + status: typing.Literal["Ignored"] + reason: str + + +TestCaseStatus = typing.Union[CaseStatusSuccess, CaseStatusFailure, CaseStatusIgnored] +"""A union type of all of the possible statuses that could be reported for a case.""" + +TestSpecifier = str +"""A test specifier string. For example resolc-compiler-tests/fixtures/solidity/test.json::0::Y+""" + +ModeString = str +"""The mode string. For example Y+ >=0.8.13""" + +MetadataFilePathString = str +"""The path to a metadata file. For example resolc-compiler-tests/fixtures/solidity/test.json""" + +CaseIdxString = str +"""The index of a case as a string. For example '0'""" + + +def main() -> None: + with open(sys.argv[1], "r") as file: + report: Report = json.load(file) + + # Counting the total number of test cases, successes, failures, and ignored tests + for _, mode_to_case_mapping in report["execution_information"].items(): + for _, case_idx_to_report_mapping in mode_to_case_mapping[ + "case_reports" + ].items(): + for _, execution_report in case_idx_to_report_mapping[ + "mode_execution_reports" + ].items(): + if execution_report["status"]["status"] == "Failed": + exit(1) + + exit(0) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 9fb417542999b..e62362e98e5aa 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -96,6 +96,9 @@ jobs: with: header: diff-tests-report-${{ matrix.platform }} path: report.md + - name: Check for Failures + if: ${{ matrix.platform == 'revive-dev-node-revm-solc' }} + run: ./.github/scripts/differential-tests-error-out-on-failures.py report.json evm-test-suite: needs: [preflight] From c096a7504e817600690f3205e6d5238dc78afe33 Mon Sep 17 00:00:00 2001 From: "cmd[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:02:52 +0000 Subject: [PATCH 2/5] Update from github-actions[bot] running command 'prdoc --audience runtime_dev' --- prdoc/pr_10544.prdoc | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 prdoc/pr_10544.prdoc diff --git a/prdoc/pr_10544.prdoc b/prdoc/pr_10544.prdoc new file mode 100644 index 0000000000000..cc1d708abc527 --- /dev/null +++ b/prdoc/pr_10544.prdoc @@ -0,0 +1,8 @@ +title: Check for failures in DT CI +doc: +- audience: Runtime Dev + description: |- + # Description + + This is a small PR that makes the CI of the differential testing framework fail if any failure is encountered. We only do this with the `revive-dev-node-revm-solc` target since this is the only target at the moment that has no failures. +crates: [] From 2c48ad2779ebba2f1497267069b66d4bdf633cab Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Thu, 4 Dec 2025 18:05:36 +0300 Subject: [PATCH 3/5] Remove un-needed comment --- .github/scripts/differential-tests-error-out-on-failures.py | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/scripts/differential-tests-error-out-on-failures.py b/.github/scripts/differential-tests-error-out-on-failures.py index c825b9232cf22..75530a453a6e5 100644 --- a/.github/scripts/differential-tests-error-out-on-failures.py +++ b/.github/scripts/differential-tests-error-out-on-failures.py @@ -72,7 +72,6 @@ def main() -> None: with open(sys.argv[1], "r") as file: report: Report = json.load(file) - # Counting the total number of test cases, successes, failures, and ignored tests for _, mode_to_case_mapping in report["execution_information"].items(): for _, case_idx_to_report_mapping in mode_to_case_mapping[ "case_reports" From 9a9e1ace8c45358cfd340878bf24d75caad86a1a Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Thu, 4 Dec 2025 18:08:17 +0300 Subject: [PATCH 4/5] Update doc comment --- .github/scripts/differential-tests-error-out-on-failures.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/scripts/differential-tests-error-out-on-failures.py b/.github/scripts/differential-tests-error-out-on-failures.py index 75530a453a6e5..85b0c8d187e3a 100644 --- a/.github/scripts/differential-tests-error-out-on-failures.py +++ b/.github/scripts/differential-tests-error-out-on-failures.py @@ -1,8 +1,6 @@ """ -This script is used to turn the JSON report produced by the revive differential tests tool into an -easy to consume markdown document for the purpose of reporting this information in the Polkadot SDK -CI. The full models used in the JSON report can be found in the revive differential tests repo and -the models used in this script are just a partial reproduction of the full report models. +This script is used to turn the JSON report produced by the revive differential tests, detecting +failures and exiting with the appropriate code if a failure is encountered. """ import json, typing, sys From f86486551f81942c345e8662d47f4de27e98deab Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Fri, 5 Dec 2025 10:48:43 +0300 Subject: [PATCH 5/5] Update the commit hash of the DT framework --- .github/workflows/tests-evm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index e62362e98e5aa..4db614a2a0179 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -51,7 +51,7 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: repository: paritytech/revive-differential-tests - ref: 66feb36b4ef2c79415ca8ea765d8235d48dfa8f8 + ref: cd6b7969acafbd3c29907e543a9751d886df6d00 path: revive-differential-tests submodules: recursive - name: Installing Retester