diff --git a/.github/workflows/vm-pr.yml b/.github/workflows/vm-pr.yml index e194581979a..ec5aeec8f30 100644 --- a/.github/workflows/vm-pr.yml +++ b/.github/workflows/vm-pr.yml @@ -116,7 +116,7 @@ jobs: fail-on-cache-miss: true - if: contains(join(github.event.pull_request.labels.*.name, ' '), 'skip most VM') == false - run: npm run test:state -- --fork=${{ matrix.fork }} --verify-test-amount-alltests + run: VITE_FORK=${{matrix.fork}} VITE_VERIFY_TEST_AMOUNT_ALL_TESTS=1 npx vitest run test/tester/stateRunner.spec.ts - if: contains(join(github.event.pull_request.labels.*.name, ' '), 'skip most VM') run: npm run test:buildIntegrity @@ -181,7 +181,7 @@ jobs: fail-on-cache-miss: true - - run: npm run test:state -- --fork=${{ matrix.fork }} --verify-test-amount-alltests + - run: VITE_FORK=${{matrix.fork}} VITE_VERIFY_TEST_AMOUNT_ALL_TESTS=1 npx vitest run test/tester/stateRunner.spec.ts vm-blockchain: runs-on: ubuntu-latest diff --git a/packages/vm/package.json b/packages/vm/package.json index b55041547a1..c112dbe3b0e 100644 --- a/packages/vm/package.json +++ b/packages/vm/package.json @@ -56,10 +56,10 @@ "test:buildIntegrity": "npm run test:state -- --test='stackOverflow'", "test:state": "npm run tester -- --state", "test:state:allForks": "npm run test:state:newForks && test:state:oldForks && test:state:transitionForks", - "test:state:newForks": "echo 'Prague' | xargs -n1 | xargs -I v1 npm run tester -- --state --fork=v1 --verify-test-amount-alltests", - "test:state:oldForks": "echo 'Chainstart Homestead dao TangerineWhistle SpuriousDragon Byzantium Constantinople Petersburg Istanbul MuirGlacier Berlin London Paris Shanghai Cancun' | xargs -n1 | xargs -I v1 npm run tester -- --state --fork=v1 --verify-test-amount-alltests", - "test:state:transitionForks": "echo 'ByzantiumToConstantinopleFixAt5 EIP158ToByzantiumAt5 FrontierToHomesteadAt5 HomesteadToDaoAt5 HomesteadToEIP150At5 BerlinToLondonAt5' | xargs -n1 | xargs -I v1 npm run tester -- --state --fork=v1 --verify-test-amount-alltests", - "test:state:slow": "npm run test:state -- --runSkipped=slow", + "test:state:newForks": "echo 'Prague' | xargs -n1 | xargs -I v1 VITEST_FORK=v1 VITE_VERIFY_TEST_AMOUNT_ALL_TESTS=1 npx vitest test/tester/stateRunner.spec.ts", + "test:state:oldForks": "echo 'Chainstart Homestead dao TangerineWhistle SpuriousDragon Byzantium Constantinople Petersburg Istanbul MuirGlacier Berlin London Paris Shanghai Cancun' | xargs -n1 | xargs -I v1 VITEST_FORK=v1 VITE_VERIFY_TEST_AMOUNT_ALL_TESTS=1 npx vitest test/tester/stateRunner.spec.ts", + "test:state:transitionForks": "echo 'ByzantiumToConstantinopleFixAt5 EIP158ToByzantiumAt5 FrontierToHomesteadAt5 HomesteadToDaoAt5 HomesteadToEIP150At5 BerlinToLondonAt5' | xargs -n1 | xargs -I v1 VITEST_FORK=v1 VITE_VERIFY_TEST_AMOUNT_ALL_TESTS=1 npx vitest test/tester/stateRunner.spec.ts", + "test:state:slow": "VITE_RUN_SKIPPED=slow npx vitest test/tester/stateRunner.spec.ts", "tester": "tsx --conditions=typescript ./test/tester --stack-size=1500", "tsc": "../../config/cli/ts-compile.sh" }, diff --git a/packages/vm/test/tester/config.ts b/packages/vm/test/tester/config.ts index 9621a1cdf8d..f49d7515dcc 100644 --- a/packages/vm/test/tester/config.ts +++ b/packages/vm/test/tester/config.ts @@ -507,7 +507,7 @@ export function getExpectedTests( * @param defaultChoice if to use `NONE` or `ALL` as default choice * @returns array with test names */ -export function getSkipTests(choices: string, defaultChoice: string): string[] { +export function getSkipTests(choices: string | undefined, defaultChoice: string): string[] { let skipTests: string[] = [] if (!choices) { choices = defaultChoice diff --git a/packages/vm/test/tester/runners/GeneralStateTestsRunner.ts b/packages/vm/test/tester/runners/GeneralStateTestsRunner.ts index 1cec47175fb..887086cde99 100644 --- a/packages/vm/test/tester/runners/GeneralStateTestsRunner.ts +++ b/packages/vm/test/tester/runners/GeneralStateTestsRunner.ts @@ -14,12 +14,11 @@ import { import { createVerkleTree } from '@ethereumjs/verkle' import * as verkle from 'micro-eth-signer/verkle.js' -import { createVM, runTx } from '../../../src/index.ts' -import { makeBlockFromEnv, makeTx, setupPreConditions } from '../../util.ts' - import type { StateManagerInterface } from '@ethereumjs/common' import type { VerkleTree } from '@ethereumjs/verkle' import type * as tape from 'tape' +import { createVM, runTx } from '../../../src/index.ts' +import { makeBlockFromEnv, makeTx, setupPreConditions } from '../../util.ts' const loadVerkleCrypto = () => Promise.resolve(verkle) function parseTestCases( @@ -76,7 +75,12 @@ function parseTestCases( return testCases } -async function runTestCase(options: any, testData: any, t: tape.Test) { +function isTape(t: tape.Test | Chai.AssertStatic): t is tape.Test { + // tape.Test has .comment, chai.AssertStatic does not + return typeof (t as tape.Test).comment === 'function' +} + +async function runTestCase(options: any, testData: any, t: tape.Test | Chai.AssertStatic) { const begin = Date.now() // Copy the common object to not create long-lasting // references in memory which might prevent GC @@ -147,7 +151,7 @@ async function runTestCase(options: any, testData: any, t: tape.Test) { opName: e.opcode.name, } - t.comment(JSON.stringify(opTrace)) + isTape(t) && t.comment(JSON.stringify(opTrace)) resolve?.() } @@ -155,7 +159,7 @@ async function runTestCase(options: any, testData: any, t: tape.Test) { const stateRoot = { stateRoot: bytesToHex(await vm.stateManager.getStateRoot()), } - t.comment(JSON.stringify(stateRoot)) + isTape(t) && t.comment(JSON.stringify(stateRoot)) resolve?.() } @@ -189,7 +193,12 @@ async function runTestCase(options: any, testData: any, t: tape.Test) { const end = Date.now() const timeSpent = `${(end - begin) / 1000} secs` - t.ok(stateRootsAreEqual, `[ ${timeSpent} ] the state roots should match (${execInfo})`) + const msg = `error running test case for fork: ${options.forkConfigTestSuite}` + if (isTape(t)) { + t.ok(stateRootsAreEqual, `[ ${timeSpent} ] the state roots should match (${execInfo})`) + } else { + t.deepEqual(stateManagerStateRoot, testDataPostStateRoot, msg) + } vm.evm.events!.removeListener('step', stepHandler) vm.events.removeListener('afterTx', afterTxHandler) @@ -199,32 +208,29 @@ async function runTestCase(options: any, testData: any, t: tape.Test) { return parseFloat(timeSpent) } -export async function runStateTest(options: any, testData: any, t: tape.Test) { - try { - const testCases = parseTestCases( - options.forkConfigTestSuite, - testData, - options.data, - options.gasLimit, - options.value, - ) - if (testCases.length === 0) { - t.comment(`No ${options.forkConfigTestSuite} post state defined, skip test`) - return - } - for (const testCase of testCases) { - if (options.reps !== undefined && options.reps > 0) { - let totalTimeSpent = 0 - for (let x = 0; x < options.reps; x++) { - totalTimeSpent += await runTestCase(options, testCase, t) - } - t.comment(`Average test run: ${(totalTimeSpent / options.reps).toLocaleString()} s`) - } else { - await runTestCase(options, testCase, t) +export async function runStateTest(options: any, testData: any, t: tape.Test | Chai.AssertStatic) { + const testCases = parseTestCases( + options.forkConfigTestSuite, + testData, + options.data, + options.gasLimit, + options.value, + ) + if (testCases.length === 0) { + isTape(t) && t.comment(`No ${options.forkConfigTestSuite} post state defined, skip test`) + return + } + for (const testCase of testCases) { + if (options.reps !== undefined && options.reps > 0) { + let totalTimeSpent = 0 + for (let x = 0; x < options.reps; x++) { + totalTimeSpent += await runTestCase(options, testCase, t) } + isTape(t) && + t.comment(`Average test run: ${(totalTimeSpent / options.reps).toLocaleString()} s`) + } else { + await runTestCase(options, testCase, t) + options.testCount++ } - } catch (e: any) { - console.log(e) - t.fail(`error running test case for fork: ${options.forkConfigTestSuite}`) } } diff --git a/packages/vm/test/tester/stateRunner.spec.ts b/packages/vm/test/tester/stateRunner.spec.ts new file mode 100644 index 00000000000..a224afa6435 --- /dev/null +++ b/packages/vm/test/tester/stateRunner.spec.ts @@ -0,0 +1,253 @@ +import type { Common } from '@ethereumjs/common' + +import { trustedSetup } from '@paulmillr/trusted-setups/fast.js' +import * as mcl from 'mcl-wasm' +import { assert, afterAll, describe, it } from 'vitest' + +import { KZG as microEthKZG } from 'micro-eth-signer/kzg' + +import path from 'path' +import { + type EVMBLSInterface, + type EVMBN254Interface, + MCLBLS, + NobleBLS, + NobleBN254, + RustBN254, +} from '@ethereumjs/evm' +import { initRustBN } from 'rustbn-wasm' +import { + DEFAULT_FORK_CONFIG, + DEFAULT_TESTS_PATH, + getCommon, + getExpectedTests, + getRequiredForkConfigAlias, + getSkipTests, + getTestDirs, +} from './config.ts' +import { runStateTest } from './runners/GeneralStateTestsRunner.ts' +import { getTestsFromArgs } from './testLoader.ts' + +// use VITE_ as prefix for env arguments +const argv: { + fork?: string + bls?: string + bn254?: string + stateManager?: string + forkConfig?: string + file?: string + test?: string + dir?: string + excludeDir?: string + testsPath?: string + customStateTest?: string + directory?: string + skip?: string + skipTests?: string + runSkipped?: string + customTestsPath?: string + data?: number + gas?: number + value?: number + reps?: number + verifyTestAmountAllTests?: number + expectedTestAmount?: number + debug?: boolean + profile?: boolean + jsontrace?: boolean + dist?: boolean +} = { + // string flags + fork: process.env.VITE_FORK, + bls: process.env.VITE_BLS, + bn254: process.env.VITE_BN254, + stateManager: process.env.VITE_STATE_MANAGER, + forkConfig: process.env.VITE_FORK_CONFIG, + file: process.env.VITE_FILE, + test: process.env.VITE_TEST, + dir: process.env.VITE_DIR, + excludeDir: process.env.VITE_EXCLUDE_DIR, + testsPath: process.env.VITE_TESTS_PATH, + customStateTest: process.env.VITE_CUSTOM_STATE_TEST, + directory: process.env.VITE_DIRECTORY, + skip: process.env.VITE_SKIP, + customTestsPath: process.env.VITE_CUSTOM_TESTS_PATH, + + // boolean flags + jsontrace: process.env.VITE_JSONTRACE === 'true', + dist: process.env.VITE_DIST === 'true', + debug: process.env.VITE_DEBUG === 'true', + profile: process.env.VITE_PROFILE === 'true', + + // numeric flags + data: process.env.VITE_DATA !== undefined ? Number(process.env.VITE_DATA) : undefined, + gas: process.env.VITE_GAS !== undefined ? Number(process.env.VITE_GAS) : undefined, + value: process.env.VITE_VALUE !== undefined ? Number(process.env.VITE_VALUE) : undefined, + reps: process.env.VITE_REPS !== undefined ? Number(process.env.VITE_REPS) : undefined, + verifyTestAmountAllTests: + process.env.VITE_VERIFY_TEST_AMOUNT_ALL_TESTS !== undefined + ? Number(process.env.VITE_VERIFY_TEST_AMOUNT_ALL_TESTS) + : undefined, + expectedTestAmount: + process.env.VITE_EXPECTED_TEST_AMOUNT !== undefined + ? Number(process.env.VITE_EXPECTED_TEST_AMOUNT) + : undefined, + + // array flags + skipTests: process.env.VITE_SKIP_TESTS, + runSkipped: process.env.VITE_RUN_SKIPPED, +} + +const RUN_PROFILER: boolean = argv.profile ?? false +const FORK_CONFIG: string = argv.fork ?? DEFAULT_FORK_CONFIG +const FORK_CONFIG_TEST_SUITE = getRequiredForkConfigAlias(FORK_CONFIG) + +// Examples: Istanbul -> istanbul, MuirGlacier -> muirGlacier +const FORK_CONFIG_VM = FORK_CONFIG.charAt(0).toLowerCase() + FORK_CONFIG.substring(1) + +let bls: EVMBLSInterface +if (argv.bls !== undefined && argv.bls.toLowerCase() === 'mcl') { + await mcl.init(mcl.BLS12_381) + bls = new MCLBLS(mcl) + console.log('BLS library used: MCL (WASM)') +} else { + console.log('BLS library used: Noble (JavaScript)') + bls = new NobleBLS() +} + +let bn254: EVMBN254Interface +if (argv.bn254 !== undefined && argv.bn254.toLowerCase() === 'mcl') { + const rustBN = await initRustBN() + bn254 = new RustBN254(rustBN) + console.log('BN254 (alt_BN128) library used: rustbn.js (WASM)') +} else { + console.log('BN254 (alt_BN128) library used: Noble (JavaScript)') + bn254 = new NobleBN254() +} + +const kzg = new microEthKZG(trustedSetup) +const runnerArgs: { + forkConfigVM: string + forkConfigTestSuite: string + common: Common + jsontrace?: boolean + dist?: boolean + data?: number + gasLimit?: number + value?: number + debug?: boolean + reps?: number + profile: boolean + bls: EVMBLSInterface + bn254: EVMBN254Interface + stateManager?: string + testCount: number +} = { + forkConfigVM: FORK_CONFIG_VM, + forkConfigTestSuite: FORK_CONFIG_TEST_SUITE, + common: getCommon(FORK_CONFIG_VM, kzg), + jsontrace: argv.jsontrace, + dist: argv.dist, + data: argv.data, // GeneralStateTests + gasLimit: argv.gas, // GeneralStateTests + value: argv.value, // GeneralStateTests + debug: argv.debug, // BlockchainTests + reps: argv.reps, // test repetitions + bls, + profile: RUN_PROFILER, + bn254, + stateManager: argv.stateManager, + testCount: 0, +} + +/** + * Configuration for getting the tests from the ethereum/tests repository + */ +const testGetterArgs: { + skipTests: string[] + runSkipped: string[] + forkConfig: string + file?: string + test?: string + dir?: string + excludeDir?: string + testsPath?: string + customStateTest?: string + directory?: string +} = { + skipTests: getSkipTests(argv.skip, argv.runSkipped !== undefined ? 'NONE' : 'ALL'), + runSkipped: getSkipTests(argv.runSkipped, 'NONE'), + forkConfig: FORK_CONFIG_TEST_SUITE, + file: argv.file, + test: argv.test, + dir: argv.dir, + excludeDir: argv.excludeDir, + testsPath: argv.testsPath, + customStateTest: argv.customStateTest, +} + +interface LoadedTest { + dir: string + fileName: string + subDir: string + testName: string + testData: any +} +const allTests: LoadedTest[] = [] +const dirs = getTestDirs(FORK_CONFIG_VM, 'GeneralStateTests') +for (const dir of dirs) { + if (argv.customTestsPath !== undefined) { + testGetterArgs.directory = argv.customTestsPath as string + } else { + const testDir = testGetterArgs.dir ?? '' + const testsPath = testGetterArgs.testsPath ?? DEFAULT_TESTS_PATH + testGetterArgs.directory = path.join(testsPath, dir, testDir) + } + + const tests: LoadedTest[] = [] + try { + await getTestsFromArgs( + dir, + async (fileName: string, subDir: string, testName: string, testData: any) => { + const runSkipped = testGetterArgs.runSkipped + const inRunSkipped = runSkipped.includes(fileName) + if (runSkipped.length === 0 || inRunSkipped === true) { + tests.push({ dir, fileName, subDir, testName, testData }) + } + }, + testGetterArgs, + ) + } catch (e) { + console.log(e) + continue + } + + allTests.push(...tests) +} + +const expectedTests: number | undefined = + argv.verifyTestAmountAllTests !== undefined && argv.verifyTestAmountAllTests > 0 + ? getExpectedTests(FORK_CONFIG_VM, 'GeneralStateTests') + : argv.expectedTestAmount !== undefined && argv.expectedTestAmount > 0 + ? argv.expectedTestAmount + : undefined +describe('GeneralStateTests', () => { + for (const { subDir, testName, testData } of allTests) { + it(`file: ${subDir} test: ${testName}`, async () => { + try { + await runStateTest(runnerArgs, testData, assert) + } catch (e: any) { + assert.fail(e?.toString()) + } + }, 120000) + } + + afterAll(() => { + if (expectedTests !== undefined) { + assert.isTrue( + runnerArgs.testCount >= expectedTests, + `expected ${expectedTests} checks, got ${runnerArgs.testCount}`, + ) + } + }) +}) diff --git a/packages/vm/vitest.config.browser.mts b/packages/vm/vitest.config.browser.mts index feec1b532ee..6438d80f1f8 100644 --- a/packages/vm/vitest.config.browser.mts +++ b/packages/vm/vitest.config.browser.mts @@ -13,6 +13,8 @@ export default mergeConfig( 'test/api/EIPs/eip-6800-verkle.spec.ts', // Uses NodeJS builtins and we don't need to fill tests in browser anyway 'test/api/t8ntool/t8ntool.spec.ts', + // test runners are ran only in ci + 'test/tester/stateRunner.spec.ts' ], }, }), diff --git a/packages/vm/vitest.config.coverage.ts b/packages/vm/vitest.config.coverage.ts index 37a90957435..f35caa7002d 100644 --- a/packages/vm/vitest.config.coverage.ts +++ b/packages/vm/vitest.config.coverage.ts @@ -13,5 +13,6 @@ export default defineConfig({ all: true, reporter: ['lcov'], }, + exclude: ['test/tester/stateRunner.spec.ts'], }, })