Skip to content

Commit 1f5e719

Browse files
ehildenbrv-jenkins
ehildenb
authored and
rv-jenkins
committed
Update test submodule to post-Istanbul tests (#590)
* tests/ethereum-tests: update to newest develop * kore-json: do not specify version of python * driver: add postStateHash key * Makefile: rename vm_tests => all_vm_tests, bchain_tests => all_bchain_tests * Makefile: add test-slow-conformance and test-failing-conformance * tests/{failing,slow}.{ocaml,llvm}: clear failing/slow lists * tests/{failing,slow}.{ocaml,llvm}: add slow/failing tests - A test is slow if it takes >120s or >10GB RAM. - A test is failing if it fails before 120s is up. * tests/failing.web3: remove now passing test * kore-json: use sys.stdout.write instead of print * kore-json.py: add back in newline
1 parent 4ea7b3c commit 1f5e719

File tree

9 files changed

+34
-42
lines changed

9 files changed

+34
-42
lines changed

‎Makefile

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ OPAM ?= opam
4444
build build-all build-ocaml build-java build-node build-haskell build-llvm build-web3 \
4545
defn java-defn ocaml-defn node-defn web3-defn haskell-defn llvm-defn \
4646
split-tests \
47-
test test-all test-conformance test-rest-conformance test-all-conformance \
47+
test test-all test-conformance test-rest-conformance test-all-conformance test-slow-conformance test-failing-conformance \
4848
test-vm test-rest-vm test-all-vm test-bchain test-rest-bchain test-all-bchain \
4949
test-web3 test-all-web3 test-failing-web3 \
5050
test-prove test-prove-benchmarks test-prove-functional test-prove-opcodes test-prove-erc20 test-prove-bihu test-prove-examples test-klab-prove \
@@ -453,21 +453,23 @@ failing_conformance_tests = $(shell cat tests/failing.$(TEST_CONCRETE_BACKEND))
453453

454454
test-all-conformance: test-all-vm test-all-bchain
455455
test-rest-conformance: test-rest-vm test-rest-bchain
456+
test-slow-conformance: $(slow_conformance_tests:=.run)
457+
test-failing-conformance: $(failing_conformance_tests:=.run)
456458
test-conformance: test-vm test-bchain
457459

458-
vm_tests = $(wildcard tests/ethereum-tests/VMTests/*/*.json)
459-
quick_vm_tests = $(filter-out $(slow_conformance_tests), $(vm_tests))
460+
all_vm_tests = $(wildcard tests/ethereum-tests/VMTests/*/*.json)
461+
quick_vm_tests = $(filter-out $(slow_conformance_tests), $(all_vm_tests))
460462
passing_vm_tests = $(filter-out $(failing_conformance_tests), $(quick_vm_tests))
461-
rest_vm_tests = $(filter-out $(passing_vm_tests), $(vm_tests))
463+
rest_vm_tests = $(filter-out $(passing_vm_tests), $(all_vm_tests))
462464

463-
test-all-vm: $(vm_tests:=.run)
465+
test-all-vm: $(all_vm_tests:=.run)
464466
test-rest-vm: $(rest_vm_tests:=.run)
465467
test-vm: $(passing_vm_tests:=.run)
466468

467-
bchain_tests = $(wildcard tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/*/*.json)
468-
quick_bchain_tests = $(filter-out $(slow_conformance_tests), $(bchain_tests))
469+
all_bchain_tests = $(wildcard tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/*/*.json)
470+
quick_bchain_tests = $(filter-out $(slow_conformance_tests), $(all_bchain_tests))
469471
passing_bchain_tests = $(filter-out $(failing_conformance_tests), $(quick_bchain_tests))
470-
rest_bchain_tests = $(filter-out $(passing_bchain_tests), $(bchain_tests))
472+
rest_bchain_tests = $(filter-out $(passing_bchain_tests), $(all_bchain_tests))
471473

472474
test-all-bchain: $(all_bchain_tests:=.run)
473475
test-rest-bchain: $(rest_bchain_tests:=.run)

‎driver.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -294,7 +294,7 @@ Note that `TEST` is sorted here so that key `"network"` comes before key `"pre"`
294294
```k
295295
syntax Set ::= "#postKeys" [function] | "#allPostKeys" [function] | "#checkKeys" [function]
296296
// -------------------------------------------------------------------------------------------
297-
rule #postKeys => ( SetItem("post") SetItem("postState") )
297+
rule #postKeys => ( SetItem("post") SetItem("postState") SetItem("postStateHash") )
298298
rule #allPostKeys => ( #postKeys SetItem("expect") SetItem("export") SetItem("expet") )
299299
rule #checkKeys => ( #allPostKeys SetItem("logs") SetItem("out") SetItem("gas")
300300
SetItem("blockHeader") SetItem("transactions") SetItem("uncleHeaders") SetItem("genesisBlockHeader")

‎kore-json.py

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/usr/bin/env python3
1+
#!/usr/bin/env python
22

33
import json
44
import sys
@@ -48,13 +48,14 @@ def print_kast(data, sort="SortJSON"):
4848
raise AssertionError
4949

5050
def print_klabel(s):
51-
print("Lbl" + s.replace("_", "'Unds'").replace("`", "").replace("(.KList)", "{}"), end=' ')
51+
sys.stdout.write("Lbl" + s.replace("_", "'Unds'").replace("`", "").replace("(.KList)", "{}") + " ")
5252

53-
print("LblinitGeneratedTopCell{}(Lbl'Unds'Map'Unds'{}(Lbl'Unds'Map'Unds'{}(Lbl'Unds'Map'Unds'{}(Lbl'Stop'Map{}(),Lbl'UndsPipe'-'-GT-Unds'{}(inj{SortKConfigVar{}, SortKItem{}}(\dv{SortKConfigVar{}}(\"$PGM\")),inj{SortJSON{}, SortKItem{}}(", end=' ')
53+
sys.stdout.write("LblinitGeneratedTopCell{}(Lbl'Unds'Map'Unds'{}(Lbl'Unds'Map'Unds'{}(Lbl'Unds'Map'Unds'{}(Lbl'Stop'Map{}(),Lbl'UndsPipe'-'-GT-Unds'{}(inj{SortKConfigVar{}, SortKItem{}}(\dv{SortKConfigVar{}}(\"$PGM\")),inj{SortJSON{}, SortKItem{}}( ")
5454
print_kast(data)
55-
print("))),Lbl'UndsPipe'-'-GT-Unds'{}(inj{SortKConfigVar{}, SortKItem{}}(\dv{SortKConfigVar{}}(\"$SCHEDULE\")),inj{SortSchedule{}, SortKItem{}}(", end=' ')
55+
sys.stdout.write("))),Lbl'UndsPipe'-'-GT-Unds'{}(inj{SortKConfigVar{}, SortKItem{}}(\dv{SortKConfigVar{}}(\"$SCHEDULE\")),inj{SortSchedule{}, SortKItem{}}( ")
5656
print_klabel(sys.argv[2])
57-
print("()))),Lbl'UndsPipe'-'-GT-Unds'{}(inj{SortKConfigVar{}, SortKItem{}}(\dv{SortKConfigVar{}}(\"$MODE\")),inj{SortMode{}, SortKItem{}}(", end=' ')
57+
sys.stdout.write("()))),Lbl'UndsPipe'-'-GT-Unds'{}(inj{SortKConfigVar{}, SortKItem{}}(\dv{SortKConfigVar{}}(\"$MODE\")),inj{SortMode{}, SortKItem{}}( ")
5858
print_klabel(sys.argv[3])
59-
print("()))))")
60-
print()
59+
sys.stdout.write("()))))\n")
60+
sys.stdout.write("\n")
61+
sys.stdout.flush()

‎tests/ethereum-tests

Submodule ethereum-tests updated 8753 files

‎tests/failing.llvm

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stAttackTest/ContractCreationSpam_d0g0v0.json
21
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreate2/Create2Recursive_d0g0v0.json
32
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreate2/Create2Recursive_d0g1v0.json
3+
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d1g0v0.json
4+
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d2g0v0.json
5+
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Create1000Byzantium_d0g1v0.json
46
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Create1000_d0g1v0.json
57
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stRecursiveCreate/recursiveCreateReturnValue_d0g0v0.json
8+
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call50000bytesContract50_1_d1g0v0.json

‎tests/failing.ocaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stAttackTest/ContractCreationSpam_d0g0v0.json
22
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreate2/Create2Recursive_d0g0v0.json
33
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreate2/Create2Recursive_d0g1v0.json
4+
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d1g0v0.json
5+
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d2g0v0.json
6+
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Create1000Byzantium_d0g1v0.json
7+
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Create1000_d0g1v0.json
48
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stRecursiveCreate/recursiveCreateReturnValue_d0g0v0.json

‎tests/failing.web3

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
tests/web3/erc20/erc20capped.in.json
22
tests/web3/erc20/erc20capped-partial.in.json
33
tests/web3/erc20/erc20detailed.in.json
4-
tests/web3/erc20/erc20detailed-partial.in.json
54
tests/web3/erc20/tokentimelock.in.json
65
tests/web3/erc20/tokentimelock-partial.in.json
76
tests/web3/eth_getBlockByNumber1Tx-2.in.json

‎tests/slow.llvm

Lines changed: 4 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,15 @@
1-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d1g0v0.json
2-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d2g0v0.json
3-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call1MB1024Calldepth_d0g1v0.json
1+
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stAttackTest/ContractCreationSpam_d0g0v0.json
2+
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCodeSizeLimit/codesizeOOGInvalidSize_d0g0v0.json
3+
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCodeSizeLimit/codesizeValid_d0g0v0.json
4+
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_1_d0g0v0.json
45
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_1_d0g1v0.json
56
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_2_d0g0v0.json
67
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_2_d0g1v0.json
78
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_3_d0g0v0.json
89
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_3_d0g1v0.json
910
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_d0g1v0.json
10-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_ecrec_d0g1v0.json
11-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_identity2_d0g1v0.json
12-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_identity_d0g1v0.json
13-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_rip160_d0g1v0.json
1411
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_sha256_d0g1v0.json
1512
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Callcode50000_d0g1v0.json
16-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Create1000Byzantium_d0g1v0.json
1713
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/QuadraticComplexitySolidity_CallDataCopy_d0g1v0.json
1814
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Return50000_2_d0g1v0.json
1915
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Return50000_d0g1v0.json
@@ -22,7 +18,6 @@ tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTest
2218
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stSpecialTest/JUMPDEST_AttackwithJump_d0g0v0.json
2319
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call1MB1024Calldepth_d1g0v0.json
2420
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call50000bytesContract50_1_d0g0v0.json
25-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call50000bytesContract50_1_d1g0v0.json
2621
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call50000bytesContract50_2_d0g0v0.json
2722
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call50000bytesContract50_2_d1g0v0.json
2823
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call50000bytesContract50_3_d0g0v0.json
@@ -43,7 +38,3 @@ tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTest
4338
tests/ethereum-tests/VMTests/vmPerformance/loop-add-10M.json
4439
tests/ethereum-tests/VMTests/vmPerformance/loop-divadd-10M.json
4540
tests/ethereum-tests/VMTests/vmPerformance/loop-divadd-unr100-10M.json
46-
tests/ethereum-tests/VMTests/vmPerformance/loop-exp-1b-1M.json
47-
tests/ethereum-tests/VMTests/vmPerformance/loop-exp-nop-1M.json
48-
tests/ethereum-tests/VMTests/vmPerformance/loop-mul.json
49-
tests/ethereum-tests/VMTests/vmPerformance/loop-mulmod-2M.json

‎tests/slow.ocaml

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d1g0v0.json
2-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d2g0v0.json
3-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call1MB1024Calldepth_d0g1v0.json
1+
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/CREATE_ContractRETURNBigOffset_d0g0v0.json
2+
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_1_d0g0v0.json
43
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_1_d0g1v0.json
54
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_2_d0g0v0.json
65
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000bytesContract50_2_d0g1v0.json
@@ -10,16 +9,11 @@ tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTest
109
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_ecrec_d0g1v0.json
1110
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_identity2_d0g1v0.json
1211
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_identity_d0g1v0.json
13-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_rip160_d0g1v0.json
1412
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Call50000_sha256_d0g1v0.json
1513
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Callcode50000_d0g1v0.json
16-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Create1000Byzantium_d0g1v0.json
17-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Create1000_d0g1v0.json
1814
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/QuadraticComplexitySolidity_CallDataCopy_d0g1v0.json
1915
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Return50000_2_d0g1v0.json
2016
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/Return50000_d0g1v0.json
21-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stSpecialTest/JUMPDEST_Attack_d0g0v0.json
22-
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stSpecialTest/JUMPDEST_AttackwithJump_d0g0v0.json
2317
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call1MB1024Calldepth_d1g0v0.json
2418
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call50000bytesContract50_1_d0g0v0.json
2519
tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/static_Call50000bytesContract50_1_d1g0v0.json
@@ -43,7 +37,5 @@ tests/ethereum-tests/LegacyTests/Constantinople/BlockchainTests/GeneralStateTest
4337
tests/ethereum-tests/VMTests/vmPerformance/loop-add-10M.json
4438
tests/ethereum-tests/VMTests/vmPerformance/loop-divadd-10M.json
4539
tests/ethereum-tests/VMTests/vmPerformance/loop-divadd-unr100-10M.json
46-
tests/ethereum-tests/VMTests/vmPerformance/loop-exp-1b-1M.json
47-
tests/ethereum-tests/VMTests/vmPerformance/loop-exp-nop-1M.json
4840
tests/ethereum-tests/VMTests/vmPerformance/loop-mul.json
4941
tests/ethereum-tests/VMTests/vmPerformance/loop-mulmod-2M.json

0 commit comments

Comments
 (0)