From 1d714325f79bc7bee472545b577e2034770fdca4 Mon Sep 17 00:00:00 2001 From: bhartnett Date: Thu, 7 Aug 2025 14:41:18 +0800 Subject: [PATCH 01/11] Disable state root checks. --- execution_chain/stateless/witness_generation.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/execution_chain/stateless/witness_generation.nim b/execution_chain/stateless/witness_generation.nim index 057c541ff6..3466a40814 100644 --- a/execution_chain/stateless/witness_generation.nim +++ b/execution_chain/stateless/witness_generation.nim @@ -80,9 +80,10 @@ proc build*( preStateLedger: LedgerRef, ledger: LedgerRef, parent: Header, - header: Header): T = + header: Header, + validateStateRoot = false): T = - if parent.number > 0: + if validateStateRoot and parent.number > 0: doAssert preStateLedger.getStateRoot() == parent.stateRoot var witness = Witness.build(ledger.getWitnessKeys(), preStateLedger) From 7d32b1dff8e154a0e41fb654df81b9e39fdd8899 Mon Sep 17 00:00:00 2001 From: bhartnett Date: Fri, 8 Aug 2025 12:33:03 +0800 Subject: [PATCH 02/11] Append rlp bytes directly to seq of trie nodes. --- .../db/aristo/aristo_desc/desc_structural.nim | 2 +- execution_chain/db/aristo/aristo_proof.nim | 2 +- execution_chain/db/aristo/aristo_serialise.nim | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/execution_chain/db/aristo/aristo_desc/desc_structural.nim b/execution_chain/db/aristo/aristo_desc/desc_structural.nim index 85fa93c3b4..76684edaa1 100644 --- a/execution_chain/db/aristo/aristo_desc/desc_structural.nim +++ b/execution_chain/db/aristo/aristo_desc/desc_structural.nim @@ -70,7 +70,7 @@ type ## Combined record for a *traditional* ``Merkle Patricia Tree` node merged ## with a structural `VertexRef` type object. vtx*: VertexRef - key*: array[16,HashKey] ## Merkle hash/es for vertices + key*: array[16, HashKey] ## Merkle hash/es for vertices # ---------------------- diff --git a/execution_chain/db/aristo/aristo_proof.nim b/execution_chain/db/aristo/aristo_proof.nim index ca8489ddf0..46dde6cfbe 100644 --- a/execution_chain/db/aristo/aristo_proof.nim +++ b/execution_chain/db/aristo/aristo_proof.nim @@ -46,7 +46,7 @@ proc chainRlpNodes( return err(PartChnNodeConvError) # Save rpl encoded node(s) - chain &= node.to(seq[seq[byte]]) + node.appendRlpBytesTo(chain) # Follow up child node case vtx.vType: diff --git a/execution_chain/db/aristo/aristo_serialise.nim b/execution_chain/db/aristo/aristo_serialise.nim index 79f42702a6..f447b3060b 100644 --- a/execution_chain/db/aristo/aristo_serialise.nim +++ b/execution_chain/db/aristo/aristo_serialise.nim @@ -27,7 +27,7 @@ proc toRlpBytes*(acc: AristoAccount, key: HashKey): seq[byte] = codeHash: acc.codeHash, ) -proc to*(node: NodeRef, T: type seq[seq[byte]]): T = +proc appendRlpBytesTo*(node: NodeRef, chain: var seq[seq[byte]]) = ## Convert the argument pait `w` to a single or a double item list item of ## `` type entries. Only in case of a combined extension ## and branch vertex argument, there will be a double item list result. @@ -51,11 +51,11 @@ proc to*(node: NodeRef, T: type seq[seq[byte]]): T = wrx.append node.vtx.pfx.toHexPrefix(isleaf = false).data() wrx.append brHash - result.add wrx.finish() - result.add brData + chain.add wrx.finish() + chain.add brData else: # Do for pure branch node - result.add brData + chain.add brData of AccLeaf: let vtx = AccLeafRef(node.vtx) var wr = initRlpWriter() @@ -63,7 +63,7 @@ proc to*(node: NodeRef, T: type seq[seq[byte]]): T = wr.append vtx.pfx.toHexPrefix(isleaf = true).data() wr.append vtx.account.toRlpBytes(node.key[0]) - result.add (wr.finish()) + chain.add (wr.finish()) of StoLeaf: let vtx = StoLeafRef(node.vtx) var wr = initRlpWriter() @@ -71,7 +71,7 @@ proc to*(node: NodeRef, T: type seq[seq[byte]]): T = wr.append vtx.pfx.toHexPrefix(isleaf = true).data() wr.append rlp.encode vtx.stoData - result.add (wr.finish()) + chain.add (wr.finish()) proc digestTo*(node: NodeRef; T: type HashKey): T = ## Convert the argument `node` to the corresponding Merkle hash key. Note From 93a46a4424398df427db37d51149548a05afc3ea Mon Sep 17 00:00:00 2001 From: bhartnett Date: Fri, 8 Aug 2025 15:24:37 +0800 Subject: [PATCH 03/11] Cache rlp trie nodes when building multiple slot proofs. --- execution_chain/db/aristo/aristo_desc.nim | 16 ++--- execution_chain/db/aristo/aristo_proof.nim | 72 ++++++++++++++----- .../db/aristo/aristo_serialise.nim | 12 ++-- execution_chain/db/aristo/aristo_utils.nim | 2 +- execution_chain/db/core_db/base.nim | 8 +-- execution_chain/db/ledger.nim | 27 +++---- 6 files changed, 83 insertions(+), 54 deletions(-) diff --git a/execution_chain/db/aristo/aristo_desc.nim b/execution_chain/db/aristo/aristo_desc.nim index 3ad8182c47..32557daa35 100644 --- a/execution_chain/db/aristo/aristo_desc.nim +++ b/execution_chain/db/aristo/aristo_desc.nim @@ -61,17 +61,17 @@ type ## `sTab[]` tables must correspond to a hash entry held on the `kMap[]` ## tables. So a corresponding zero value or missing entry produces an ## inconsistent state that must be resolved. - db*: AristoDbRef ## Database descriptor - parent*: AristoTxRef ## Previous transaction + db*: AristoDbRef ## Database descriptor + parent*: AristoTxRef ## Previous transaction - sTab*: Table[RootedVertexID,VertexRef] ## Structural vertex table - kMap*: Table[RootedVertexID,HashKey] ## Merkle hash key mapping - vTop*: VertexID ## Last used vertex ID + sTab*: Table[RootedVertexID, VertexRef] ## Structural vertex table + kMap*: Table[RootedVertexID, HashKey] ## Merkle hash key mapping + vTop*: VertexID ## Last used vertex ID - accLeaves*: Table[Hash32, AccLeafRef] ## Account path -> VertexRef - stoLeaves*: Table[Hash32, StoLeafRef] ## Storage path -> VertexRef + accLeaves*: Table[Hash32, AccLeafRef] ## Account path -> VertexRef + stoLeaves*: Table[Hash32, StoLeafRef] ## Storage path -> VertexRef - blockNumber*: Opt[uint64] ## Block number set when checkpointing the frame + blockNumber*: Opt[uint64] ## Block number set when checkpointing the frame snapshot*: Snapshot ## Optional snapshot containing the cumulative changes from ancestors and diff --git a/execution_chain/db/aristo/aristo_proof.nim b/execution_chain/db/aristo/aristo_proof.nim index 46dde6cfbe..271e79815d 100644 --- a/execution_chain/db/aristo/aristo_proof.nim +++ b/execution_chain/db/aristo/aristo_proof.nim @@ -11,6 +11,7 @@ ## Aristo DB -- Create and verify MPT proofs ## =========================================================== ## + {.push raises: [].} import @@ -33,20 +34,29 @@ const ## This is the opposite of `ChainRlpNodesNoEntry` when verifying that a ## node does not exist. +type + NodesCache = Table[RootedVertexID, seq[seq[byte]]] + ## Caches up to two rlp encoded trie nodes in each value + proc chainRlpNodes( - db: AristoTxRef; - rvid: RootedVertexID; + db: AristoTxRef, + rvid: RootedVertexID, path: NibblesBuf, - chain: var seq[seq[byte]]; - ): Result[void,AristoError] = + chain: var seq[seq[byte]], + nodesCache: var NodesCache): Result[void, AristoError] = ## Inspired by the `getBranchAux()` function from `hexary.nim` - let - (vtx,_) = ? db.getVtxRc rvid - node = vtx.toNode(rvid.root, db).valueOr: + let (vtx, _) = ?db.getVtxRc(rvid) + + nodesCache.withValue(rvid, value): + chain &= value[] + do: + let node = vtx.toNode(rvid.root, db).valueOr: return err(PartChnNodeConvError) - # Save rpl encoded node(s) - node.appendRlpBytesTo(chain) + # Save rpl encoded node(s) + let rlpNodes = node.to(seq[seq[byte]]) + nodesCache[rvid] = rlpNodes + chain &= rlpNodes # Follow up child node case vtx.vType: @@ -70,7 +80,7 @@ proc chainRlpNodes( if not vtx.bVid(nibble).isValid: return err(PartChnBranchVoidEdge) # Recursion! - db.chainRlpNodes((rvid.root,vtx.bVid(nibble)), rest, chain) + db.chainRlpNodes((rvid.root,vtx.bVid(nibble)), rest, chain, nodesCache) proc trackRlpNodes( @@ -78,7 +88,7 @@ proc trackRlpNodes( topKey: HashKey; path: NibblesBuf; start = false; - ): Result[seq[byte],AristoError] + ): Result[seq[byte], AristoError] {.gcsafe, raises: [RlpError]} = ## Verify rlp-encoded node chain created by `chainRlpNodes()`. if path.len == 0: @@ -126,7 +136,8 @@ proc makeProof( db: AristoTxRef; root: VertexID; path: NibblesBuf; - ): Result[(seq[seq[byte]],bool), AristoError] = + nodesCache: var NodesCache; + ): Result[(seq[seq[byte]], bool), AristoError] = ## This function returns a chain of rlp-encoded nodes along the argument ## path `(root,path)` followed by a `true` value if the `path` argument ## exists in the database. If the argument `path` is not on the database, @@ -135,7 +146,7 @@ proc makeProof( ## Errors will only be returned for invalid paths. ## var chain: seq[seq[byte]] - let rc = db.chainRlpNodes((root,root), path, chain) + let rc = db.chainRlpNodes((root,root), path, chain, nodesCache) if rc.isOk: ok((chain, true)) elif rc.error in ChainRlpNodesNoEntry: @@ -146,21 +157,46 @@ proc makeProof( proc makeAccountProof*( db: AristoTxRef; accPath: Hash32; - ): Result[(seq[seq[byte]],bool), AristoError] = - db.makeProof(STATE_ROOT_VID, NibblesBuf.fromBytes accPath.data) + ): Result[(seq[seq[byte]], bool), AristoError] = + var nodesCache: NodesCache + db.makeProof(STATE_ROOT_VID, NibblesBuf.fromBytes accPath.data, nodesCache) proc makeStorageProof*( db: AristoTxRef; accPath: Hash32; stoPath: Hash32; - ): Result[(seq[seq[byte]],bool), AristoError] = + ): Result[(seq[seq[byte]], bool), AristoError] = ## Note that the function returns an error unless ## the argument `accPath` is valid. let vid = db.fetchStorageID(accPath).valueOr: if error == FetchPathStoRootMissing: return ok((@[],false)) return err(error) - db.makeProof(vid, NibblesBuf.fromBytes stoPath.data) + var nodesCache: NodesCache + db.makeProof(vid, NibblesBuf.fromBytes stoPath.data, nodesCache) + +proc makeStorageProofs*( + db: AristoTxRef; + accPath: Hash32; + stoPaths: openArray[Hash32]; + ): Result[seq[seq[seq[byte]]], AristoError] = + ## Note that the function returns an error unless + ## the argument `accPath` is valid. + let vid = db.fetchStorageID(accPath).valueOr: + if error == FetchPathStoRootMissing: + let emptyProofs = newSeq[seq[seq[byte]]](stoPaths.len()) + return ok(emptyProofs) + return err(error) + + var + nodesCache: NodesCache + proofs = newSeqOfCap[seq[seq[byte]]](stoPaths.len()) + + for stoPath in stoPaths: + let (proof, _) = ?db.makeProof(vid, NibblesBuf.fromBytes stoPath.data, nodesCache) + proofs.add(proof) + + ok(proofs) # ---------- @@ -168,7 +204,7 @@ proc verifyProof*( chain: openArray[seq[byte]]; root: Hash32; path: Hash32; - ): Result[Opt[seq[byte]],AristoError] = + ): Result[Opt[seq[byte]], AristoError] = ## Variant of `partUntwigGeneric()`. try: let diff --git a/execution_chain/db/aristo/aristo_serialise.nim b/execution_chain/db/aristo/aristo_serialise.nim index f447b3060b..79f42702a6 100644 --- a/execution_chain/db/aristo/aristo_serialise.nim +++ b/execution_chain/db/aristo/aristo_serialise.nim @@ -27,7 +27,7 @@ proc toRlpBytes*(acc: AristoAccount, key: HashKey): seq[byte] = codeHash: acc.codeHash, ) -proc appendRlpBytesTo*(node: NodeRef, chain: var seq[seq[byte]]) = +proc to*(node: NodeRef, T: type seq[seq[byte]]): T = ## Convert the argument pait `w` to a single or a double item list item of ## `` type entries. Only in case of a combined extension ## and branch vertex argument, there will be a double item list result. @@ -51,11 +51,11 @@ proc appendRlpBytesTo*(node: NodeRef, chain: var seq[seq[byte]]) = wrx.append node.vtx.pfx.toHexPrefix(isleaf = false).data() wrx.append brHash - chain.add wrx.finish() - chain.add brData + result.add wrx.finish() + result.add brData else: # Do for pure branch node - chain.add brData + result.add brData of AccLeaf: let vtx = AccLeafRef(node.vtx) var wr = initRlpWriter() @@ -63,7 +63,7 @@ proc appendRlpBytesTo*(node: NodeRef, chain: var seq[seq[byte]]) = wr.append vtx.pfx.toHexPrefix(isleaf = true).data() wr.append vtx.account.toRlpBytes(node.key[0]) - chain.add (wr.finish()) + result.add (wr.finish()) of StoLeaf: let vtx = StoLeafRef(node.vtx) var wr = initRlpWriter() @@ -71,7 +71,7 @@ proc appendRlpBytesTo*(node: NodeRef, chain: var seq[seq[byte]]) = wr.append vtx.pfx.toHexPrefix(isleaf = true).data() wr.append rlp.encode vtx.stoData - chain.add (wr.finish()) + result.add (wr.finish()) proc digestTo*(node: NodeRef; T: type HashKey): T = ## Convert the argument `node` to the corresponding Merkle hash key. Note diff --git a/execution_chain/db/aristo/aristo_utils.nim b/execution_chain/db/aristo/aristo_utils.nim index 0a59a17936..61f700f524 100644 --- a/execution_chain/db/aristo/aristo_utils.nim +++ b/execution_chain/db/aristo/aristo_utils.nim @@ -25,7 +25,7 @@ proc toNode*( vtx: VertexRef; # Vertex to convert root: VertexID; # Sub-tree root the `vtx` belongs to db: AristoTxRef; # Database - ): Result[NodeRef,seq[VertexID]] = + ): Result[NodeRef, seq[VertexID]] = ## Convert argument the vertex `vtx` to a node type. Missing Merkle hash ## keys are searched for on the argument database `db`. ## diff --git a/execution_chain/db/core_db/base.nim b/execution_chain/db/core_db/base.nim index 69a25bd396..ad43693c10 100644 --- a/execution_chain/db/core_db/base.nim +++ b/execution_chain/db/core_db/base.nim @@ -309,11 +309,11 @@ proc getStateRoot*(acc: CoreDbTxRef): CoreDbRc[Hash32] = # ------------ storage --------------- -proc slotProof*( +proc slotProofs*( acc: CoreDbTxRef; accPath: Hash32; - stoPath: Hash32; - ): CoreDbRc[(seq[seq[byte]],bool)] = + stoPaths: openArray[Hash32]; + ): CoreDbRc[seq[seq[seq[byte]]]] = ## On the storage MPT related to the argument account `acPath`, collect the ## nodes along the `stoPath` interpreted as path. Return these path nodes as ## a chain of rlp-encoded blobs followed by a bool value which is `true` if @@ -324,7 +324,7 @@ proc slotProof*( ## Note that the function always returns an error unless the `accPath` is ## valid. ## - let rc = acc.aTx.makeStorageProof(accPath, stoPath).valueOr: + let rc = acc.aTx.makeStorageProofs(accPath, stoPaths).valueOr: return err(error.toError("", ProofCreate)) ok(rc) diff --git a/execution_chain/db/ledger.nim b/execution_chain/db/ledger.nim index cbc356a0f3..3058b21f5e 100644 --- a/execution_chain/db/ledger.nim +++ b/execution_chain/db/ledger.nim @@ -886,30 +886,23 @@ proc getAccountProof*(ac: LedgerRef, address: Address): seq[seq[byte]] = accProof[0] proc getStorageProof*(ac: LedgerRef, address: Address, slots: openArray[UInt256]): seq[seq[seq[byte]]] = - var storageProof = newSeqOfCap[seq[seq[byte]]](slots.len) - let addressHash = address.toAccountKey accountExists = ac.txFrame.hasPath(addressHash).valueOr: raiseAssert "Call to hasPath failed: " & $$error + if not accountExists: + let emptyProofs = newSeq[seq[seq[byte]]](slots.len) + return emptyProofs + + var slotKeys: seq[Hash32] for slot in slots: - if not accountExists: - storageProof.add(@[]) - continue + let slotKey = ac.slots.get(slot).valueOr: + slot.toBytesBE().keccak256() + slotKeys.add(slotKey) - let - slotKey = ac.slots.get(slot).valueOr: - slot.toBytesBE.keccak256 - slotProof = ac.txFrame.slotProof(addressHash, slotKey).valueOr: - if error.aErr == FetchPathNotFound: - storageProof.add(@[]) - continue - else: - raiseAssert "Failed to get slot proof: " & $$error - storageProof.add(slotProof[0]) - - storageProof + ac.txFrame.slotProofs(addressHash, slotKeys).valueOr: + raiseAssert "Failed to get slot proof: " & $$error # ------------------------------------------------------------------------------ # Public virtual read-only methods From 6883ef4de270f507cfe6b800a1fa802a7ec0738a Mon Sep 17 00:00:00 2001 From: bhartnett Date: Fri, 8 Aug 2025 21:33:44 +0800 Subject: [PATCH 04/11] Use array instead of seq as NodesCache value. --- execution_chain/db/aristo/aristo_proof.nim | 13 +++++++++---- execution_chain/db/aristo/aristo_serialise.nim | 11 +++++------ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/execution_chain/db/aristo/aristo_proof.nim b/execution_chain/db/aristo/aristo_proof.nim index 271e79815d..45541a60ec 100644 --- a/execution_chain/db/aristo/aristo_proof.nim +++ b/execution_chain/db/aristo/aristo_proof.nim @@ -35,9 +35,14 @@ const ## node does not exist. type - NodesCache = Table[RootedVertexID, seq[seq[byte]]] + NodesCache = Table[RootedVertexID, array[2, seq[byte]]] ## Caches up to two rlp encoded trie nodes in each value +template appendNodes(chain: var seq[seq[byte]], nodePair: array[2, seq[byte]]) = + chain.add(nodePair[0]) + if nodePair[1].len() > 0: + chain.add(nodePair[1]) + proc chainRlpNodes( db: AristoTxRef, rvid: RootedVertexID, @@ -48,15 +53,15 @@ proc chainRlpNodes( let (vtx, _) = ?db.getVtxRc(rvid) nodesCache.withValue(rvid, value): - chain &= value[] + chain.appendNodes(value[]) do: let node = vtx.toNode(rvid.root, db).valueOr: return err(PartChnNodeConvError) # Save rpl encoded node(s) - let rlpNodes = node.to(seq[seq[byte]]) + let rlpNodes = node.to(array[2, seq[byte]]) nodesCache[rvid] = rlpNodes - chain &= rlpNodes + chain.appendNodes(rlpNodes) # Follow up child node case vtx.vType: diff --git a/execution_chain/db/aristo/aristo_serialise.nim b/execution_chain/db/aristo/aristo_serialise.nim index 79f42702a6..8f1a391628 100644 --- a/execution_chain/db/aristo/aristo_serialise.nim +++ b/execution_chain/db/aristo/aristo_serialise.nim @@ -27,7 +27,7 @@ proc toRlpBytes*(acc: AristoAccount, key: HashKey): seq[byte] = codeHash: acc.codeHash, ) -proc to*(node: NodeRef, T: type seq[seq[byte]]): T = +proc to*(node: NodeRef, T: type array[2, seq[byte]]): T = ## Convert the argument pait `w` to a single or a double item list item of ## `` type entries. Only in case of a combined extension ## and branch vertex argument, there will be a double item list result. @@ -51,11 +51,10 @@ proc to*(node: NodeRef, T: type seq[seq[byte]]): T = wrx.append node.vtx.pfx.toHexPrefix(isleaf = false).data() wrx.append brHash - result.add wrx.finish() - result.add brData + [wrx.finish(), brData] else: # Do for pure branch node - result.add brData + [brData, @[]] of AccLeaf: let vtx = AccLeafRef(node.vtx) var wr = initRlpWriter() @@ -63,7 +62,7 @@ proc to*(node: NodeRef, T: type seq[seq[byte]]): T = wr.append vtx.pfx.toHexPrefix(isleaf = true).data() wr.append vtx.account.toRlpBytes(node.key[0]) - result.add (wr.finish()) + [wr.finish(), @[]] of StoLeaf: let vtx = StoLeafRef(node.vtx) var wr = initRlpWriter() @@ -71,7 +70,7 @@ proc to*(node: NodeRef, T: type seq[seq[byte]]): T = wr.append vtx.pfx.toHexPrefix(isleaf = true).data() wr.append rlp.encode vtx.stoData - result.add (wr.finish()) + [wr.finish(), @[]] proc digestTo*(node: NodeRef; T: type HashKey): T = ## Convert the argument `node` to the corresponding Merkle hash key. Note From fe4b3fc9452c34bf7dfaa2c00b36cc6950d2aebb Mon Sep 17 00:00:00 2001 From: bhartnett Date: Sat, 9 Aug 2025 21:29:58 +0800 Subject: [PATCH 05/11] Implement multiproof function. --- execution_chain/db/aristo/aristo_proof.nim | 51 ++++++++++++------- execution_chain/db/core_db/base.nim | 13 +++++ .../stateless/witness_generation.nim | 36 +++++-------- 3 files changed, 59 insertions(+), 41 deletions(-) diff --git a/execution_chain/db/aristo/aristo_proof.nim b/execution_chain/db/aristo/aristo_proof.nim index 45541a60ec..eb8765f9ff 100644 --- a/execution_chain/db/aristo/aristo_proof.nim +++ b/execution_chain/db/aristo/aristo_proof.nim @@ -15,15 +15,11 @@ {.push raises: [].} import + std/[tables, sets, sequtils], eth/common/hashes, results, ./[aristo_desc, aristo_fetch, aristo_get, aristo_serialise, aristo_utils] -# ------------------------------------------------------------------------------ -# Public functions -# ------------------------------------------------------------------------------ - - const ChainRlpNodesNoEntry* = { PartChnLeafPathMismatch, PartChnExtPfxMismatch, PartChnBranchVoidEdge} @@ -133,10 +129,6 @@ proc trackRlpNodes( return err(PartTrkLinkExpected) chain.toOpenArray(1,chain.len-1).trackRlpNodes(nextKey, path.slice nChewOff) -# ------------------------------------------------------------------------------ -# Public functions -# ------------------------------------------------------------------------------ - proc makeProof( db: AristoTxRef; root: VertexID; @@ -180,10 +172,11 @@ proc makeStorageProof*( var nodesCache: NodesCache db.makeProof(vid, NibblesBuf.fromBytes stoPath.data, nodesCache) -proc makeStorageProofs*( +proc makeStorageProofs( db: AristoTxRef; accPath: Hash32; stoPaths: openArray[Hash32]; + nodesCache: var NodesCache; ): Result[seq[seq[seq[byte]]], AristoError] = ## Note that the function returns an error unless ## the argument `accPath` is valid. @@ -193,17 +186,41 @@ proc makeStorageProofs*( return ok(emptyProofs) return err(error) - var - nodesCache: NodesCache - proofs = newSeqOfCap[seq[seq[byte]]](stoPaths.len()) - + var proofs = newSeqOfCap[seq[seq[byte]]](stoPaths.len()) for stoPath in stoPaths: let (proof, _) = ?db.makeProof(vid, NibblesBuf.fromBytes stoPath.data, nodesCache) proofs.add(proof) ok(proofs) -# ---------- +proc makeStorageProofs*( + db: AristoTxRef; + accPath: Hash32; + stoPaths: openArray[Hash32]; + ): Result[seq[seq[seq[byte]]], AristoError] = + var nodesCache: NodesCache + makeStorageProofs(db, accPath, stoPaths, nodesCache) + +proc makeMultiProof*( + db: AristoTxRef; + paths: Table[Hash32, seq[Hash32]] # maps each account path to a list of storage paths + ): Result[seq[seq[byte]], AristoError] = + var + nodesCache: NodesCache + multiProof: HashSet[seq[byte]] + + for accPath, stoPaths in paths: + let (accProof, _) = ?db.makeProof(STATE_ROOT_VID, NibblesBuf.fromBytes accPath.data, nodesCache) + + for node in accProof: + multiProof.incl(node) + + let storageProofs = ?db.makeStorageProofs(accPath, stoPaths, nodesCache) + for storageProof in storageProofs: + for node in storageProof: + multiProof.incl(node) + + ok(multiProof.toSeq()) proc verifyProof*( chain: openArray[seq[byte]]; @@ -222,7 +239,3 @@ proc verifyProof*( return err(rc.error) except RlpError: return err(PartTrkRlpError) - -# ------------------------------------------------------------------------------ -# End -# ------------------------------------------------------------------------------ diff --git a/execution_chain/db/core_db/base.nim b/execution_chain/db/core_db/base.nim index ad43693c10..011fc3ffc2 100644 --- a/execution_chain/db/core_db/base.nim +++ b/execution_chain/db/core_db/base.nim @@ -307,6 +307,19 @@ proc getStateRoot*(acc: CoreDbTxRef): CoreDbRc[Hash32] = ok(rc) +proc multiProof*( + acc: CoreDbTxRef; + paths: Table[Hash32, seq[Hash32]]; + ): CoreDbRc[seq[seq[byte]]] = + ## Returns a multiproof for every account and storage path specified + ## in the paths table. All rlp-encoded trie nodes from all account + ## and storage proofs are returned in a single list. + + let rc = acc.aTx.makeMultiProof(paths).valueOr: + return err(error.toError("", ProofCreate)) + + ok(rc) + # ------------ storage --------------- proc slotProofs*( diff --git a/execution_chain/stateless/witness_generation.nim b/execution_chain/stateless/witness_generation.nim index 3466a40814..994e04ab64 100644 --- a/execution_chain/stateless/witness_generation.nim +++ b/execution_chain/stateless/witness_generation.nim @@ -27,16 +27,13 @@ proc build*( preStateLedger: LedgerRef): T = var witness = Witness.init() - addedState = initHashSet[seq[byte]]() - addedCodeHashes = initHashSet[Hash32]() + addedCodeHashes: HashSet[Hash32] + proofPaths: Table[Hash32, seq[Hash32]] for key, codeTouched in witnessKeys: if key.slot.isNone(): # Is an account key - witness.addKey(key.address.data()) - - let proof = preStateLedger.getAccountProof(key.address) - for trieNode in proof: - addedState.incl(trieNode) + let addressBytes = key.address.data() + witness.addKey(addressBytes) if codeTouched: let codeHash = preStateLedger.getCodeHash(key.address) @@ -44,23 +41,18 @@ proc build*( witness.addCodeHash(codeHash) addedCodeHashes.incl(codeHash) - # Add the storage slots for this account - var slots: seq[UInt256] + # Add the storage paths for this account + var storagePaths: seq[Hash32] for key2, codeTouched2 in witnessKeys: if key2.address == key.address and key2.slot.isSome(): - let slot = key2.slot.get() - slots.add(slot) - witness.addKey(slot.toBytesBE()) - - if slots.len() > 0: - let proofs = preStateLedger.getStorageProof(key.address, slots) - doAssert(proofs.len() == slots.len()) - for proof in proofs: - for trieNode in proof: - addedState.incl(trieNode) - - for s in addedState.items(): - witness.addState(s) + let slotBytes = key2.slot.get().toBytesBE() + witness.addKey(slotBytes) + storagePaths.add(keccak256(slotBytes)) + + proofPaths[keccak256(addressBytes)] = storagePaths + + witness.state = preStateLedger.txFrame.multiProof(proofPaths).valueOr: + raiseAssert "Failed to get multiproof: " & $$error witness From 46f2e307b0f923e001ca3bb1bd70f533d09c2cca Mon Sep 17 00:00:00 2001 From: bhartnett Date: Sat, 9 Aug 2025 23:05:52 +0800 Subject: [PATCH 06/11] Only fetch storage proofs when account exists. --- execution_chain/db/aristo/aristo_proof.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/execution_chain/db/aristo/aristo_proof.nim b/execution_chain/db/aristo/aristo_proof.nim index eb8765f9ff..b3bb2bfce9 100644 --- a/execution_chain/db/aristo/aristo_proof.nim +++ b/execution_chain/db/aristo/aristo_proof.nim @@ -210,15 +210,15 @@ proc makeMultiProof*( multiProof: HashSet[seq[byte]] for accPath, stoPaths in paths: - let (accProof, _) = ?db.makeProof(STATE_ROOT_VID, NibblesBuf.fromBytes accPath.data, nodesCache) - + let (accProof, exists) = ?db.makeProof(STATE_ROOT_VID, NibblesBuf.fromBytes accPath.data, nodesCache) for node in accProof: multiProof.incl(node) - let storageProofs = ?db.makeStorageProofs(accPath, stoPaths, nodesCache) - for storageProof in storageProofs: - for node in storageProof: - multiProof.incl(node) + if exists: + let storageProofs = ?db.makeStorageProofs(accPath, stoPaths, nodesCache) + for storageProof in storageProofs: + for node in storageProof: + multiProof.incl(node) ok(multiProof.toSeq()) From 3c3e91fe4ba2b9a9881bdae6fc8448222a2ee7b9 Mon Sep 17 00:00:00 2001 From: bhartnett Date: Tue, 12 Aug 2025 19:57:03 +0800 Subject: [PATCH 07/11] Improve performance of witness building when using multiproof using a single loop. --- .../stateless/witness_generation.nim | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/execution_chain/stateless/witness_generation.nim b/execution_chain/stateless/witness_generation.nim index 994e04ab64..2e326f6be3 100644 --- a/execution_chain/stateless/witness_generation.nim +++ b/execution_chain/stateless/witness_generation.nim @@ -26,30 +26,43 @@ proc build*( witnessKeys: WitnessTable, preStateLedger: LedgerRef): T = var - witness = Witness.init() - addedCodeHashes: HashSet[Hash32] proofPaths: Table[Hash32, seq[Hash32]] + addedCodeHashes: HashSet[Hash32] + witness = Witness.init() for key, codeTouched in witnessKeys: + let + addressBytes = key.address.data() + accPath = keccak256(addressBytes) + if key.slot.isNone(): # Is an account key - let addressBytes = key.address.data() - witness.addKey(addressBytes) + witness.addKey(key.address.data()) + + proofPaths.withValue(accPath, v): + discard + do: + proofPaths[accPath] = @[] + # codeTouched is only set for account keys if codeTouched: let codeHash = preStateLedger.getCodeHash(key.address) if codeHash != EMPTY_CODE_HASH and codeHash notin addedCodeHashes: witness.addCodeHash(codeHash) addedCodeHashes.incl(codeHash) - # Add the storage paths for this account - var storagePaths: seq[Hash32] - for key2, codeTouched2 in witnessKeys: - if key2.address == key.address and key2.slot.isSome(): - let slotBytes = key2.slot.get().toBytesBE() - witness.addKey(slotBytes) - storagePaths.add(keccak256(slotBytes)) + else: # Is a slot key + let + slotBytes = key.slot.get().toBytesBE() + slotPath = keccak256(slotBytes) + + witness.addKey(slotBytes) - proofPaths[keccak256(addressBytes)] = storagePaths + proofPaths.withValue(accPath, v): + v[].add(slotPath) + do: + var paths: seq[Hash32] + paths.add(slotPath) + proofPaths[accPath] = paths witness.state = preStateLedger.txFrame.multiProof(proofPaths).valueOr: raiseAssert "Failed to get multiproof: " & $$error From b31c024a7a3da03f416ac6c4e5b2cc4ede59f702 Mon Sep 17 00:00:00 2001 From: bhartnett Date: Wed, 13 Aug 2025 14:13:25 +0800 Subject: [PATCH 08/11] Use var parameters to return proofs to reduce seq copies and create a multiStorageProof function. --- execution_chain/db/aristo/aristo_proof.nim | 75 ++++++++++++------- execution_chain/db/core_db/base.nim | 7 +- .../stateless/witness_generation.nim | 4 +- 3 files changed, 56 insertions(+), 30 deletions(-) diff --git a/execution_chain/db/aristo/aristo_proof.nim b/execution_chain/db/aristo/aristo_proof.nim index b3bb2bfce9..47e19a9bd3 100644 --- a/execution_chain/db/aristo/aristo_proof.nim +++ b/execution_chain/db/aristo/aristo_proof.nim @@ -134,7 +134,8 @@ proc makeProof( root: VertexID; path: NibblesBuf; nodesCache: var NodesCache; - ): Result[(seq[seq[byte]], bool), AristoError] = + chain: var seq[seq[byte]]; + ): Result[bool, AristoError] = ## This function returns a chain of rlp-encoded nodes along the argument ## path `(root,path)` followed by a `true` value if the `path` argument ## exists in the database. If the argument `path` is not on the database, @@ -142,12 +143,11 @@ proc makeProof( ## ## Errors will only be returned for invalid paths. ## - var chain: seq[seq[byte]] let rc = db.chainRlpNodes((root,root), path, chain, nodesCache) if rc.isOk: - ok((chain, true)) + ok(true) elif rc.error in ChainRlpNodesNoEntry: - ok((chain, false)) + ok(false) else: err(rc.error) @@ -155,8 +155,11 @@ proc makeAccountProof*( db: AristoTxRef; accPath: Hash32; ): Result[(seq[seq[byte]], bool), AristoError] = - var nodesCache: NodesCache - db.makeProof(STATE_ROOT_VID, NibblesBuf.fromBytes accPath.data, nodesCache) + var + nodesCache: NodesCache + proof: seq[seq[byte]] + let exists = ?db.makeProof(STATE_ROOT_VID, NibblesBuf.fromBytes accPath.data, nodesCache, proof) + ok((proof, exists)) proc makeStorageProof*( db: AristoTxRef; @@ -169,14 +172,16 @@ proc makeStorageProof*( if error == FetchPathStoRootMissing: return ok((@[],false)) return err(error) - var nodesCache: NodesCache - db.makeProof(vid, NibblesBuf.fromBytes stoPath.data, nodesCache) + var + nodesCache: NodesCache + proof: seq[seq[byte]] + let exists = ?db.makeProof(vid, NibblesBuf.fromBytes stoPath.data, nodesCache, proof) + ok((proof, exists)) -proc makeStorageProofs( +proc makeStorageProofs*( db: AristoTxRef; accPath: Hash32; stoPaths: openArray[Hash32]; - nodesCache: var NodesCache; ): Result[seq[seq[seq[byte]]], AristoError] = ## Note that the function returns an error unless ## the argument `accPath` is valid. @@ -186,41 +191,59 @@ proc makeStorageProofs( return ok(emptyProofs) return err(error) - var proofs = newSeqOfCap[seq[seq[byte]]](stoPaths.len()) + var + nodesCache: NodesCache + proofs = newSeqOfCap[seq[seq[byte]]](stoPaths.len()) for stoPath in stoPaths: - let (proof, _) = ?db.makeProof(vid, NibblesBuf.fromBytes stoPath.data, nodesCache) + var proof: seq[seq[byte]] + discard ?db.makeProof(vid, NibblesBuf.fromBytes stoPath.data, nodesCache, proof) proofs.add(proof) ok(proofs) -proc makeStorageProofs*( +proc makeStorageMultiProof( db: AristoTxRef; accPath: Hash32; stoPaths: openArray[Hash32]; - ): Result[seq[seq[seq[byte]]], AristoError] = - var nodesCache: NodesCache - makeStorageProofs(db, accPath, stoPaths, nodesCache) + nodesCache: var NodesCache; + multiProof: var HashSet[seq[byte]] + ): Result[void, AristoError] = + ## Note that the function returns an error unless + ## the argument `accPath` is valid. + let vid = db.fetchStorageID(accPath).valueOr: + if error == FetchPathStoRootMissing: + return ok() + return err(error) + + for stoPath in stoPaths: + var proof: seq[seq[byte]] + discard ?db.makeProof(vid, NibblesBuf.fromBytes stoPath.data, nodesCache, proof) + for node in proof: + multiProof.incl(node) + + ok() proc makeMultiProof*( db: AristoTxRef; - paths: Table[Hash32, seq[Hash32]] # maps each account path to a list of storage paths - ): Result[seq[seq[byte]], AristoError] = + paths: Table[Hash32, seq[Hash32]], # maps each account path to a list of storage paths + multiProof: var seq[seq[byte]] + ): Result[void, AristoError] = var nodesCache: NodesCache - multiProof: HashSet[seq[byte]] + proofNodes: HashSet[seq[byte]] for accPath, stoPaths in paths: - let (accProof, exists) = ?db.makeProof(STATE_ROOT_VID, NibblesBuf.fromBytes accPath.data, nodesCache) + var accProof: seq[seq[byte]] + let exists = ?db.makeProof(STATE_ROOT_VID, NibblesBuf.fromBytes accPath.data, nodesCache, accProof) for node in accProof: - multiProof.incl(node) + proofNodes.incl(node) if exists: - let storageProofs = ?db.makeStorageProofs(accPath, stoPaths, nodesCache) - for storageProof in storageProofs: - for node in storageProof: - multiProof.incl(node) + ?db.makeStorageMultiProof(accPath, stoPaths, nodesCache, proofNodes) + + multiProof = proofNodes.toSeq() - ok(multiProof.toSeq()) + ok() proc verifyProof*( chain: openArray[seq[byte]]; diff --git a/execution_chain/db/core_db/base.nim b/execution_chain/db/core_db/base.nim index 011fc3ffc2..8715cd9503 100644 --- a/execution_chain/db/core_db/base.nim +++ b/execution_chain/db/core_db/base.nim @@ -310,15 +310,16 @@ proc getStateRoot*(acc: CoreDbTxRef): CoreDbRc[Hash32] = proc multiProof*( acc: CoreDbTxRef; paths: Table[Hash32, seq[Hash32]]; - ): CoreDbRc[seq[seq[byte]]] = + multiProof: var seq[seq[byte]] + ): CoreDbRc[void] = ## Returns a multiproof for every account and storage path specified ## in the paths table. All rlp-encoded trie nodes from all account ## and storage proofs are returned in a single list. - let rc = acc.aTx.makeMultiProof(paths).valueOr: + acc.aTx.makeMultiProof(paths, multiProof).isOkOr: return err(error.toError("", ProofCreate)) - ok(rc) + ok() # ------------ storage --------------- diff --git a/execution_chain/stateless/witness_generation.nim b/execution_chain/stateless/witness_generation.nim index 2e326f6be3..824093eaee 100644 --- a/execution_chain/stateless/witness_generation.nim +++ b/execution_chain/stateless/witness_generation.nim @@ -64,9 +64,11 @@ proc build*( paths.add(slotPath) proofPaths[accPath] = paths - witness.state = preStateLedger.txFrame.multiProof(proofPaths).valueOr: + var multiProof: seq[seq[byte]] + preStateLedger.txFrame.multiProof(proofPaths, multiProof).isOkOr: raiseAssert "Failed to get multiproof: " & $$error + witness.state = move(multiProof) witness proc getEarliestCachedBlockNumber(blockHashes: BlockHashesCache): Opt[BlockNumber] = From fca6fe9119e62d276ace413a7ac3e2a0db763c69 Mon Sep 17 00:00:00 2001 From: bhartnett Date: Wed, 13 Aug 2025 14:38:18 +0800 Subject: [PATCH 09/11] Fix order of keys. --- execution_chain/stateless/witness_generation.nim | 14 ++++++++++---- tests/test_stateless_witness_generation.nim | 10 +++++----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/execution_chain/stateless/witness_generation.nim b/execution_chain/stateless/witness_generation.nim index 824093eaee..e622c2991e 100644 --- a/execution_chain/stateless/witness_generation.nim +++ b/execution_chain/stateless/witness_generation.nim @@ -28,16 +28,16 @@ proc build*( var proofPaths: Table[Hash32, seq[Hash32]] addedCodeHashes: HashSet[Hash32] + pathToKeys: Table[Hash32, seq[byte]] witness = Witness.init() for key, codeTouched in witnessKeys: let addressBytes = key.address.data() accPath = keccak256(addressBytes) + pathToKeys[accPath] = @addressBytes if key.slot.isNone(): # Is an account key - witness.addKey(key.address.data()) - proofPaths.withValue(accPath, v): discard do: @@ -54,8 +54,7 @@ proc build*( let slotBytes = key.slot.get().toBytesBE() slotPath = keccak256(slotBytes) - - witness.addKey(slotBytes) + pathToKeys[slotPath] = @slotBytes proofPaths.withValue(accPath, v): v[].add(slotPath) @@ -68,7 +67,14 @@ proc build*( preStateLedger.txFrame.multiProof(proofPaths, multiProof).isOkOr: raiseAssert "Failed to get multiproof: " & $$error + var keys: seq[seq[byte]] + for accPath, stoPaths in proofPaths: + keys.add(pathToKeys.getOrDefault(accPath)) + for stoPath in stoPaths: + keys.add(pathToKeys.getOrDefault(stoPath)) + witness.state = move(multiProof) + witness.keys = move(keys) witness proc getEarliestCachedBlockNumber(blockHashes: BlockHashesCache): Opt[BlockNumber] = diff --git a/tests/test_stateless_witness_generation.nim b/tests/test_stateless_witness_generation.nim index 9dd7623235..7a1c54790e 100644 --- a/tests/test_stateless_witness_generation.nim +++ b/tests/test_stateless_witness_generation.nim @@ -144,8 +144,8 @@ suite "Stateless: Witness Generation": check: witness.keys.len() == 5 - witness.keys[0] == addr1.data() - witness.keys[1] == slot1.toBytesBE() - witness.keys[2] == slot2.toBytesBE() - witness.keys[3] == slot3.toBytesBE() - witness.keys[4] == addr2.data() + witness.keys[0] == addr2.data() + witness.keys[1] == addr1.data() + witness.keys[2] == slot1.toBytesBE() + witness.keys[3] == slot2.toBytesBE() + witness.keys[4] == slot3.toBytesBE() From 11cb9e3c550a7e39ebd3abd319e435792576804c Mon Sep 17 00:00:00 2001 From: bhartnett Date: Wed, 13 Aug 2025 20:00:34 +0800 Subject: [PATCH 10/11] Use separate account and storage preimages tables. --- execution_chain/stateless/witness_generation.nim | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/execution_chain/stateless/witness_generation.nim b/execution_chain/stateless/witness_generation.nim index e622c2991e..52f97d8233 100644 --- a/execution_chain/stateless/witness_generation.nim +++ b/execution_chain/stateless/witness_generation.nim @@ -25,17 +25,19 @@ proc build*( T: type Witness, witnessKeys: WitnessTable, preStateLedger: LedgerRef): T = + var proofPaths: Table[Hash32, seq[Hash32]] addedCodeHashes: HashSet[Hash32] - pathToKeys: Table[Hash32, seq[byte]] + accPreimages: Table[Hash32, array[20, byte]] + stoPreimages: Table[Hash32, array[32, byte]] witness = Witness.init() for key, codeTouched in witnessKeys: let addressBytes = key.address.data() accPath = keccak256(addressBytes) - pathToKeys[accPath] = @addressBytes + accPreimages[accPath] = addressBytes if key.slot.isNone(): # Is an account key proofPaths.withValue(accPath, v): @@ -54,7 +56,7 @@ proc build*( let slotBytes = key.slot.get().toBytesBE() slotPath = keccak256(slotBytes) - pathToKeys[slotPath] = @slotBytes + stoPreimages[slotPath] = slotBytes proofPaths.withValue(accPath, v): v[].add(slotPath) @@ -66,15 +68,13 @@ proc build*( var multiProof: seq[seq[byte]] preStateLedger.txFrame.multiProof(proofPaths, multiProof).isOkOr: raiseAssert "Failed to get multiproof: " & $$error + witness.state = move(multiProof) - var keys: seq[seq[byte]] for accPath, stoPaths in proofPaths: - keys.add(pathToKeys.getOrDefault(accPath)) + witness.addKey(accPreimages.getOrDefault(accPath)) for stoPath in stoPaths: - keys.add(pathToKeys.getOrDefault(stoPath)) + witness.addKey(stoPreimages.getOrDefault(stoPath)) - witness.state = move(multiProof) - witness.keys = move(keys) witness proc getEarliestCachedBlockNumber(blockHashes: BlockHashesCache): Opt[BlockNumber] = From 985eb9ad6c7e7feb1cf4d1964e3c036418dcfce9 Mon Sep 17 00:00:00 2001 From: bhartnett Date: Wed, 13 Aug 2025 23:19:34 +0800 Subject: [PATCH 11/11] Fix possible underflow when processing block hashes. --- execution_chain/stateless/witness_generation.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/execution_chain/stateless/witness_generation.nim b/execution_chain/stateless/witness_generation.nim index 52f97d8233..522eaf1360 100644 --- a/execution_chain/stateless/witness_generation.nim +++ b/execution_chain/stateless/witness_generation.nim @@ -105,10 +105,11 @@ proc build*( let blockHashes = ledger.getBlockHashesCache() earliestBlockNumber = getEarliestCachedBlockNumber(blockHashes) + if earliestBlockNumber.isSome(): - var n = parent.number - 1 + var n = parent.number while n >= earliestBlockNumber.get(): + dec n let blockHash = ledger.getBlockHash(BlockNumber(n)) doAssert(blockHash != default(Hash32)) witness.addHeaderHash(blockHash) - dec n