|
| 1 | +# Fluffy |
| 2 | +# Copyright (c) 2025 Status Research & Development GmbH |
| 3 | +# Licensed and distributed under either of |
| 4 | +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). |
| 5 | +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). |
| 6 | +# at your option. This file may not be copied, modified, or distributed except according to those terms. |
| 7 | + |
| 8 | +{.push raises: [].} |
| 9 | + |
| 10 | +import |
| 11 | + std/sets, |
| 12 | + stew/byteutils, |
| 13 | + chronos, |
| 14 | + chronicles, |
| 15 | + stint, |
| 16 | + results, |
| 17 | + eth/common/[hashes, addresses, accounts, headers], |
| 18 | + ../../execution_chain/db/ledger, |
| 19 | + ../../execution_chain/common/common, |
| 20 | + ../../execution_chain/transaction/call_evm, |
| 21 | + ../../execution_chain/evm/[types, state, evm_errors], |
| 22 | + ../network/history/history_network, |
| 23 | + ../network/state/[state_endpoints, state_network] |
| 24 | + |
| 25 | +from web3/eth_api_types import TransactionArgs |
| 26 | + |
| 27 | +export |
| 28 | + results, chronos, hashes, history_network, state_network, TransactionArgs, CallResult |
| 29 | + |
| 30 | +logScope: |
| 31 | + topics = "portal_evm" |
| 32 | + |
| 33 | +# The Portal EVM uses the Nimbus in-memory EVM to execute transactions using the |
| 34 | +# portal state network state data. Currently only call is supported. |
| 35 | +# |
| 36 | +# Rather than wire in the portal state lookups into the EVM directly, the approach |
| 37 | +# taken here is to optimistically execute the transaction multiple times with the |
| 38 | +# goal of building the correct access list so that we can then lookup the accessed |
| 39 | +# state from the portal network, store the state in the in-memory EVM and then |
| 40 | +# finally execute the transaction using the correct state. The Portal EVM makes |
| 41 | +# use of data in memory during the call and therefore each piece of state is never |
| 42 | +# fetched more than once. We know we have found the correct access list if it |
| 43 | +# doesn't change after another execution of the transaction. |
| 44 | +# |
| 45 | +# The assumption here is that network lookups for state data are generally much |
| 46 | +# slower than the time it takes to execute a transaction in the EVM and therefore |
| 47 | +# executing the transaction multiple times should not significally slow down the |
| 48 | +# call given that we gain the ability to fetch the state concurrently. |
| 49 | +# |
| 50 | +# There are multiple reasons for choosing this approach: |
| 51 | +# - Firstly updating the existing Nimbus EVM to support using a different state |
| 52 | +# backend (portal state in this case) is difficult and would require making |
| 53 | +# non-trivial changes to the EVM. |
| 54 | +# - This new approach allows us to look up the state concurrently in the event that |
| 55 | +# multiple new state keys are discovered after executing the transaction. This |
| 56 | +# should in theory result in improved performance for certain scenarios. The |
| 57 | +# default approach where the state lookups are wired directly into the EVM gives |
| 58 | +# the worst case performance because all state accesses inside the EVM are |
| 59 | +# completely sequential. |
| 60 | + |
| 61 | +const EVM_CALL_LIMIT = 10000 |
| 62 | + |
| 63 | +type |
| 64 | + AccountQuery = object |
| 65 | + address: Address |
| 66 | + accFut: Future[Opt[Account]] |
| 67 | + |
| 68 | + StorageQuery = object |
| 69 | + address: Address |
| 70 | + slotKey: UInt256 |
| 71 | + storageFut: Future[Opt[UInt256]] |
| 72 | + |
| 73 | + CodeQuery = object |
| 74 | + address: Address |
| 75 | + codeFut: Future[Opt[Bytecode]] |
| 76 | + |
| 77 | + PortalEvm* = ref object |
| 78 | + historyNetwork: HistoryNetwork |
| 79 | + stateNetwork: StateNetwork |
| 80 | + com: CommonRef |
| 81 | + |
| 82 | +func init(T: type AccountQuery, adr: Address, fut: Future[Opt[Account]]): T = |
| 83 | + T(address: adr, accFut: fut) |
| 84 | + |
| 85 | +func init( |
| 86 | + T: type StorageQuery, adr: Address, slotKey: UInt256, fut: Future[Opt[UInt256]] |
| 87 | +): T = |
| 88 | + T(address: adr, slotKey: slotKey, storageFut: fut) |
| 89 | + |
| 90 | +func init(T: type CodeQuery, adr: Address, fut: Future[Opt[Bytecode]]): T = |
| 91 | + T(address: adr, codeFut: fut) |
| 92 | + |
| 93 | +proc init*(T: type PortalEvm, hn: HistoryNetwork, sn: StateNetwork): T = |
| 94 | + let config = |
| 95 | + try: |
| 96 | + networkParams(MainNet).config |
| 97 | + except ValueError as e: |
| 98 | + raiseAssert(e.msg) # Should not fail |
| 99 | + except RlpError as e: |
| 100 | + raiseAssert(e.msg) # Should not fail |
| 101 | + |
| 102 | + let com = CommonRef.new( |
| 103 | + DefaultDbMemory.newCoreDbRef(), |
| 104 | + taskpool = nil, |
| 105 | + config = config, |
| 106 | + initializeDb = false, |
| 107 | + ) |
| 108 | + |
| 109 | + PortalEvm(historyNetwork: hn, stateNetwork: sn, com: com) |
| 110 | + |
| 111 | +proc call*( |
| 112 | + evm: PortalEvm, |
| 113 | + tx: TransactionArgs, |
| 114 | + blockNumOrHash: uint64 | Hash32, |
| 115 | + optimisticStateFetch = true, |
| 116 | +): Future[Result[CallResult, string]] {.async: (raises: [CancelledError]).} = |
| 117 | + let |
| 118 | + to = tx.to.valueOr: |
| 119 | + return err("to address is required") |
| 120 | + header = (await evm.historyNetwork.getVerifiedBlockHeader(blockNumOrHash)).valueOr: |
| 121 | + return err("Unable to get block header") |
| 122 | + # Start fetching code in the background while setting up the EVM |
| 123 | + codeFut = evm.stateNetwork.getCodeByStateRoot(header.stateRoot, to) |
| 124 | + |
| 125 | + debug "Executing call", to, blockNumOrHash |
| 126 | + |
| 127 | + let txFrame = evm.com.db.baseTxFrame().txFrameBegin() |
| 128 | + defer: |
| 129 | + txFrame.dispose() # always dispose state changes |
| 130 | + |
| 131 | + # TODO: review what child header to use here (second parameter) |
| 132 | + let vmState = BaseVMState.new(header, header, evm.com, txFrame) |
| 133 | + |
| 134 | + var |
| 135 | + # Record the keys of fetched accounts, storage and code so that we don't |
| 136 | + # bother to fetch them multiple times |
| 137 | + fetchedAccounts = initHashSet[Address]() |
| 138 | + fetchedStorage = initHashSet[(Address, UInt256)]() |
| 139 | + fetchedCode = initHashSet[Address]() |
| 140 | +
|
| 141 | + # Set code of the 'to' address in the EVM so that we can execute the transaction |
| 142 | + let code = (await codeFut).valueOr: |
| 143 | + return err("Unable to get code") |
| 144 | + vmState.ledger.setCode(to, code.asSeq()) |
| 145 | + fetchedCode.incl(to) |
| 146 | + debug "Code to be executed", code = code.asSeq().to0xHex() |
| 147 | +
|
| 148 | + var |
| 149 | + lastWitnessKeys: WitnessTable |
| 150 | + witnessKeys = vmState.ledger.getWitnessKeys() |
| 151 | + callResult: EvmResult[CallResult] |
| 152 | + evmCallCount = 0 |
| 153 | +
|
| 154 | + # Limit the max number of calls to prevent infinite loops and/or DOS in the |
| 155 | + # event of a bug in the implementation. |
| 156 | + while evmCallCount < EVM_CALL_LIMIT: |
| 157 | + debug "Starting PortalEvm execution", evmCallCount |
| 158 | +
|
| 159 | + let sp = vmState.ledger.beginSavepoint() |
| 160 | + callResult = rpcCallEvm(tx, header, vmState) |
| 161 | + inc evmCallCount |
| 162 | + vmState.ledger.rollback(sp) # all state changes from the call are reverted |
| 163 | +
|
| 164 | + # Collect the keys after executing the transaction |
| 165 | + lastWitnessKeys = ensureMove(witnessKeys) |
| 166 | + witnessKeys = vmState.ledger.getWitnessKeys() |
| 167 | + vmState.ledger.clearWitnessKeys() |
| 168 | +
|
| 169 | + try: |
| 170 | + var |
| 171 | + accountQueries = newSeq[AccountQuery]() |
| 172 | + storageQueries = newSeq[StorageQuery]() |
| 173 | + codeQueries = newSeq[CodeQuery]() |
| 174 | +
|
| 175 | + # Loop through the collected keys and fetch the state concurrently. |
| 176 | + # If optimisticStateFetch is enabled then we fetch state for all the witness |
| 177 | + # keys and await all queries before continuing to the next call. |
| 178 | + # If optimisticStateFetch is disabled then we only fetch and then await on |
| 179 | + # one piece of state (the next in the ordered witness keys) while the remaining |
| 180 | + # state queries are still issued in the background just incase the state is |
| 181 | + # needed in the next iteration. |
| 182 | + var stateFetchDone = false |
| 183 | + for k, v in witnessKeys: |
| 184 | + let (adr, _) = k |
| 185 | +
|
| 186 | + if v.storageMode: |
| 187 | + let slotIdx = (adr, v.storageSlot) |
| 188 | + if slotIdx notin fetchedStorage: |
| 189 | + debug "Fetching storage slot", address = adr, slotKey = v.storageSlot |
| 190 | + let storageFut = evm.stateNetwork.getStorageAtByStateRoot( |
| 191 | + header.stateRoot, adr, v.storageSlot |
| 192 | + ) |
| 193 | + if not stateFetchDone: |
| 194 | + storageQueries.add(StorageQuery.init(adr, v.storageSlot, storageFut)) |
| 195 | + if not optimisticStateFetch: |
| 196 | + stateFetchDone = true |
| 197 | + elif adr != default(Address): |
| 198 | + doAssert(adr == v.address) |
| 199 | +
|
| 200 | + if adr notin fetchedAccounts: |
| 201 | + debug "Fetching account", address = adr |
| 202 | + let accFut = evm.stateNetwork.getAccount(header.stateRoot, adr) |
| 203 | + if not stateFetchDone: |
| 204 | + accountQueries.add(AccountQuery.init(adr, accFut)) |
| 205 | + if not optimisticStateFetch: |
| 206 | + stateFetchDone = true |
| 207 | +
|
| 208 | + if v.codeTouched and adr notin fetchedCode: |
| 209 | + debug "Fetching code", address = adr |
| 210 | + let codeFut = evm.stateNetwork.getCodeByStateRoot(header.stateRoot, adr) |
| 211 | + if not stateFetchDone: |
| 212 | + codeQueries.add(CodeQuery.init(adr, codeFut)) |
| 213 | + if not optimisticStateFetch: |
| 214 | + stateFetchDone = true |
| 215 | +
|
| 216 | + if optimisticStateFetch: |
| 217 | + # If the witness keys did not change after the last execution then we can |
| 218 | + # stop the execution loop because we have already executed the transaction |
| 219 | + # with the correct state. |
| 220 | + if lastWitnessKeys == witnessKeys: |
| 221 | + break |
| 222 | + else: |
| 223 | + # When optimisticStateFetch is disabled and stateFetchDone is not set then |
| 224 | + # we know that all the state has already been fetched in the last iteration |
| 225 | + # of the loop and therefore we have already executed the transaction with |
| 226 | + # the correct state. |
| 227 | + if not stateFetchDone: |
| 228 | + break |
| 229 | +
|
| 230 | + # Store fetched state in the in-memory EVM |
| 231 | + for q in accountQueries: |
| 232 | + let acc = (await q.accFut).valueOr: |
| 233 | + return err("Unable to get account") |
| 234 | + vmState.ledger.setBalance(q.address, acc.balance) |
| 235 | + vmState.ledger.setNonce(q.address, acc.nonce) |
| 236 | + fetchedAccounts.incl(q.address) |
| 237 | +
|
| 238 | + for q in storageQueries: |
| 239 | + let slotValue = (await q.storageFut).valueOr: |
| 240 | + return err("Unable to get slot") |
| 241 | + vmState.ledger.setStorage(q.address, q.slotKey, slotValue) |
| 242 | + fetchedStorage.incl((q.address, q.slotKey)) |
| 243 | +
|
| 244 | + for q in codeQueries: |
| 245 | + let code = (await q.codeFut).valueOr: |
| 246 | + return err("Unable to get code") |
| 247 | + vmState.ledger.setCode(q.address, code.asSeq()) |
| 248 | + fetchedCode.incl(q.address) |
| 249 | + except CatchableError as e: |
| 250 | + # TODO: why do the above futures throw a CatchableError and not CancelledError? |
| 251 | + raiseAssert(e.msg) |
| 252 | +
|
| 253 | + callResult.mapErr( |
| 254 | + proc(e: EvmErrorObj): string = |
| 255 | + "EVM execution failed: " & $e.code |
| 256 | + ) |
0 commit comments